great-cto 3.25.0 → 3.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/board/.claude-plugin/plugin.json +1 -1
- package/board/packages/board/lib/routes.mjs +120 -0
- package/board/packages/board/public/index.html +115 -0
- package/board/scripts/lib/codex-exec.mjs +167 -0
- package/board/scripts/lib/dag-metric.mjs +297 -0
- package/board/scripts/lib/second-opinion.mjs +325 -0
- package/board/scripts/lib/stack-capabilities.mjs +206 -0
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "great-cto",
|
|
3
3
|
"description": "You already have the agent. This is everything around it. great_cto runs Claude Code as a pipeline of 70 specialist agents \u2014 an independent model checks each stage before the next builds on it, spending caps refuse rather than warn, and three decisions stay yours: what gets built, how, and whether it ships.",
|
|
4
|
-
"version": "3.
|
|
4
|
+
"version": "3.26.0",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Alexander Velikiy",
|
|
7
7
|
"url": "https://hashnode.com/@Greatcto"
|
|
@@ -21,6 +21,9 @@ import { bdCacheInvalidate, checkBeadsAvailable, bdWriteSerialised, bd, bdErr, g
|
|
|
21
21
|
import { getMetrics } from './metrics.mjs';
|
|
22
22
|
import { readVerdicts } from './verdicts.mjs';
|
|
23
23
|
import { parseAgentBudgets, upsertAgentBudget, removeAgentBudget } from '../../../scripts/lib/agent-budget.mjs';
|
|
24
|
+
import { resolveSecondOpinion, SECOND_OPINION_PROVIDERS } from '../../../scripts/lib/second-opinion.mjs';
|
|
25
|
+
import { detectCodex } from '../../../scripts/lib/codex-exec.mjs';
|
|
26
|
+
import { upsertCapability, capabilitiesFromProjectMd } from '../../../scripts/lib/stack-capabilities.mjs';
|
|
24
27
|
import { getAgentsFleet, getAgentProfile, retireAgent, restoreAgent, appendDecisionLog, readDecisionsLog } from './fleet.mjs';
|
|
25
28
|
import { getResume, getShareState, toggleShare } from './share.mjs';
|
|
26
29
|
import { listSessions, readSession, editedFiles, searchSessions } from './transcripts.mjs';
|
|
@@ -1595,6 +1598,123 @@ async function dispatch(req, res, url, cwd) {
|
|
|
1595
1598
|
|
|
1596
1599
|
// Set or clear one agent's spending cap, by writing PROJECT.md.
|
|
1597
1600
|
//
|
|
1601
|
+
// GET /api/harnesses — the two harnesses and which one gives the second opinion.
|
|
1602
|
+
//
|
|
1603
|
+
// Claude Code is the host: it runs the pipeline, so it is always "here". Codex
|
|
1604
|
+
// is detected, not assumed — version, login and the model it would run, read
|
|
1605
|
+
// from disk — and reported in three states, because a card that shows an
|
|
1606
|
+
// absent Codex as a quiet toggle in the off position is the same defect as a
|
|
1607
|
+
// pending gate that renders as passed. The evidence section is the tail of
|
|
1608
|
+
// `.great_cto/cross-review.log`: what the second opinion DID, not what it is
|
|
1609
|
+
// set to.
|
|
1610
|
+
if (pathname === '/api/harnesses' && req.method === 'GET') {
|
|
1611
|
+
const c = url.searchParams.get('project') ? resolveProjectCwd(url.searchParams.get('project')) : cwd;
|
|
1612
|
+
const mdPath = path.join(c, '.great_cto', 'PROJECT.md');
|
|
1613
|
+
let projectMd = null;
|
|
1614
|
+
try { projectMd = fs.readFileSync(mdPath, 'utf8'); } catch { projectMd = null; }
|
|
1615
|
+
const codex = detectCodex();
|
|
1616
|
+
const second = resolveSecondOpinion({ projectMd: projectMd ?? '', codex });
|
|
1617
|
+
const declared = projectMd == null ? null : capabilitiesFromProjectMd(projectMd).map.second_opinion;
|
|
1618
|
+
|
|
1619
|
+
// Last 20 review lines, newest first; unparseable lines are counted, not
|
|
1620
|
+
// dropped, so a corrupted log does not read as a quiet one.
|
|
1621
|
+
let evidence = [], unreadable = 0, logState = 'absent';
|
|
1622
|
+
try {
|
|
1623
|
+
const raw = fs.readFileSync(path.join(c, '.great_cto', 'cross-review.log'), 'utf8');
|
|
1624
|
+
logState = 'ok';
|
|
1625
|
+
for (const line of raw.trim().split('\n').filter(Boolean)) {
|
|
1626
|
+
try { evidence.push(JSON.parse(line)); } catch { unreadable += 1; }
|
|
1627
|
+
}
|
|
1628
|
+
evidence = evidence.slice(-20).reverse();
|
|
1629
|
+
} catch (e) { logState = e.code === 'ENOENT' ? 'absent' : 'unreadable'; }
|
|
1630
|
+
const reviewed = evidence.filter((r) => r.state === 'ok');
|
|
1631
|
+
const summary = {
|
|
1632
|
+
runs: evidence.length,
|
|
1633
|
+
reviewed: reviewed.length,
|
|
1634
|
+
skipped: evidence.filter((r) => r.state !== 'ok').length,
|
|
1635
|
+
blocked: reviewed.filter((r) => r.verdict === 'BLOCK').length,
|
|
1636
|
+
unreadable_lines: unreadable,
|
|
1637
|
+
};
|
|
1638
|
+
|
|
1639
|
+
res.writeHead(200, verdictHeaders(c));
|
|
1640
|
+
res.end(JSON.stringify({
|
|
1641
|
+
claude_code: { state: 'host', version: BUILD_VERSION },
|
|
1642
|
+
codex,
|
|
1643
|
+
second_opinion: {
|
|
1644
|
+
...second,
|
|
1645
|
+
declared: declared ?? { state: projectMd == null ? 'no-project-md' : 'undeclared', tool: null },
|
|
1646
|
+
providers: SECOND_OPINION_PROVIDERS,
|
|
1647
|
+
},
|
|
1648
|
+
evidence: { state: logState, summary, recent: evidence },
|
|
1649
|
+
}));
|
|
1650
|
+
return true;
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
// POST /api/harnesses/second-opinion { provider: codex|openrouter|none|null }
|
|
1654
|
+
//
|
|
1655
|
+
// Writes `capabilities: second_opinion:` into PROJECT.md — a file the operator
|
|
1656
|
+
// owns and git tracks — from a browser, behind the same origin gate as
|
|
1657
|
+
// /api/agent-budgets. `null` removes the key: back to UNDECLARED, which is a
|
|
1658
|
+
// different state from `none` and is offered on purpose.
|
|
1659
|
+
//
|
|
1660
|
+
// The reply carries the previous value, and the resolved state AFTER the
|
|
1661
|
+
// write: choosing codex on a machine without one succeeds as a declaration
|
|
1662
|
+
// and comes back `unavailable`, so the card can say so at the moment of the
|
|
1663
|
+
// click rather than on the next review.
|
|
1664
|
+
if (pathname === '/api/harnesses/second-opinion' && req.method === 'POST') {
|
|
1665
|
+
if (!originAllowed(req)) {
|
|
1666
|
+
res.writeHead(403, { 'Content-Type': 'application/json' });
|
|
1667
|
+
res.end(JSON.stringify({ error: 'origin not allowed' }));
|
|
1668
|
+
return true;
|
|
1669
|
+
}
|
|
1670
|
+
let body = '';
|
|
1671
|
+
req.on('data', (ch) => { body += ch; if (body.length > 1024) req.destroy(); });
|
|
1672
|
+
req.on('end', () => {
|
|
1673
|
+
let parsed;
|
|
1674
|
+
try { parsed = JSON.parse(body || '{}'); }
|
|
1675
|
+
catch (e) {
|
|
1676
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1677
|
+
res.end(JSON.stringify({ error: 'invalid_json', message: String(e.message || e) }));
|
|
1678
|
+
return;
|
|
1679
|
+
}
|
|
1680
|
+
const provider = parsed.provider == null ? null : String(parsed.provider).toLowerCase();
|
|
1681
|
+
if (provider != null && !SECOND_OPINION_PROVIDERS.includes(provider)) {
|
|
1682
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1683
|
+
res.end(JSON.stringify({ error: `provider must be one of ${SECOND_OPINION_PROVIDERS.join(', ')}, or null to undeclare` }));
|
|
1684
|
+
return;
|
|
1685
|
+
}
|
|
1686
|
+
const c = url.searchParams.get('project') ? resolveProjectCwd(url.searchParams.get('project')) : cwd;
|
|
1687
|
+
const mdPath = path.join(c, '.great_cto', 'PROJECT.md');
|
|
1688
|
+
let before;
|
|
1689
|
+
try { before = fs.readFileSync(mdPath, 'utf8'); }
|
|
1690
|
+
catch (e) {
|
|
1691
|
+
res.writeHead(409, { 'Content-Type': 'application/json' });
|
|
1692
|
+
res.end(JSON.stringify({ error: 'no PROJECT.md to write to', detail: String(e.message || e) }));
|
|
1693
|
+
return;
|
|
1694
|
+
}
|
|
1695
|
+
let out;
|
|
1696
|
+
try { out = upsertCapability(before, 'second_opinion', provider); }
|
|
1697
|
+
catch (e) {
|
|
1698
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
1699
|
+
res.end(JSON.stringify({ error: String(e.message || e) }));
|
|
1700
|
+
return;
|
|
1701
|
+
}
|
|
1702
|
+
try {
|
|
1703
|
+
const tmp = `${mdPath}.tmp-${process.pid}`;
|
|
1704
|
+
fs.writeFileSync(tmp, out.text);
|
|
1705
|
+
fs.renameSync(tmp, mdPath);
|
|
1706
|
+
} catch (e) {
|
|
1707
|
+
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
1708
|
+
res.end(JSON.stringify({ error: `could not write PROJECT.md: ${String(e.message || e)}` }));
|
|
1709
|
+
return;
|
|
1710
|
+
}
|
|
1711
|
+
const resolved = resolveSecondOpinion({ projectMd: out.text, codex: detectCodex() });
|
|
1712
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
1713
|
+
res.end(JSON.stringify({ ok: true, provider, previous: out.previous, created_block: out.created === true, resolved }));
|
|
1714
|
+
});
|
|
1715
|
+
return true;
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1598
1718
|
// POST /api/agent-budgets { agent, limit_usd } set / replace
|
|
1599
1719
|
// POST /api/agent-budgets { agent, remove: true } clear
|
|
1600
1720
|
//
|
|
@@ -1419,6 +1419,20 @@ h1, h2, h3, h4 { font-weight: 600; font-family: var(--sans); }
|
|
|
1419
1419
|
}
|
|
1420
1420
|
|
|
1421
1421
|
/* ── change_tier badge ────────────────────────────────────────────────────── */
|
|
1422
|
+
/* `.warn` was used by the Harnesses card before it existed: five spans that
|
|
1423
|
+
said "declared but unavailable" and rendered as ordinary prose. A warning
|
|
1424
|
+
that does not look like a warning is this repository's own defect class,
|
|
1425
|
+
and the css-tokens guard could not see it — it checks var() tokens, not
|
|
1426
|
+
class names. Declared here, and asserted by the harnesses UI test. */
|
|
1427
|
+
.warn { color: var(--status-blocked); }
|
|
1428
|
+
/* `.muted` is used 68 times across this board and was declared nowhere, so
|
|
1429
|
+
every de-emphasised line rendered at full weight — found by the guard added
|
|
1430
|
+
for the Harnesses card, not by the eye. One declaration, the same token the
|
|
1431
|
+
rest of the board de-emphasises with. */
|
|
1432
|
+
.muted { color: var(--text3); }
|
|
1433
|
+
/* Same shape as `.muted`: the `--mono` token exists and is used 115×, the
|
|
1434
|
+
CLASS that applies it did not. */
|
|
1435
|
+
.mono { font-family: var(--mono); }
|
|
1422
1436
|
.tier-badge { font-size: var(--fs-caption); padding: 1px 8px; border-radius: 999px; margin-left: 6px;
|
|
1423
1437
|
font-family: var(--mono); white-space: nowrap; }
|
|
1424
1438
|
.tier-badge:empty { display: none; }
|
|
@@ -3174,9 +3188,24 @@ id var(--border);
|
|
|
3174
3188
|
<h3>Active pipeline</h3>
|
|
3175
3189
|
<span class="inbox-count" id="pipeline-status">idle</span>
|
|
3176
3190
|
<span class="tier-badge" id="tier-badge" title="change_tier for the current working-tree diff — which gates + judge open (ADR-003/004)"></span>
|
|
3191
|
+
<span class="tier-badge" id="harness-badge" title="which harness gives the second opinion in review — from capabilities: second_opinion in PROJECT.md"></span>
|
|
3177
3192
|
</div>
|
|
3178
3193
|
<div class="pipeline-track" id="pipeline-track"></div>
|
|
3179
3194
|
</div>
|
|
3195
|
+
|
|
3196
|
+
<!-- Harnesses: the host that runs the pipeline, and the second harness
|
|
3197
|
+
that reviews beside it. Codex is DETECTED, in three states — an
|
|
3198
|
+
absent Codex must not render as a switch in the off position. The
|
|
3199
|
+
evidence rows are the tail of .great_cto/cross-review.log: what the
|
|
3200
|
+
second opinion did, not what it is set to. -->
|
|
3201
|
+
<div class="inbox-section" id="harnesses-section">
|
|
3202
|
+
<div class="inbox-section-head">
|
|
3203
|
+
<span class="dot dot-blue"></span>
|
|
3204
|
+
<h3>Harnesses</h3>
|
|
3205
|
+
<span class="inbox-count" id="harnesses-status">…</span>
|
|
3206
|
+
</div>
|
|
3207
|
+
<div id="harnesses-body" class="muted" style="font-size:13px;line-height:1.5">loading…</div>
|
|
3208
|
+
</div>
|
|
3180
3209
|
</div>
|
|
3181
3210
|
</details>
|
|
3182
3211
|
|
|
@@ -5106,9 +5135,95 @@ async function refreshPipeline() {
|
|
|
5106
5135
|
const d = await api(`/api/pipeline${pqs()}`);
|
|
5107
5136
|
if (Array.isArray(d)) renderPipeline(d);
|
|
5108
5137
|
renderTierBadge();
|
|
5138
|
+
renderHarnesses();
|
|
5109
5139
|
}
|
|
5110
5140
|
|
|
5111
5141
|
// change_tier badge — the gate + judge plan for the current working-tree diff (ADR-003/004).
|
|
5142
|
+
// The Harnesses card. Four states for the second opinion, and the fourth is
|
|
5143
|
+
// the reason the card exists: `unavailable` — declared codex, no codex here —
|
|
5144
|
+
// must never look like `none` or like a switch someone turned off.
|
|
5145
|
+
async function renderHarnesses() {
|
|
5146
|
+
const body = document.getElementById('harnesses-body');
|
|
5147
|
+
const status = document.getElementById('harnesses-status');
|
|
5148
|
+
if (!body) return;
|
|
5149
|
+
let h;
|
|
5150
|
+
try { h = await api(`/api/harnesses${pqs()}`); }
|
|
5151
|
+
catch (e) { status.textContent = 'unreadable'; body.innerHTML = `<b>Could not read the harnesses.</b> ${esc(String(e.message || e))}`; return; }
|
|
5152
|
+
|
|
5153
|
+
const so = h.second_opinion || {};
|
|
5154
|
+
const cx = h.codex || {};
|
|
5155
|
+
const ev = h.evidence || {};
|
|
5156
|
+
const sum = ev.summary || {};
|
|
5157
|
+
|
|
5158
|
+
const codexLine = cx.state === 'available'
|
|
5159
|
+
? `<b>OpenAI Codex</b> ${esc(cx.version || '')} · logged in (${esc(cx.auth || '?')}) · runs <code>${esc(cx.model || 'default model')}</code>`
|
|
5160
|
+
: cx.state === 'no-auth'
|
|
5161
|
+
? `<b>OpenAI Codex</b> ${esc(cx.version || '')} · <span class="warn">not logged in</span> — ${esc(cx.why || '')}`
|
|
5162
|
+
: `<b>OpenAI Codex</b> · <span class="warn">not installed</span> — ${esc(cx.why || '')}`;
|
|
5163
|
+
|
|
5164
|
+
const stateLabel = {
|
|
5165
|
+
declared: `on — <b>${esc(so.provider)}</b> reviews in parallel with the Claude reviewer`,
|
|
5166
|
+
none: 'off — the project decided: one model family only',
|
|
5167
|
+
undeclared: 'not declared — <b>not</b> the same as off; nobody has said',
|
|
5168
|
+
unavailable: `<span class="warn">declared <b>${esc(so.provider)}</b>, but unavailable here</span> — ${esc(so.why || '')}`,
|
|
5169
|
+
}[so.state] || esc(so.state || '?');
|
|
5170
|
+
|
|
5171
|
+
const sel = (so.providers || ['codex', 'openrouter', 'none']).map((p) =>
|
|
5172
|
+
`<option value="${p}" ${so.declared && so.declared.tool === p ? 'selected' : ''}>${p}</option>`).join('');
|
|
5173
|
+
const undecl = `<option value="" ${!so.declared || so.declared.state === 'undeclared' ? 'selected' : ''}>(undeclared)</option>`;
|
|
5174
|
+
|
|
5175
|
+
const rows = (ev.recent || []).slice(0, 6).map((r) => {
|
|
5176
|
+
const when = (r.ts || '').slice(0, 16).replace('T', ' ');
|
|
5177
|
+
const what = r.state === 'ok'
|
|
5178
|
+
? `${r.verdict === 'BLOCK' ? '<b>BLOCK</b>' : 'PASS'} · ${r.findings ?? 0} finding${r.findings === 1 ? '' : 's'}${r.p0 ? ` · ${r.p0} P0` : ''}${r.cost != null ? ` · $${r.cost}` : ' · unpriced'}`
|
|
5179
|
+
: `<span class="muted">skipped (${esc(r.state)})</span>`;
|
|
5180
|
+
return `<div class="mono" style="font-size:12px">${esc(when)} · ${esc(r.provider || '?')}${r.model ? ':' + esc(r.model) : ''} · ${what}</div>`;
|
|
5181
|
+
}).join('');
|
|
5182
|
+
const evidence = ev.state === 'absent'
|
|
5183
|
+
? '<span class="muted">no cross-model review has run in this project yet</span>'
|
|
5184
|
+
: ev.state === 'unreadable'
|
|
5185
|
+
? '<span class="warn">the review log is unreadable</span>'
|
|
5186
|
+
: `${sum.reviewed ?? 0} reviewed · ${sum.skipped ?? 0} skipped · ${sum.blocked ?? 0} blocked${sum.unreadable_lines ? ` · <span class="warn">${sum.unreadable_lines} unreadable line(s)</span>` : ''}<div style="margin-top:4px">${rows}</div>`;
|
|
5187
|
+
|
|
5188
|
+
status.textContent = so.state === 'declared' ? `2 · ${so.provider}` : so.state === 'unavailable' ? 'unavailable' : so.state === 'undeclared' ? 'undeclared' : '1';
|
|
5189
|
+
status.className = 'inbox-count' + (so.state === 'unavailable' ? ' warn' : '');
|
|
5190
|
+
// The header badge: two harnesses when a second one actually reviews.
|
|
5191
|
+
// `unavailable` is shown as such — a declared-but-absent Codex in the header
|
|
5192
|
+
// reading "2 harnesses" would be the pending-gate-as-passed defect in a badge.
|
|
5193
|
+
const hb = document.getElementById('harness-badge');
|
|
5194
|
+
if (hb) {
|
|
5195
|
+
hb.textContent = so.state === 'declared' ? `2 harnesses · ${so.provider}`
|
|
5196
|
+
: so.state === 'unavailable' ? `${so.provider} unavailable` : '';
|
|
5197
|
+
hb.className = 'tier-badge';
|
|
5198
|
+
hb.style.color = so.state === 'unavailable' ? 'var(--status-blocked)' : '';
|
|
5199
|
+
hb.style.display = hb.textContent ? '' : 'none';
|
|
5200
|
+
}
|
|
5201
|
+
body.innerHTML = `
|
|
5202
|
+
<div><b>Claude Code</b> · host · runs the pipeline, gates and hooks</div>
|
|
5203
|
+
<div>${codexLine}</div>
|
|
5204
|
+
<div style="margin-top:8px">Second opinion: ${stateLabel}</div>
|
|
5205
|
+
<div style="margin-top:6px;display:flex;gap:8px;align-items:center;flex-wrap:wrap">
|
|
5206
|
+
<label for="so-select" class="muted">set</label>
|
|
5207
|
+
<select id="so-select">${undecl}${sel}</select>
|
|
5208
|
+
<button id="so-apply" class="btn-ghost-sm">apply</button>
|
|
5209
|
+
<span id="so-result" class="muted"></span>
|
|
5210
|
+
</div>
|
|
5211
|
+
<div style="margin-top:8px"><span class="muted">Evidence:</span> ${evidence}</div>`;
|
|
5212
|
+
|
|
5213
|
+
document.getElementById('so-apply').onclick = async () => {
|
|
5214
|
+
const v = document.getElementById('so-select').value;
|
|
5215
|
+
const out = document.getElementById('so-result');
|
|
5216
|
+
out.textContent = 'writing PROJECT.md…';
|
|
5217
|
+
try {
|
|
5218
|
+
const r = await api(`/api/harnesses/second-opinion${pqs()}`, { method: 'POST', body: JSON.stringify({ provider: v || null }) });
|
|
5219
|
+
// Say what it replaced and what it resolved to NOW, so choosing codex on a
|
|
5220
|
+
// machine without one is answered at the click, not on the next review.
|
|
5221
|
+
out.textContent = `was ${r.previous ?? 'undeclared'} → now ${r.provider ?? 'undeclared'} (${r.resolved?.state}${r.resolved?.why ? ': ' + r.resolved.why : ''})`;
|
|
5222
|
+
renderHarnesses();
|
|
5223
|
+
} catch (e) { out.textContent = `failed: ${String(e.message || e)}`; }
|
|
5224
|
+
};
|
|
5225
|
+
}
|
|
5226
|
+
|
|
5112
5227
|
async function renderTierBadge() {
|
|
5113
5228
|
const el = document.getElementById('tier-badge');
|
|
5114
5229
|
if (!el) return;
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* codex-exec — the Codex CLI as a subprocess, and whether one is here to run.
|
|
3
|
+
*
|
|
4
|
+
* Moved out of tests/eval/arm-codex.mjs the day Codex became a participant in
|
|
5
|
+
* the pipeline rather than a thing the evals compare against. The parser and
|
|
6
|
+
* runner are unchanged in behaviour; what is new is `detectCodex`, because a
|
|
7
|
+
* second opinion that is silently absent reads exactly like a second opinion
|
|
8
|
+
* that agreed.
|
|
9
|
+
*
|
|
10
|
+
* Three things a caller must be able to tell apart, and none may look like PASS:
|
|
11
|
+
* absent no `codex` on PATH
|
|
12
|
+
* no-auth a binary and no login — `codex exec` returns 400 on every model
|
|
13
|
+
* available version, auth mode and the model it will run, read from disk
|
|
14
|
+
*/
|
|
15
|
+
import { spawn, spawnSync } from 'node:child_process';
|
|
16
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
17
|
+
import { homedir } from 'node:os';
|
|
18
|
+
import { join } from 'node:path';
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* `codex exec --json` output → the answer, its cost, and what went wrong.
|
|
22
|
+
*
|
|
23
|
+
* Four states, and the last three are why this is not a one-liner:
|
|
24
|
+
*
|
|
25
|
+
* ok an agent message was produced
|
|
26
|
+
* empty the stream parsed and carried no agent message
|
|
27
|
+
* unreadable nothing in the stream parsed at all
|
|
28
|
+
* (errors) non-fatal problems Codex reported mid-run, always surfaced
|
|
29
|
+
*
|
|
30
|
+
* `empty` returns `text: null`, never `""`. An empty answer graded as an answer
|
|
31
|
+
* scores like a real one. `usage` is null when the turn did not report it — a
|
|
32
|
+
* cost comparison that reads a missing measurement as zero makes one harness
|
|
33
|
+
* look free.
|
|
34
|
+
*/
|
|
35
|
+
export function parseCodexStream(raw) {
|
|
36
|
+
const messages = [];
|
|
37
|
+
const errors = [];
|
|
38
|
+
let usage = null;
|
|
39
|
+
let parsedAny = false;
|
|
40
|
+
|
|
41
|
+
for (const line of String(raw ?? '').split('\n')) {
|
|
42
|
+
const s = line.trim();
|
|
43
|
+
// The CLI interleaves human-readable lines with the JSON stream, so a line
|
|
44
|
+
// that does not parse is noise to step over, not a failure.
|
|
45
|
+
if (!s.startsWith('{')) continue;
|
|
46
|
+
let ev;
|
|
47
|
+
try { ev = JSON.parse(s); } catch { continue; }
|
|
48
|
+
parsedAny = true;
|
|
49
|
+
|
|
50
|
+
if (ev.type === 'turn.completed' && ev.usage) usage = ev.usage;
|
|
51
|
+
const item = ev.item;
|
|
52
|
+
if (!item) continue;
|
|
53
|
+
if (item.type === 'agent_message' && typeof item.text === 'string') messages.push(item.text);
|
|
54
|
+
// Codex reports recoverable problems as error items: a rejected plugin
|
|
55
|
+
// config, a truncated skill budget. A verdict built over a degraded run is
|
|
56
|
+
// a verdict about the wrong thing, so these always reach the caller.
|
|
57
|
+
else if (item.type === 'error' && item.message) errors.push(String(item.message));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (!parsedAny) return { state: 'unreadable', text: null, usage: null, errors };
|
|
61
|
+
if (!messages.length) return { state: 'empty', text: null, usage, errors };
|
|
62
|
+
return { state: 'ok', text: messages.join('\n'), usage, errors };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Run one prompt through the Codex CLI and parse the result.
|
|
67
|
+
*
|
|
68
|
+
* The prompt goes on stdin: passing it as an argument alongside `-c` overrides
|
|
69
|
+
* made the CLI wait on stdin instead, which looks exactly like a hung model.
|
|
70
|
+
*
|
|
71
|
+
* `sandbox` defaults to read-only: a reviewer that can write is not a reviewer.
|
|
72
|
+
* `ephemeral` keeps the review out of the user's Codex session history.
|
|
73
|
+
*
|
|
74
|
+
* @returns the shape of `parseCodexStream`, plus `{ code, model }`
|
|
75
|
+
*/
|
|
76
|
+
export function runCodexExec({
|
|
77
|
+
prompt, cwd, timeoutMs = 300000, bin = 'codex', model = null,
|
|
78
|
+
sandbox = 'read-only', ephemeral = true, extraArgs = [],
|
|
79
|
+
}) {
|
|
80
|
+
return new Promise((resolve) => {
|
|
81
|
+
const args = ['exec', '--json', '--skip-git-repo-check'];
|
|
82
|
+
if (sandbox) args.push('-s', sandbox);
|
|
83
|
+
if (ephemeral) args.push('--ephemeral');
|
|
84
|
+
if (model) args.push('-m', model);
|
|
85
|
+
if (cwd) args.push('-C', cwd);
|
|
86
|
+
args.push(...extraArgs, '-');
|
|
87
|
+
|
|
88
|
+
const proc = spawn(bin, args, { cwd, stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env } });
|
|
89
|
+
let out = '';
|
|
90
|
+
let err = '';
|
|
91
|
+
const timer = setTimeout(() => { try { proc.kill('SIGKILL'); } catch { /* gone */ } }, timeoutMs);
|
|
92
|
+
|
|
93
|
+
proc.stdout.on('data', (b) => { out += String(b); });
|
|
94
|
+
proc.stderr.on('data', (b) => { err += String(b); });
|
|
95
|
+
proc.on('error', (e) => {
|
|
96
|
+
clearTimeout(timer);
|
|
97
|
+
resolve({ state: 'unreadable', text: null, usage: null, errors: [String(e.message || e)], code: null, model });
|
|
98
|
+
});
|
|
99
|
+
proc.on('close', (code) => {
|
|
100
|
+
clearTimeout(timer);
|
|
101
|
+
const parsed = parseCodexStream(out);
|
|
102
|
+
// stderr is kept even on success: Codex writes warnings there that change
|
|
103
|
+
// how a result should be read.
|
|
104
|
+
if (err.trim()) parsed.errors.push(err.trim().slice(0, 500));
|
|
105
|
+
resolve({ ...parsed, code, model });
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
proc.stdin.write(prompt);
|
|
109
|
+
proc.stdin.end();
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Kept under the old name for the eval arm. */
|
|
114
|
+
export const runCodexArm = (opts) => runCodexExec({ ...opts, sandbox: opts.sandbox ?? null, ephemeral: opts.ephemeral ?? false });
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* What a Codex install on this machine is, from three readings — pure, so the
|
|
118
|
+
* three states can be tested without a Codex.
|
|
119
|
+
*
|
|
120
|
+
* @param {{versionOut:string|null, authJson:string|null, configToml:string|null}} r
|
|
121
|
+
* `null` for a reading that could not be taken (binary absent, file missing).
|
|
122
|
+
* @returns {{state:'absent'|'no-auth'|'available', version:string|null,
|
|
123
|
+
* auth:string|null, model:string|null, why:string}}
|
|
124
|
+
*/
|
|
125
|
+
export function codexStatusFrom({ versionOut, authJson, configToml }) {
|
|
126
|
+
if (versionOut == null) {
|
|
127
|
+
return { state: 'absent', version: null, auth: null, model: null, why: 'codex is not on PATH — npm i -g @openai/codex' };
|
|
128
|
+
}
|
|
129
|
+
const version = (String(versionOut).match(/(\d+\.\d+\.\d+)/) || [])[1] ?? null;
|
|
130
|
+
|
|
131
|
+
let auth = null;
|
|
132
|
+
if (authJson != null) {
|
|
133
|
+
try { auth = JSON.parse(authJson)?.auth_mode ?? (JSON.parse(authJson)?.tokens ? 'chatgpt' : null); }
|
|
134
|
+
catch { auth = null; }
|
|
135
|
+
}
|
|
136
|
+
// config.toml may name the model on its own line; a `[profiles.x]` table
|
|
137
|
+
// below can name others, so only the first top-level `model =` counts.
|
|
138
|
+
let model = null;
|
|
139
|
+
if (configToml != null) {
|
|
140
|
+
const top = String(configToml).split(/^\[/m)[0];
|
|
141
|
+
model = (top.match(/^\s*model\s*=\s*"([^"]+)"/m) || [])[1] ?? null;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (!auth) {
|
|
145
|
+
return { state: 'no-auth', version, auth: null, model, why: 'codex is installed but not logged in — run `codex login`' };
|
|
146
|
+
}
|
|
147
|
+
return { state: 'available', version, auth, model, why: '' };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** The three readings, taken. */
|
|
151
|
+
// `GREAT_CTO_CODEX_BIN` is the one lever for all three consumers — reviewer,
|
|
152
|
+
// verifier, board — so a test (or an operator with two Codex installs) points
|
|
153
|
+
// them at the same binary, and so an absent Codex can be simulated without
|
|
154
|
+
// stripping PATH of the node that runs the server.
|
|
155
|
+
export function detectCodex({ bin = process.env.GREAT_CTO_CODEX_BIN || 'codex', home = homedir() } = {}) {
|
|
156
|
+
let versionOut = null;
|
|
157
|
+
try {
|
|
158
|
+
const r = spawnSync(bin, ['--version'], { encoding: 'utf8', timeout: 5000 });
|
|
159
|
+
if (r.status === 0) versionOut = String(r.stdout || r.stderr || '');
|
|
160
|
+
} catch { /* absent */ }
|
|
161
|
+
const read = (p) => { try { return existsSync(p) ? readFileSync(p, 'utf8') : null; } catch { return null; } };
|
|
162
|
+
return codexStatusFrom({
|
|
163
|
+
versionOut,
|
|
164
|
+
authJson: read(join(home, '.codex', 'auth.json')),
|
|
165
|
+
configToml: read(join(home, '.codex', 'config.toml')),
|
|
166
|
+
});
|
|
167
|
+
}
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* dag-metric — score a judgement by walking a graph of narrow questions, instead
|
|
4
|
+
* of asking a model for a number.
|
|
5
|
+
*
|
|
6
|
+
* Why this exists
|
|
7
|
+
* ---------------
|
|
8
|
+
* Our eval judge returns a rate per scenario, and the recorded history says what
|
|
9
|
+
* that costs: 14 of 18 results below their own threshold, one eval at 0.44
|
|
10
|
+
* against a 1.00 bar, and the runner's own header notes ±0.09 of variance that
|
|
11
|
+
* more samples do not remove. That variance is inherent to asking "score this
|
|
12
|
+
* 0–1" — the number is sampled from a distribution, not derived from anything.
|
|
13
|
+
*
|
|
14
|
+
* The idea is DeepEval's, and it is the one thing worth taking from it: reduce
|
|
15
|
+
* the model's job to questions it can answer the same way twice ("does this
|
|
16
|
+
* finding cite a file:line?"), and COMPUTE the score from the path those answers
|
|
17
|
+
* take. The model still judges; it no longer does arithmetic.
|
|
18
|
+
*
|
|
19
|
+
* Two properties follow, and both matter more than the score:
|
|
20
|
+
* - the same answers always give the same score
|
|
21
|
+
* - the verdict is a path you can print, so a disagreement is about a specific
|
|
22
|
+
* question rather than about a number nobody can locate
|
|
23
|
+
*
|
|
24
|
+
* Shape (plain JSON, so a DAG is data an agent can write):
|
|
25
|
+
* { root, nodes: { id: { question, edges: { <answer>: <nextId> } } },
|
|
26
|
+
* leaves: { id: { score: 0..1, reason } } }
|
|
27
|
+
*
|
|
28
|
+
* CLI:
|
|
29
|
+
* node scripts/lib/dag-metric.mjs <dag.json> --answers '{"n1":"yes"}'
|
|
30
|
+
* node scripts/lib/dag-metric.mjs <dag.json> --validate
|
|
31
|
+
* node scripts/lib/dag-metric.mjs <dag.json> --next --answers '{...}'
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { createHash } from 'node:crypto';
|
|
35
|
+
|
|
36
|
+
const isLeafScore = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= 1;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* A 12-hex-char fingerprint over the DAG's scoring structure — `root`, each
|
|
40
|
+
* node's `question` and `edges`, each leaf's `score` — with every level's keys
|
|
41
|
+
* sorted so key order in the source file cannot move the hash.
|
|
42
|
+
*
|
|
43
|
+
* Deliberately excludes `note` (top-level) and each leaf's `reason`: those are
|
|
44
|
+
* documentation, not the metric. Two runs scored by the same questions and the
|
|
45
|
+
* same edges must fingerprint identically even if someone rewrote a comment —
|
|
46
|
+
* and two runs whose questions or edges differ must not, because that is
|
|
47
|
+
* exactly the case (ARCH-judge-provenance §1/§3) that makes their rates
|
|
48
|
+
* incomparable.
|
|
49
|
+
*/
|
|
50
|
+
export function dagFingerprint(dag) {
|
|
51
|
+
const sortKeys = (o) => Object.fromEntries(Object.entries(o || {}).sort(([a], [b]) => (a < b ? -1 : 1)));
|
|
52
|
+
const scoring = {
|
|
53
|
+
root: dag && dag.root,
|
|
54
|
+
nodes: sortKeys(Object.fromEntries(Object.entries((dag && dag.nodes) || {})
|
|
55
|
+
// `n?.` rather than `n.`: a null entry inside `nodes` threw where an
|
|
56
|
+
// absent `nodes` did not, so the function was safe against the malformed
|
|
57
|
+
// shapes anyone thought to try and unsafe against one nobody did. This
|
|
58
|
+
// runs inside an eval that has already been paid for.
|
|
59
|
+
.map(([id, n]) => [id, { question: n?.question, edges: sortKeys(n?.edges) }]))),
|
|
60
|
+
leaves: sortKeys(Object.fromEntries(Object.entries((dag && dag.leaves) || {})
|
|
61
|
+
.map(([id, l]) => [id, { score: l?.score }]))),
|
|
62
|
+
};
|
|
63
|
+
return createHash('sha256').update(JSON.stringify(scoring)).digest('hex').slice(0, 12);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Check the graph before anyone judges anything: a lost branch or a cycle is a
|
|
68
|
+
* bug in the metric, and finding it mid-run means a wasted judging pass.
|
|
69
|
+
*
|
|
70
|
+
* @returns {{errors: string[], warnings: string[]}}
|
|
71
|
+
*/
|
|
72
|
+
export function validateDag(dag) {
|
|
73
|
+
const errors = [];
|
|
74
|
+
const warnings = [];
|
|
75
|
+
const nodes = (dag && dag.nodes) || {};
|
|
76
|
+
const leaves = (dag && dag.leaves) || {};
|
|
77
|
+
const root = dag && dag.root;
|
|
78
|
+
|
|
79
|
+
if (!root) errors.push('no `root`');
|
|
80
|
+
if (!Object.keys(nodes).length) errors.push('no `nodes`');
|
|
81
|
+
if (!Object.keys(leaves).length) errors.push('no `leaves`');
|
|
82
|
+
if (root && !nodes[root] && !leaves[root]) errors.push(`root '${root}' is neither a node nor a leaf`);
|
|
83
|
+
|
|
84
|
+
for (const [id, leaf] of Object.entries(leaves)) {
|
|
85
|
+
if (!leaf || !('score' in leaf)) { errors.push(`leaf '${id}' has no score`); continue; }
|
|
86
|
+
if (!isLeafScore(leaf.score)) errors.push(`leaf '${id}' score must be a number in 0..1`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
for (const [id, node] of Object.entries(nodes)) {
|
|
90
|
+
if (!node || !node.question) errors.push(`node '${id}' has no question`);
|
|
91
|
+
const edges = (node && node.edges) || {};
|
|
92
|
+
if (!Object.keys(edges).length) errors.push(`node '${id}' has no edges`);
|
|
93
|
+
for (const [answer, target] of Object.entries(edges)) {
|
|
94
|
+
if (!nodes[target] && !leaves[target]) {
|
|
95
|
+
errors.push(`node '${id}' edge '${answer}' points at '${target}', which does not exist`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Cycles: a question graph that loops can never reach a score.
|
|
101
|
+
const state = new Map(); // id → 'open' | 'done'
|
|
102
|
+
const walk = (id, trail) => {
|
|
103
|
+
if (leaves[id]) return;
|
|
104
|
+
if (state.get(id) === 'open') { errors.push(`cycle: ${[...trail, id].join(' → ')}`); return; }
|
|
105
|
+
if (state.get(id) === 'done' || !nodes[id]) return;
|
|
106
|
+
state.set(id, 'open');
|
|
107
|
+
for (const target of Object.values(nodes[id].edges || {})) walk(target, [...trail, id]);
|
|
108
|
+
state.set(id, 'done');
|
|
109
|
+
};
|
|
110
|
+
if (root) walk(root, []);
|
|
111
|
+
|
|
112
|
+
// Reachability: an orphaned leaf usually means a branch was dropped in an edit.
|
|
113
|
+
const seen = new Set();
|
|
114
|
+
const reach = (id) => {
|
|
115
|
+
if (!id || seen.has(id)) return;
|
|
116
|
+
seen.add(id);
|
|
117
|
+
for (const target of Object.values((nodes[id] || {}).edges || {})) reach(target);
|
|
118
|
+
};
|
|
119
|
+
if (root && !errors.some((e) => e.startsWith('cycle'))) reach(root);
|
|
120
|
+
for (const id of Object.keys(leaves)) if (!seen.has(id)) warnings.push(`leaf '${id}' is unreachable`);
|
|
121
|
+
for (const id of Object.keys(nodes)) if (!seen.has(id)) warnings.push(`node '${id}' is unreachable`);
|
|
122
|
+
|
|
123
|
+
return { errors, warnings };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Walk the graph with the answers given.
|
|
128
|
+
*
|
|
129
|
+
* A missing answer returns `score: null` and names the question. It never
|
|
130
|
+
* defaults, because a default is exactly the invented number this module exists
|
|
131
|
+
* to remove.
|
|
132
|
+
*/
|
|
133
|
+
export function evaluateDag(dag, answers = {}) {
|
|
134
|
+
const { errors } = validateDag(dag);
|
|
135
|
+
if (errors.length) return { score: null, path: [], leaf: null, error: errors[0] };
|
|
136
|
+
|
|
137
|
+
const { nodes, leaves, root } = dag;
|
|
138
|
+
const path = [];
|
|
139
|
+
const steps = []; // {id, question, answer} — the audit trail
|
|
140
|
+
let id = root;
|
|
141
|
+
const guard = Object.keys(nodes).length + 1; // validation rules out cycles; this is belt
|
|
142
|
+
|
|
143
|
+
for (let step = 0; step <= guard; step++) {
|
|
144
|
+
if (leaves[id]) {
|
|
145
|
+
path.push(id);
|
|
146
|
+
return { score: leaves[id].score, leaf: id, reason: leaves[id].reason || '', path, steps };
|
|
147
|
+
}
|
|
148
|
+
const node = nodes[id];
|
|
149
|
+
path.push(id);
|
|
150
|
+
if (!(id in answers)) {
|
|
151
|
+
return { score: null, path, leaf: null, steps, pending: id, question: node.question };
|
|
152
|
+
}
|
|
153
|
+
const answer = answers[id];
|
|
154
|
+
const next = node.edges[answer];
|
|
155
|
+
if (!next) {
|
|
156
|
+
return {
|
|
157
|
+
score: null, path, leaf: null, steps,
|
|
158
|
+
error: `node '${id}' has no edge for answer '${answer}' (expected: ${Object.keys(node.edges).join(', ')})`,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
steps.push({ id, question: node.question, answer });
|
|
162
|
+
id = next;
|
|
163
|
+
}
|
|
164
|
+
return { score: null, path, leaf: null, steps, error: 'walk did not terminate' };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** The next question to put to a judge, with the answers it may give. */
|
|
168
|
+
export function pendingQuestion(dag, answers = {}) {
|
|
169
|
+
const r = evaluateDag(dag, answers);
|
|
170
|
+
if (!r.pending) return null;
|
|
171
|
+
return { id: r.pending, question: r.question, answers: Object.keys(dag.nodes[r.pending].edges) };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** The verdict a human reads: every question, its answer, and where the score came from. */
|
|
175
|
+
export function explain(result, dag) {
|
|
176
|
+
if (!result) return 'no result';
|
|
177
|
+
if (result.error) return `unscored — ${result.error}`;
|
|
178
|
+
|
|
179
|
+
const lines = (result.steps || []).map((s) => ` ${s.question}\n ${s.answer}`);
|
|
180
|
+
|
|
181
|
+
if (result.score === null) {
|
|
182
|
+
const q = result.question ? `\n ${result.question}\n (unanswered)` : '';
|
|
183
|
+
return [`unscored — pending question: ${result.pending || 'unknown'}`, ...lines].join('\n') + q;
|
|
184
|
+
}
|
|
185
|
+
if (result.leaf) lines.push(` → ${result.reason || result.leaf}`);
|
|
186
|
+
return [`score: ${result.score}`, ...lines].join('\n');
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* The prompt for one node. The judge is given the allowed answers and told to
|
|
191
|
+
* reply with nothing else — the whole point of the graph is that its job is a
|
|
192
|
+
* choice from a closed set, not a number it has to invent.
|
|
193
|
+
*/
|
|
194
|
+
export function questionPrompt(question, allowed, context = {}) {
|
|
195
|
+
const list = allowed.join(' or ');
|
|
196
|
+
const system =
|
|
197
|
+
'You are grading one narrow factual question about an agent response. ' +
|
|
198
|
+
`Answer with exactly one word: ${list}. ` +
|
|
199
|
+
'No explanation, no punctuation, no hedging. If the response genuinely does not ' +
|
|
200
|
+
`settle the question, answer with the option that assumes the agent did NOT do it (${allowed[allowed.length - 1]}).`;
|
|
201
|
+
const user = [
|
|
202
|
+
context.scenario ? `Scenario: ${context.scenario}` : '',
|
|
203
|
+
context.test ? `Test case: ${context.test}` : '',
|
|
204
|
+
context.expected ? `Expected behaviour: ${context.expected}` : '',
|
|
205
|
+
context.actorResponse ? `Agent response:\n${context.actorResponse}` : '',
|
|
206
|
+
'',
|
|
207
|
+
`Question: ${question}`,
|
|
208
|
+
`Answer (${list}):`,
|
|
209
|
+
].filter(Boolean).join('\n\n');
|
|
210
|
+
return { system, user };
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Map a judge reply onto one of the allowed answers, or null.
|
|
215
|
+
*
|
|
216
|
+
* Null matters: an unparseable reply must stop the walk rather than pick a side,
|
|
217
|
+
* because a guessed answer is a guessed score wearing a decision tree.
|
|
218
|
+
*/
|
|
219
|
+
export function parseAnswer(reply, allowed) {
|
|
220
|
+
const t = String(reply ?? '').trim().toLowerCase();
|
|
221
|
+
if (!t) return null;
|
|
222
|
+
// Ambiguity is checked BEFORE any shortcut: a reply that echoes the option
|
|
223
|
+
// list ("yes or no") starts with a valid answer, and taking the first token
|
|
224
|
+
// would turn the judge's non-answer into a decision.
|
|
225
|
+
// Delimit by "not a word character" on either side rather than \b: a label is
|
|
226
|
+
// whatever the graph author wrote, and \b needs a word character to sit against,
|
|
227
|
+
// so `yes(1)` or `p0.` could never match their own exact reply.
|
|
228
|
+
const hits = allowed.filter((a) => {
|
|
229
|
+
const esc = a.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
230
|
+
return new RegExp(`(^|[^\\w-])${esc}([^\\w-]|$)`).test(t);
|
|
231
|
+
});
|
|
232
|
+
return hits.length === 1 ? hits[0] : null;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Score one case by walking the graph, asking `ask(question, allowed)` for each
|
|
237
|
+
* node on the path. `ask` returns a string reply; anything unparseable ends the
|
|
238
|
+
* walk unscored rather than guessing.
|
|
239
|
+
*
|
|
240
|
+
* Kept here rather than in the runner so it is testable with a stub `ask`.
|
|
241
|
+
*/
|
|
242
|
+
export async function judgeWithDag(dag, ask, context = {}) {
|
|
243
|
+
const answers = {};
|
|
244
|
+
const asked = [];
|
|
245
|
+
for (let i = 0; i <= Object.keys(dag.nodes || {}).length; i++) {
|
|
246
|
+
const q = pendingQuestion(dag, answers);
|
|
247
|
+
if (!q) break;
|
|
248
|
+
const reply = await ask(q.question, q.answers, context);
|
|
249
|
+
const answer = parseAnswer(reply, q.answers);
|
|
250
|
+
asked.push({ id: q.id, question: q.question, reply, answer });
|
|
251
|
+
if (answer === null) {
|
|
252
|
+
return { score: null, answers, asked, error: `judge gave no usable answer for '${q.id}': ${String(reply).slice(0, 60)}` };
|
|
253
|
+
}
|
|
254
|
+
answers[q.id] = answer;
|
|
255
|
+
}
|
|
256
|
+
return { ...evaluateDag(dag, answers), answers, asked };
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// ── CLI ─────────────────────────────────────────────────────────────────────
|
|
260
|
+
|
|
261
|
+
async function main(argv) {
|
|
262
|
+
const { readFileSync } = await import('node:fs');
|
|
263
|
+
const file = argv.find((a) => !a.startsWith('--'));
|
|
264
|
+
if (!file) {
|
|
265
|
+
console.error('usage: dag-metric.mjs <dag.json> [--validate] [--next] [--answers \'{"id":"yes"}\']');
|
|
266
|
+
return 2;
|
|
267
|
+
}
|
|
268
|
+
let dag;
|
|
269
|
+
try { dag = JSON.parse(readFileSync(file, 'utf8')); }
|
|
270
|
+
catch (e) { console.error(`cannot read ${file}: ${e.message}`); return 2; }
|
|
271
|
+
|
|
272
|
+
const i = argv.indexOf('--answers');
|
|
273
|
+
let answers = {};
|
|
274
|
+
if (i >= 0) {
|
|
275
|
+
try { answers = JSON.parse(argv[i + 1]); }
|
|
276
|
+
catch { console.error('--answers must be JSON'); return 2; }
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const { errors, warnings } = validateDag(dag);
|
|
280
|
+
for (const w of warnings) console.error(`warn: ${w}`);
|
|
281
|
+
if (errors.length) { for (const e of errors) console.error(`error: ${e}`); return 1; }
|
|
282
|
+
if (argv.includes('--validate')) { console.log(`ok: ${Object.keys(dag.nodes).length} nodes, ${Object.keys(dag.leaves).length} leaves`); return 0; }
|
|
283
|
+
|
|
284
|
+
if (argv.includes('--next')) {
|
|
285
|
+
const q = pendingQuestion(dag, answers);
|
|
286
|
+
console.log(q ? JSON.stringify(q, null, 2) : 'null');
|
|
287
|
+
return 0;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
const result = evaluateDag(dag, answers);
|
|
291
|
+
console.log(explain(result, dag));
|
|
292
|
+
return result.score === null ? 1 : 0;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
296
|
+
main(process.argv.slice(2)).then((c) => { process.exitCode = c; });
|
|
297
|
+
}
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* second-opinion — two judges, one graph, and only the disagreement is a signal.
|
|
4
|
+
*
|
|
5
|
+
* Why this exists
|
|
6
|
+
* ---------------
|
|
7
|
+
* `mcp__great_cto_llm_router__ask_kimi` has been in the `tools:` list of
|
|
8
|
+
* nineteen agents and invoked by none of them. The capability was declared and
|
|
9
|
+
* never exercised — the same shape as the deny list that ran on every write and
|
|
10
|
+
* protected nothing.
|
|
11
|
+
*
|
|
12
|
+
* The obvious use, a second model reviewing the first one's report, is the wrong
|
|
13
|
+
* one. A reviewer reading a report judges whether it reads plausibly, and a
|
|
14
|
+
* confident wrong finding is exactly what passes that test. Worse, two models
|
|
15
|
+
* agreeing is weak evidence: they are trained on overlapping data and fail in
|
|
16
|
+
* correlated ways. "Both said yes" is close to no information.
|
|
17
|
+
*
|
|
18
|
+
* What carries information is where they DIVERGE. So this does not ask two
|
|
19
|
+
* models to grade prose. It walks both through the same DAG of closed questions
|
|
20
|
+
* (scripts/lib/dag-metric.mjs) and reports the nodes where their answers differ.
|
|
21
|
+
* A divergence names a specific question — "does this finding cite a file:line?"
|
|
22
|
+
* — which is something a human can settle in seconds, unlike "is this report
|
|
23
|
+
* good?".
|
|
24
|
+
*
|
|
25
|
+
* The reporting rule is deliberate and the opposite of the usual one:
|
|
26
|
+
*
|
|
27
|
+
* - agreement is NOT reported as confidence. It is reported as "nothing to
|
|
28
|
+
* look at here", which is a weaker claim and the honest one.
|
|
29
|
+
* - divergence IS the output. It is where to dig.
|
|
30
|
+
* - one judge abstaining is neither. An unparseable answer is a third state,
|
|
31
|
+
* not a vote, and calling it agreement would let a broken judge confirm
|
|
32
|
+
* anything.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { pendingQuestion, evaluateDag, parseAnswer } from './dag-metric.mjs';
|
|
36
|
+
|
|
37
|
+
export const DIVERGENCE = Object.freeze({
|
|
38
|
+
AGREE: 'agree',
|
|
39
|
+
DIVERGE: 'diverge',
|
|
40
|
+
ABSTAINED: 'abstained',
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Walk one judge through the graph, collecting its answer at every node it
|
|
45
|
+
* reaches. Unlike judgeWithDag this does not stop at the first unusable reply —
|
|
46
|
+
* an abstention on one node still leaves the others worth comparing.
|
|
47
|
+
*
|
|
48
|
+
* @param {object} dag
|
|
49
|
+
* @param {(q:string, allowed:string[], ctx:object)=>Promise<string>} ask
|
|
50
|
+
* @returns {Promise<{answers: object, asked: Array, result: object}>}
|
|
51
|
+
*/
|
|
52
|
+
export async function walk(dag, ask, context = {}) {
|
|
53
|
+
const answers = {};
|
|
54
|
+
const asked = [];
|
|
55
|
+
const limit = Object.keys(dag.nodes || {}).length + 1;
|
|
56
|
+
for (let i = 0; i <= limit; i++) {
|
|
57
|
+
const q = pendingQuestion(dag, answers);
|
|
58
|
+
if (!q) break;
|
|
59
|
+
const reply = await ask(q.question, q.answers, context);
|
|
60
|
+
const answer = parseAnswer(reply, q.answers);
|
|
61
|
+
asked.push({ id: q.id, question: q.question, reply, answer });
|
|
62
|
+
// A node nobody could answer ends this judge's walk — the graph cannot
|
|
63
|
+
// continue without it — but what was already answered still compares.
|
|
64
|
+
if (answer === null) break;
|
|
65
|
+
answers[q.id] = answer;
|
|
66
|
+
}
|
|
67
|
+
return { answers, asked, result: evaluateDag(dag, answers) };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Compare two walks.
|
|
72
|
+
*
|
|
73
|
+
* @returns {{
|
|
74
|
+
* diverged: Array<{id, question, a, b}>,
|
|
75
|
+
* abstained: Array<{id, question, by: 'a'|'b'|'both'}>,
|
|
76
|
+
* agreed: string[],
|
|
77
|
+
* scores: {a: number|null, b: number|null},
|
|
78
|
+
* verdict: 'agree'|'diverge'|'abstained',
|
|
79
|
+
* summary: string,
|
|
80
|
+
* }}
|
|
81
|
+
*/
|
|
82
|
+
export function compare(walkA, walkB) {
|
|
83
|
+
const byId = (w) => new Map(w.asked.map((s) => [s.id, s]));
|
|
84
|
+
const A = byId(walkA);
|
|
85
|
+
const B = byId(walkB);
|
|
86
|
+
|
|
87
|
+
const diverged = [];
|
|
88
|
+
const abstained = [];
|
|
89
|
+
const agreed = [];
|
|
90
|
+
|
|
91
|
+
for (const id of new Set([...A.keys(), ...B.keys()])) {
|
|
92
|
+
const a = A.get(id);
|
|
93
|
+
const b = B.get(id);
|
|
94
|
+
// A node only one judge reached is not a disagreement — the walks parted
|
|
95
|
+
// earlier, and the node that parted them is already recorded.
|
|
96
|
+
if (!a || !b) continue;
|
|
97
|
+
if (a.answer === null || b.answer === null) {
|
|
98
|
+
abstained.push({
|
|
99
|
+
id, question: a.question,
|
|
100
|
+
by: a.answer === null && b.answer === null ? 'both' : (a.answer === null ? 'a' : 'b'),
|
|
101
|
+
});
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
if (a.answer !== b.answer) diverged.push({ id, question: a.question, a: a.answer, b: b.answer });
|
|
105
|
+
else agreed.push(id);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const scores = { a: walkA.result?.score ?? null, b: walkB.result?.score ?? null };
|
|
109
|
+
|
|
110
|
+
let verdict = DIVERGENCE.AGREE;
|
|
111
|
+
if (diverged.length) verdict = DIVERGENCE.DIVERGE;
|
|
112
|
+
else if (abstained.length) verdict = DIVERGENCE.ABSTAINED;
|
|
113
|
+
|
|
114
|
+
// The wording matters. Agreement is not evidence of correctness — two models
|
|
115
|
+
// trained on overlapping data fail together — so it is reported as an absence
|
|
116
|
+
// of signal, never as confidence.
|
|
117
|
+
const summary = diverged.length
|
|
118
|
+
? `${diverged.length} question(s) answered differently — that is where to look`
|
|
119
|
+
: abstained.length
|
|
120
|
+
? `${abstained.length} question(s) one judge could not answer — no comparison possible there`
|
|
121
|
+
: `no divergence across ${agreed.length} question(s); this is weak evidence, not confirmation`;
|
|
122
|
+
|
|
123
|
+
return { diverged, abstained, agreed, scores, verdict, summary };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Run both judges over the same graph and compare.
|
|
128
|
+
*
|
|
129
|
+
* The two `ask` functions must be independent — a second opinion that saw the
|
|
130
|
+
* first one's answer is not a second opinion.
|
|
131
|
+
*/
|
|
132
|
+
export async function secondOpinion(dag, askA, askB, context = {}) {
|
|
133
|
+
const [a, b] = await Promise.all([walk(dag, askA, context), walk(dag, askB, context)]);
|
|
134
|
+
return { ...compare(a, b), walks: { a, b } };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Human-readable report. Divergence first — it is the only actionable part. */
|
|
138
|
+
export function explainComparison(cmp) {
|
|
139
|
+
const lines = [cmp.summary];
|
|
140
|
+
for (const d of cmp.diverged) {
|
|
141
|
+
lines.push('', ` ${d.question}`, ` judge A: ${d.a}`, ` judge B: ${d.b}`);
|
|
142
|
+
}
|
|
143
|
+
for (const s of cmp.abstained) {
|
|
144
|
+
lines.push('', ` ${s.question}`, ` unanswered by: ${s.by}`);
|
|
145
|
+
}
|
|
146
|
+
if (cmp.scores.a !== cmp.scores.b) {
|
|
147
|
+
lines.push('', ` scores differ: ${cmp.scores.a} vs ${cmp.scores.b}`);
|
|
148
|
+
}
|
|
149
|
+
return lines.join('\n');
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// ── the second judge ────────────────────────────────────────────────────────
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* An `ask` backed by the llm-router MCP server (Kimi by default).
|
|
156
|
+
*
|
|
157
|
+
* Spawned per question on purpose: the server is stdio JSON-RPC and a fresh
|
|
158
|
+
* process per question guarantees the second judge cannot see its own earlier
|
|
159
|
+
* answers accumulate into a context that biases the next one. It is slower and
|
|
160
|
+
* that is the correct trade for an independence claim.
|
|
161
|
+
*/
|
|
162
|
+
export function routerAsk(serverPath, { timeoutMs = 60_000, model = null } = {}) {
|
|
163
|
+
return async (question, allowed) => {
|
|
164
|
+
const { spawn } = await import('node:child_process');
|
|
165
|
+
return new Promise((resolve) => {
|
|
166
|
+
// The model is passed through the spawned server's environment, which is
|
|
167
|
+
// where it reads it from. Without this every "second opinion" was the same
|
|
168
|
+
// model answering twice — three samples of one judge are correlated, and
|
|
169
|
+
// calling that a second opinion is the confidence-by-repetition this
|
|
170
|
+
// module was written to avoid.
|
|
171
|
+
const env = model ? { ...process.env, GREAT_CTO_ROUTER_MODEL: model } : process.env;
|
|
172
|
+
const p = spawn('python3', [serverPath], { stdio: ['pipe', 'pipe', 'ignore'], env });
|
|
173
|
+
let out = '';
|
|
174
|
+
const done = (v) => { try { p.kill(); } catch {} resolve(v); };
|
|
175
|
+
const timer = setTimeout(() => done(''), timeoutMs);
|
|
176
|
+
p.stdout.on('data', (d) => { out += d; });
|
|
177
|
+
p.on('error', () => { clearTimeout(timer); done(''); });
|
|
178
|
+
p.on('close', () => {
|
|
179
|
+
clearTimeout(timer);
|
|
180
|
+
for (const line of out.split('\n')) {
|
|
181
|
+
if (!line.trim().startsWith('{')) continue;
|
|
182
|
+
try {
|
|
183
|
+
const d = JSON.parse(line);
|
|
184
|
+
if (d.id === 2 && d.result && d.result.content) return done(d.result.content[0].text);
|
|
185
|
+
} catch { /* not our line */ }
|
|
186
|
+
}
|
|
187
|
+
done('');
|
|
188
|
+
});
|
|
189
|
+
const task = `${question}\nAnswer with exactly one word from: ${allowed.join(', ')}. No explanation.`;
|
|
190
|
+
p.stdin.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }) + '\n');
|
|
191
|
+
p.stdin.write(JSON.stringify({
|
|
192
|
+
jsonrpc: '2.0', id: 2, method: 'tools/call',
|
|
193
|
+
params: { name: 'ask_kimi', arguments: { task } },
|
|
194
|
+
}) + '\n');
|
|
195
|
+
p.stdin.end();
|
|
196
|
+
});
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// ── CLI ─────────────────────────────────────────────────────────────────────
|
|
201
|
+
//
|
|
202
|
+
// node scripts/lib/second-opinion.mjs <dag.json> --answers '{"node-id":"yes"}'
|
|
203
|
+
//
|
|
204
|
+
// You supply YOUR answers; the router answers the same questions independently
|
|
205
|
+
// and the divergence is printed. Exit 1 when the judges diverge — not because
|
|
206
|
+
// divergence is a failure, but because it is the case a human should see.
|
|
207
|
+
|
|
208
|
+
async function main(argv) {
|
|
209
|
+
const { readFileSync, existsSync } = await import('node:fs');
|
|
210
|
+
const { join, dirname } = await import('node:path');
|
|
211
|
+
const { fileURLToPath } = await import('node:url');
|
|
212
|
+
|
|
213
|
+
const file = argv.find((a) => !a.startsWith('--'));
|
|
214
|
+
const i = argv.indexOf('--answers');
|
|
215
|
+
if (!file || i === -1) {
|
|
216
|
+
console.error('usage: second-opinion.mjs <dag.json> --answers \'{"node-id":"yes"}\' [--server <path>] [--json]');
|
|
217
|
+
return 2;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
let dag, mine;
|
|
221
|
+
try { dag = JSON.parse(readFileSync(file, 'utf8')); }
|
|
222
|
+
catch (e) { console.error(`cannot read ${file}: ${e.message}`); return 2; }
|
|
223
|
+
try { mine = JSON.parse(argv[i + 1]); }
|
|
224
|
+
catch { console.error('--answers must be JSON'); return 2; }
|
|
225
|
+
|
|
226
|
+
const si = argv.indexOf('--server');
|
|
227
|
+
const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
228
|
+
const server = si >= 0 ? argv[si + 1] : join(root, 'mcp-servers', 'llm-router', 'server.py');
|
|
229
|
+
if (!existsSync(server)) { console.error(`router not found at ${server}`); return 2; }
|
|
230
|
+
|
|
231
|
+
const cmp = await secondOpinion(dag, async (q, allowed) => {
|
|
232
|
+
const id = Object.keys(dag.nodes).find((k) => dag.nodes[k].question === q);
|
|
233
|
+
// An answer you did not give is an abstention, not a default. Filling it in
|
|
234
|
+
// would manufacture the agreement this tool exists to avoid claiming.
|
|
235
|
+
return id in mine ? String(mine[id]) : '';
|
|
236
|
+
}, routerAsk(server));
|
|
237
|
+
|
|
238
|
+
if (argv.includes('--json')) console.log(JSON.stringify(cmp, null, 2));
|
|
239
|
+
else console.log(explainComparison(cmp));
|
|
240
|
+
return cmp.diverged.length ? 1 : 0;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
244
|
+
main(process.argv.slice(2)).then((c) => { process.exitCode = c; });
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// ── Which harness gives the second opinion ───────────────────────────────────
|
|
248
|
+
//
|
|
249
|
+
// Until 2026-09-05 the second opinion was whatever OpenRouter model the
|
|
250
|
+
// environment named, and the environment named it in three different variables.
|
|
251
|
+
// Codex is now a participant: `codex exec` in a read-only sandbox, authenticated
|
|
252
|
+
// by the user's ChatGPT login, a genuinely different model family with no API
|
|
253
|
+
// key to leak. Which one runs is a project decision, declared once in
|
|
254
|
+
// PROJECT.md's `capabilities:` block and read here — by the reviewer, by the
|
|
255
|
+
// verifier, and by the board, so all three agree.
|
|
256
|
+
|
|
257
|
+
import { capabilitiesFromProjectMd } from './stack-capabilities.mjs';
|
|
258
|
+
import { runCodexExec, detectCodex } from './codex-exec.mjs';
|
|
259
|
+
|
|
260
|
+
export const SECOND_OPINION_PROVIDERS = Object.freeze(['codex', 'openrouter', 'none']);
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* FOUR states, and the fourth is the reason this function exists:
|
|
264
|
+
*
|
|
265
|
+
* declared the project chose a provider and it can run
|
|
266
|
+
* none the project decided: no second opinion
|
|
267
|
+
* undeclared nobody has said — NOT the same as none
|
|
268
|
+
* unavailable the project chose codex and there is no working codex here
|
|
269
|
+
*
|
|
270
|
+
* A caller that folds `unavailable` into `none` makes an absent reviewer look
|
|
271
|
+
* like a decision not to review.
|
|
272
|
+
*
|
|
273
|
+
* @param {{projectMd?:string, codex?:object, env?:object}} o
|
|
274
|
+
* `codex` is the shape of `codexStatusFrom`; pass it to stay pure.
|
|
275
|
+
*/
|
|
276
|
+
export function resolveSecondOpinion({ projectMd = '', codex = null, env = process.env } = {}) {
|
|
277
|
+
const cap = capabilitiesFromProjectMd(projectMd).map.second_opinion ?? { state: 'undeclared', tool: null };
|
|
278
|
+
if (cap.state === 'none') {
|
|
279
|
+
return { state: 'none', provider: 'none', why: 'the project declared second_opinion: none' };
|
|
280
|
+
}
|
|
281
|
+
if (cap.state === 'undeclared') {
|
|
282
|
+
return { state: 'undeclared', provider: null, why: 'second_opinion is not declared in PROJECT.md capabilities — not declared is not none' };
|
|
283
|
+
}
|
|
284
|
+
const provider = String(cap.tool).toLowerCase();
|
|
285
|
+
if (!SECOND_OPINION_PROVIDERS.includes(provider)) {
|
|
286
|
+
return { state: 'unavailable', provider, why: `second_opinion: ${provider} is not a provider this plugin knows (${SECOND_OPINION_PROVIDERS.join(', ')})` };
|
|
287
|
+
}
|
|
288
|
+
if (provider === 'codex') {
|
|
289
|
+
const c = codex ?? detectCodex();
|
|
290
|
+
if (c.state !== 'available') return { state: 'unavailable', provider, why: c.why, codex: c };
|
|
291
|
+
return { state: 'declared', provider, why: '', codex: c };
|
|
292
|
+
}
|
|
293
|
+
if (provider === 'openrouter') {
|
|
294
|
+
if (!env.OPENROUTER_API_KEY) return { state: 'unavailable', provider, why: 'second_opinion: openrouter needs OPENROUTER_API_KEY' };
|
|
295
|
+
return { state: 'declared', provider, why: '' };
|
|
296
|
+
}
|
|
297
|
+
return { state: 'none', provider: 'none', why: '' };
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* The judge's `ask(question, allowed) → word` shape, backed by Codex.
|
|
302
|
+
*
|
|
303
|
+
* Same contract as `routerAsk` above, so `judge()` cannot tell them apart —
|
|
304
|
+
* which is the point: a second judge is only a second judge if it is
|
|
305
|
+
* interchangeable at the call site. Returns '' on anything that is not an
|
|
306
|
+
* answer, as routerAsk does; the caller already treats '' as "unparsed".
|
|
307
|
+
*/
|
|
308
|
+
export function codexAsk(cwd, { model = null, timeoutMs = 120_000, bin = 'codex' } = {}) {
|
|
309
|
+
return async (question, allowed) => {
|
|
310
|
+
const prompt = `${question}\nAnswer with exactly one word from: ${allowed.join(', ')}. No explanation.`;
|
|
311
|
+
const r = await runCodexExec({ prompt, cwd, model, timeoutMs, bin });
|
|
312
|
+
return r.state === 'ok' ? r.text : '';
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* A full review turn through Codex: system + user prompt in, text + usage out.
|
|
318
|
+
* The three non-ok states pass through untouched so the reviewer can say
|
|
319
|
+
* "skipped: unreadable" rather than "PASS".
|
|
320
|
+
*/
|
|
321
|
+
export async function codexReview({ system, user, cwd, model = null, timeoutMs = 300_000, bin = 'codex' }) {
|
|
322
|
+
const prompt = `${system}\n\n${user}`;
|
|
323
|
+
const r = await runCodexExec({ prompt, cwd, model, timeoutMs, bin });
|
|
324
|
+
return { ...r, model: r.model ?? model ?? null };
|
|
325
|
+
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* stack-capabilities — what THIS project's operational tools actually are.
|
|
3
|
+
*
|
|
4
|
+
* `l3-support` carries a routing table with thirteen alert sources: grafana,
|
|
5
|
+
* datadog, cloudwatch, eks, argocd, sentry, postgres, kafka, airflow, vercel,
|
|
6
|
+
* betterstack, mongo, and a generic row. It is a good table. What it cannot say
|
|
7
|
+
* is which of those thirteen rows is THIS project — so the agent infers the
|
|
8
|
+
* stack from whatever the alert happened to come from, at the moment somebody is
|
|
9
|
+
* being paged, and its generic fallback row guesses Grafana.
|
|
10
|
+
*
|
|
11
|
+
* `PROJECT.md` already carries a `stack:` line, but it is prose for humans
|
|
12
|
+
* ("TypeScript / Node.js 22 / Cloudflare Workers"). It names no log store, no
|
|
13
|
+
* pager, no error tracker. `stack-baseline` pins what a NEW product should be
|
|
14
|
+
* built with; `observability-baseline` wires it at scaffold time. Neither
|
|
15
|
+
* describes what an existing project has connected today.
|
|
16
|
+
*
|
|
17
|
+
* Borrowed from anthropics/oncall-kit, whose rule is that no skill names a
|
|
18
|
+
* vendor: skills refer to CAPABILITIES, and one file maps each capability to the
|
|
19
|
+
* tool the team actually has. Swap Datadog for Grafana and no skill changes.
|
|
20
|
+
*
|
|
21
|
+
* The vocabulary is deliberately small and closed. An open-ended map becomes a
|
|
22
|
+
* place to write anything, and then nothing can be resolved against it.
|
|
23
|
+
*
|
|
24
|
+
* Declared in PROJECT.md:
|
|
25
|
+
*
|
|
26
|
+
* capabilities:
|
|
27
|
+
* logs: grafana-loki
|
|
28
|
+
* metrics: grafana
|
|
29
|
+
* errors: sentry
|
|
30
|
+
* pager: none
|
|
31
|
+
*
|
|
32
|
+
* THREE STATES PER CAPABILITY, never two. `undeclared` and `none` are different
|
|
33
|
+
* answers: the first means nobody has said, the second means the project has
|
|
34
|
+
* deliberately decided it has no pager. An agent that treats "nobody said" as
|
|
35
|
+
* "there is none" stops looking for something that exists.
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
/** The closed vocabulary. Adding one is a deliberate edit, not a free-text key. */
|
|
39
|
+
export const CAPABILITIES = Object.freeze([
|
|
40
|
+
'logs', // where log lines are searched
|
|
41
|
+
'metrics', // time-series and dashboards
|
|
42
|
+
'traces', // distributed traces
|
|
43
|
+
'errors', // exception capture and grouping
|
|
44
|
+
'alerts', // what fires, and where its rules live
|
|
45
|
+
'pager', // who gets woken
|
|
46
|
+
'deploys', // what shipped, and when
|
|
47
|
+
'code-host', // where the diff and the PR live
|
|
48
|
+
// Not an incident tool: which harness gives the SECOND opinion in review and
|
|
49
|
+
// verification — `codex` (a different model family, through `codex exec`),
|
|
50
|
+
// `openrouter`, or `none`. It lives in this vocabulary rather than in a new
|
|
51
|
+
// file because the three-state rule is the whole point: a project that has
|
|
52
|
+
// not said is not a project that has turned it off.
|
|
53
|
+
'second_opinion',
|
|
54
|
+
]);
|
|
55
|
+
|
|
56
|
+
/** `none` is a decision. Anything else is a tool name we pass through verbatim. */
|
|
57
|
+
const NONE = 'none';
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* @returns {{state:'declared'|'none'|'undeclared', tool:string|null}}
|
|
61
|
+
*/
|
|
62
|
+
function stateOf(raw) {
|
|
63
|
+
if (raw == null) return { state: 'undeclared', tool: null };
|
|
64
|
+
const v = String(raw).trim();
|
|
65
|
+
if (!v) return { state: 'undeclared', tool: null };
|
|
66
|
+
if (v.toLowerCase() === NONE) return { state: NONE, tool: null };
|
|
67
|
+
return { state: 'declared', tool: v };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Read the capability block out of a PROJECT.md body.
|
|
72
|
+
*
|
|
73
|
+
* Parsed with the same shape as `levelFromProjectMd` — a top-level key followed
|
|
74
|
+
* by indented `name: value` lines — rather than by pulling in a YAML parser the
|
|
75
|
+
* board's zero-dependency rule forbids.
|
|
76
|
+
*
|
|
77
|
+
* @returns {{map: Record<string,{state:string,tool:string|null}>, declaredCount:number, unknownKeys:string[]}}
|
|
78
|
+
*/
|
|
79
|
+
export function capabilitiesFromProjectMd(text = '') {
|
|
80
|
+
const body = String(text);
|
|
81
|
+
const start = body.match(/^capabilities:[ \t]*$/m);
|
|
82
|
+
const raw = {};
|
|
83
|
+
const unknownKeys = [];
|
|
84
|
+
|
|
85
|
+
if (start) {
|
|
86
|
+
const after = body.slice(start.index + start[0].length);
|
|
87
|
+
for (const line of after.split('\n')) {
|
|
88
|
+
if (/^\S/.test(line)) break; // dedent ends the block
|
|
89
|
+
const m = line.match(/^[ \t]+([a-z][a-z0-9_-]*)\s*:\s*(.*)$/i);
|
|
90
|
+
if (!m) continue;
|
|
91
|
+
const key = m[1].toLowerCase();
|
|
92
|
+
// An unrecognised key is REPORTED, not dropped. A capability nobody reads
|
|
93
|
+
// because it was misspelled is the same as one nobody wrote, except that
|
|
94
|
+
// the author believes it is there.
|
|
95
|
+
if (!CAPABILITIES.includes(key)) { unknownKeys.push(key); continue; }
|
|
96
|
+
raw[key] = m[2];
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const map = {};
|
|
101
|
+
let declaredCount = 0;
|
|
102
|
+
for (const cap of CAPABILITIES) {
|
|
103
|
+
map[cap] = stateOf(raw[cap]);
|
|
104
|
+
if (map[cap].state === 'declared') declaredCount++;
|
|
105
|
+
}
|
|
106
|
+
return { map, declaredCount, unknownKeys };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* A line an agent can act on, for one capability.
|
|
111
|
+
*
|
|
112
|
+
* The `undeclared` wording matters: it must send the reader to find out, not let
|
|
113
|
+
* them conclude there is nothing there. This is the sentence l3-support reads at
|
|
114
|
+
* 3am, so it says what to do rather than what is missing.
|
|
115
|
+
*/
|
|
116
|
+
export function describeCapability(cap, entry) {
|
|
117
|
+
if (entry.state === 'declared') return `${cap}: ${entry.tool}`;
|
|
118
|
+
if (entry.state === NONE) return `${cap}: none — this project has decided it has no ${cap}`;
|
|
119
|
+
return `${cap}: not declared — do not assume there is none; ask, or fall back to the alert-source routing table`;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The whole block, for injection into an agent brief. */
|
|
123
|
+
export function describeCapabilities({ map, declaredCount, unknownKeys }) {
|
|
124
|
+
const lines = CAPABILITIES.map((c) => ` ${describeCapability(c, map[c])}`);
|
|
125
|
+
const head = declaredCount === 0
|
|
126
|
+
? 'This project declares NO operational capabilities. Nothing below is known;'
|
|
127
|
+
+ ' route by alert source and say so rather than implying the stack was checked.'
|
|
128
|
+
: `This project declares ${declaredCount} of ${CAPABILITIES.length} capabilities.`;
|
|
129
|
+
const warn = unknownKeys.length
|
|
130
|
+
? [` ⚠ unrecognised capability key(s) in PROJECT.md, ignored: ${unknownKeys.join(', ')}`]
|
|
131
|
+
: [];
|
|
132
|
+
return [head, ...lines, ...warn].join('\n');
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// ── CLI ──────────────────────────────────────────────────────────────────────
|
|
136
|
+
// Read by l3-support at the top of an incident, the same way /recall reaches
|
|
137
|
+
// memory-search: a block in the agent's own prompt, no new plumbing.
|
|
138
|
+
//
|
|
139
|
+
// node scripts/lib/stack-capabilities.mjs [--cwd DIR] [--json]
|
|
140
|
+
//
|
|
141
|
+
// Exit 0 always. A project with nothing declared is a fact to report, not an
|
|
142
|
+
// error to fail on — failing here would make an unconfigured project unable to
|
|
143
|
+
// run an incident, which is exactly when you need the agent most.
|
|
144
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
145
|
+
const { readFileSync } = await import('node:fs');
|
|
146
|
+
const { join } = await import('node:path');
|
|
147
|
+
const args = process.argv.slice(2);
|
|
148
|
+
const cwd = args.includes('--cwd') ? args[args.indexOf('--cwd') + 1] : process.cwd();
|
|
149
|
+
const asJson = args.includes('--json');
|
|
150
|
+
|
|
151
|
+
let text = '', read = 'ok';
|
|
152
|
+
try { text = readFileSync(join(cwd, '.great_cto', 'PROJECT.md'), 'utf8'); }
|
|
153
|
+
catch (err) { read = err.code ?? String(err.message); }
|
|
154
|
+
|
|
155
|
+
const result = capabilitiesFromProjectMd(text);
|
|
156
|
+
if (asJson) {
|
|
157
|
+
console.log(JSON.stringify({ read, ...result }, null, 2));
|
|
158
|
+
} else if (read !== 'ok') {
|
|
159
|
+
// Distinct from "declares nothing": there was no PROJECT.md to read at all.
|
|
160
|
+
console.log(`no PROJECT.md under ${cwd}/.great_cto (${read}) — the project's capabilities are unknown, not absent.`);
|
|
161
|
+
} else {
|
|
162
|
+
console.log(describeCapabilities(result));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Set one capability in a PROJECT.md body, returning the new text and what was
|
|
168
|
+
* there before. Pure, so the board's POST can be tested without a filesystem.
|
|
169
|
+
*
|
|
170
|
+
* The previous value is returned, not just the new one: a setting that
|
|
171
|
+
* silently replaced another is a change the operator cannot see they made —
|
|
172
|
+
* the same rule /api/agent-budgets follows.
|
|
173
|
+
*
|
|
174
|
+
* Creates the `capabilities:` block at the end when it is missing. `null` as
|
|
175
|
+
* the value removes the key (back to `undeclared`), which is different from
|
|
176
|
+
* writing `none`.
|
|
177
|
+
*
|
|
178
|
+
* @returns {{text:string, previous:string|null, created:boolean}}
|
|
179
|
+
*/
|
|
180
|
+
export function upsertCapability(text, key, value) {
|
|
181
|
+
if (!CAPABILITIES.includes(key)) throw new Error(`not a capability: ${key}`);
|
|
182
|
+
if (value != null && !/^[A-Za-z0-9_.:@/-]+$/.test(String(value))) throw new Error(`invalid value for ${key}`);
|
|
183
|
+
const body = String(text ?? '');
|
|
184
|
+
const lines = body.split('\n');
|
|
185
|
+
const start = lines.findIndex((l) => /^capabilities:[ \t]*$/.test(l));
|
|
186
|
+
const previous = capabilitiesFromProjectMd(body).map[key]?.tool
|
|
187
|
+
?? (capabilitiesFromProjectMd(body).map[key]?.state === NONE ? NONE : null);
|
|
188
|
+
|
|
189
|
+
if (start === -1) {
|
|
190
|
+
if (value == null) return { text: body, previous, created: false };
|
|
191
|
+
const nl = body.endsWith('\n') || body === '' ? '' : '\n';
|
|
192
|
+
return { text: `${body}${nl}\ncapabilities:\n ${key}: ${value}\n`, previous, created: true };
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
let end = start + 1;
|
|
196
|
+
while (end < lines.length && /^[ \t]+\S/.test(lines[end])) end += 1;
|
|
197
|
+
const idx = lines.findIndex((l, i) => i > start && i < end && new RegExp(`^[ \\t]+${key}[ \\t]*:`).test(l));
|
|
198
|
+
|
|
199
|
+
if (value == null) {
|
|
200
|
+
if (idx !== -1) lines.splice(idx, 1);
|
|
201
|
+
return { text: lines.join('\n'), previous, created: false };
|
|
202
|
+
}
|
|
203
|
+
const line = ` ${key}: ${value}`;
|
|
204
|
+
if (idx !== -1) lines[idx] = line; else lines.splice(end, 0, line);
|
|
205
|
+
return { text: lines.join('\n'), previous, created: false };
|
|
206
|
+
}
|
package/package.json
CHANGED