aiki-cli 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/LICENSE +21 -0
- package/README.md +275 -0
- package/dist/bench/arms.js +104 -0
- package/dist/bench/harness.js +251 -0
- package/dist/bench/results.js +70 -0
- package/dist/bench/scoring/seeded-bugs.js +31 -0
- package/dist/cli/bench.js +50 -0
- package/dist/cli/config.js +51 -0
- package/dist/cli/doctor.js +115 -0
- package/dist/cli/index.js +129 -0
- package/dist/cli/models.js +51 -0
- package/dist/cli/providers.js +31 -0
- package/dist/cli/resolve.js +159 -0
- package/dist/cli/resume.js +94 -0
- package/dist/cli/run.js +155 -0
- package/dist/cli/sessions.js +35 -0
- package/dist/cli/show.js +73 -0
- package/dist/config/config.js +102 -0
- package/dist/config/smoke-cache.js +65 -0
- package/dist/council/open.js +26 -0
- package/dist/council/view.js +873 -0
- package/dist/orchestration/cluster.js +83 -0
- package/dist/orchestration/context.js +277 -0
- package/dist/orchestration/engine.js +92 -0
- package/dist/orchestration/git.js +133 -0
- package/dist/orchestration/jsonStage.js +32 -0
- package/dist/orchestration/skills.js +39 -0
- package/dist/orchestration/stages/cr-ladder.js +63 -0
- package/dist/orchestration/stages/cr-map.js +62 -0
- package/dist/orchestration/stages/cr-report.js +83 -0
- package/dist/orchestration/stages/cr-s4-review.js +69 -0
- package/dist/orchestration/stages/cr-s8-crossexam.js +104 -0
- package/dist/orchestration/stages/cr-s9-judge.js +89 -0
- package/dist/orchestration/stages/s0-grill.js +79 -0
- package/dist/orchestration/stages/s1-intent.js +25 -0
- package/dist/orchestration/stages/s10-render.js +198 -0
- package/dist/orchestration/stages/s2-misread.js +76 -0
- package/dist/orchestration/stages/s3-prompts.js +55 -0
- package/dist/orchestration/stages/s4-analyze.js +50 -0
- package/dist/orchestration/stages/s5-drift.js +40 -0
- package/dist/orchestration/stages/s6-claims.js +56 -0
- package/dist/orchestration/stages/s7-disagreement.js +134 -0
- package/dist/orchestration/stages/s8-verify.js +56 -0
- package/dist/orchestration/stages/s9-judge.js +152 -0
- package/dist/orchestration/stages/s9b-plan.js +192 -0
- package/dist/providers/adapter-core.js +131 -0
- package/dist/providers/adapters.js +9 -0
- package/dist/providers/agy.js +29 -0
- package/dist/providers/claude.js +56 -0
- package/dist/providers/codex.js +35 -0
- package/dist/providers/detect.js +21 -0
- package/dist/providers/probe.js +43 -0
- package/dist/providers/profiles.js +38 -0
- package/dist/providers/profiles.json +5 -0
- package/dist/providers/smoke.js +26 -0
- package/dist/providers/spawn.js +152 -0
- package/dist/providers/types.js +17 -0
- package/dist/schemas/index.js +374 -0
- package/dist/skills/.gitkeep +0 -0
- package/dist/skills/code-review/judge.md +23 -0
- package/dist/skills/code-review/reviewer.md +38 -0
- package/dist/skills/idea-refinement/analyst.md +45 -0
- package/dist/skills/idea-refinement/planner.md +25 -0
- package/dist/storage/feedback.js +111 -0
- package/dist/storage/paths.js +20 -0
- package/dist/storage/replay.js +0 -0
- package/dist/storage/runs-read.js +95 -0
- package/dist/storage/runs.js +129 -0
- package/dist/storage/sessions.js +71 -0
- package/dist/tui/app.js +444 -0
- package/dist/tui/format.js +27 -0
- package/dist/tui/index.js +8 -0
- package/dist/tui/smart-entry.js +106 -0
- package/dist/tui/timeline.js +91 -0
- package/dist/workflows/code-review.js +76 -0
- package/dist/workflows/idea-refinement.js +105 -0
- package/package.json +64 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
// Bench harness (§17, T11). Runs the four arms on a versioned task set, scores each against the case's
|
|
2
|
+
// seeded bugs, and writes bench/results/<suite>-<date>.json. Cases + arms run SEQUENTIALLY and results
|
|
3
|
+
// are written INCREMENTALLY after each case, so a mid-run quota stop keeps completed work (grill 2026-07-04).
|
|
4
|
+
// The bench itself is metered — verified by a scripted-adapter e2e; the real 4-arm run is the user's.
|
|
5
|
+
import { mkdir, readdir, readFile, rename, writeFile } from 'node:fs/promises';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { makeRunId, resolveRoles, RunCtx, setupProviders } from '../orchestration/context.js';
|
|
8
|
+
import { executeRun } from '../orchestration/engine.js';
|
|
9
|
+
import { RunWriter } from '../storage/runs.js';
|
|
10
|
+
import { ARMS, ARM_IDS } from './arms.js';
|
|
11
|
+
import { BugManifest, scoreRun } from './scoring/seeded-bugs.js';
|
|
12
|
+
import { BenchResult, summarize } from './results.js';
|
|
13
|
+
/** Load every case dir under bench/sets/<suite>/<set>/ (each = {diff.patch, bugs.json, source files}). */
|
|
14
|
+
export async function loadCases(suite, set, root = process.cwd()) {
|
|
15
|
+
const base = join(root, 'bench', 'sets', suite, set);
|
|
16
|
+
let entries;
|
|
17
|
+
try {
|
|
18
|
+
entries = await readdir(base, { withFileTypes: true });
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return [];
|
|
22
|
+
}
|
|
23
|
+
const cases = [];
|
|
24
|
+
for (const e of entries.filter((x) => x.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
25
|
+
const dir = join(base, e.name);
|
|
26
|
+
const diff = await readFile(join(dir, 'diff.patch'), 'utf8');
|
|
27
|
+
const bugs = BugManifest.parse(JSON.parse(await readFile(join(dir, 'bugs.json'), 'utf8'))).bugs;
|
|
28
|
+
cases.push({ name: e.name, dir, diff, bugs });
|
|
29
|
+
}
|
|
30
|
+
return cases;
|
|
31
|
+
}
|
|
32
|
+
/** A/B/C need claude (the fixed single model); D needs claude+codex reviewers; E needs agy+codex
|
|
33
|
+
* reviewers + claude judge (the Opus-thrift variant). */
|
|
34
|
+
function armAvailable(arm, available) {
|
|
35
|
+
if (arm === 'D')
|
|
36
|
+
return available.includes('claude') && available.includes('codex');
|
|
37
|
+
if (arm === 'E')
|
|
38
|
+
return available.includes('agy') && available.includes('codex') && available.includes('claude');
|
|
39
|
+
return available.includes('claude');
|
|
40
|
+
}
|
|
41
|
+
/** Precision from FP labels in feedback.jsonl for this run, or null if the run hasn't been adjudicated. */
|
|
42
|
+
async function readPrecision(runId, reported, root = process.cwd()) {
|
|
43
|
+
if (reported === 0)
|
|
44
|
+
return null;
|
|
45
|
+
try {
|
|
46
|
+
const lines = (await readFile(join(root, '.aiki', 'feedback.jsonl'), 'utf8')).trim().split('\n');
|
|
47
|
+
const forRun = lines.map((l) => JSON.parse(l)).filter((e) => e.run_id === runId && e.item_type === 'finding');
|
|
48
|
+
if (forRun.length === 0)
|
|
49
|
+
return null;
|
|
50
|
+
const fp = forRun.filter((e) => e.verdict === 'false-positive').length;
|
|
51
|
+
return (reported - fp) / reported;
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
async function runArmOnCase(arm, c, handles, available, root) {
|
|
58
|
+
const runId = makeRunId('code-review');
|
|
59
|
+
// Arm C's synthesis judge stays same-model; Arm E swaps the product pipeline's roles to the
|
|
60
|
+
// Opus-thrift config (agy+codex hunt, claude judge). D uses code-review defaults.
|
|
61
|
+
const overrides = arm === 'C' ? { judge: 'claude' }
|
|
62
|
+
: arm === 'E' ? { s4: ['agy', 'codex'], judge: 'claude' }
|
|
63
|
+
: undefined;
|
|
64
|
+
const roles = resolveRoles('code-review', available, overrides);
|
|
65
|
+
const ctx = new RunCtx({ runId, workflow: 'code-review', handles, roles, writer: new RunWriter(runId, join(root, '.aiki')), cwd: c.dir });
|
|
66
|
+
const started = Date.now();
|
|
67
|
+
let findings = [];
|
|
68
|
+
const outcome = await executeRun(ctx, c.diff, async (cx, input) => {
|
|
69
|
+
findings = await ARMS[arm](cx, input);
|
|
70
|
+
});
|
|
71
|
+
const wallMs = Date.now() - started;
|
|
72
|
+
if (!outcome.ok) {
|
|
73
|
+
return { arm, status: 'error', runId, reason: `${outcome.error?.code}: ${outcome.error?.message}`, calls: ctx.calls.length, wallMs };
|
|
74
|
+
}
|
|
75
|
+
// A scored outcome can still hide a mid-run provider failure: fault-tolerant arms (D) finish on the
|
|
76
|
+
// surviving providers when a reviewer crashes (e.g. Opus quota mid-run), so the result is NOT the
|
|
77
|
+
// registered pipeline. Don't score it as clean — mark it error so a `--resume` run retries it.
|
|
78
|
+
const failed = ctx.calls.filter((call) => call.error);
|
|
79
|
+
if (failed.length > 0) {
|
|
80
|
+
const detail = failed.map((call) => `${call.provider}@${call.stage}=${call.error}`).join(', ');
|
|
81
|
+
return { arm, status: 'error', runId, reason: `DEGRADED: provider call failed mid-run (${detail})`, calls: ctx.calls.length, wallMs };
|
|
82
|
+
}
|
|
83
|
+
// Persist the exact findings the scorer saw — single-call arms (A/B) have no review-map artifact,
|
|
84
|
+
// so this is what `aiki resolve` annotates for FP labels (precision's denominator = this list).
|
|
85
|
+
await ctx.writer.writeRaw('bench-findings.json', JSON.stringify(findings, null, 2));
|
|
86
|
+
const s = scoreRun(findings, c.bugs);
|
|
87
|
+
return {
|
|
88
|
+
arm,
|
|
89
|
+
status: 'scored',
|
|
90
|
+
runId,
|
|
91
|
+
seeded: s.seeded,
|
|
92
|
+
matched: s.matched,
|
|
93
|
+
recall: s.recall,
|
|
94
|
+
reported: s.reported,
|
|
95
|
+
unmatched: s.unmatched,
|
|
96
|
+
precision: await readPrecision(runId, s.reported, root),
|
|
97
|
+
calls: ctx.calls.length,
|
|
98
|
+
wallMs,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
/** Approx claude/Opus calls each arm makes per case — for the pre-run quota estimate (§19). Not exact:
|
|
102
|
+
* §14 JSON repairs can add a few, and D's cross-exam/judge vary; deliberately a round upper-ish figure. */
|
|
103
|
+
export const CLAUDE_CALLS_PER_CASE = { A: 1, B: 1, C: 4, D: 2, E: 1 };
|
|
104
|
+
/** ≈ claude/Opus calls for a list of case×arm pairs (the quota-sensitive cost the user cares about). */
|
|
105
|
+
export function estimateClaudeCalls(pairs) {
|
|
106
|
+
return pairs.reduce((n, p) => n + (CLAUDE_CALLS_PER_CASE[p.arm] ?? 0), 0);
|
|
107
|
+
}
|
|
108
|
+
/** Resume target: continue the most-recent results file for this suite/set (so a run can be spread across
|
|
109
|
+
* Opus windows, even across midnight); otherwise a fresh dated file. */
|
|
110
|
+
async function resolveCampaign(resultsDir, suite, set, resume) {
|
|
111
|
+
const dated = join(resultsDir, `${suite}-${new Date().toISOString().slice(0, 10)}.json`);
|
|
112
|
+
if (!resume)
|
|
113
|
+
return { path: dated };
|
|
114
|
+
// Only real `<suite>-YYYY-MM-DD.json` campaign files — so an archived/renamed run (e.g.
|
|
115
|
+
// `<suite>-2026-07-04.void.json`) is ignored and won't be resumed by accident.
|
|
116
|
+
const campaign = new RegExp(`^${suite}-\\d{4}-\\d{2}-\\d{2}\\.json$`);
|
|
117
|
+
let names = [];
|
|
118
|
+
try {
|
|
119
|
+
names = (await readdir(resultsDir)).filter((f) => campaign.test(f));
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
return { path: dated };
|
|
123
|
+
}
|
|
124
|
+
names.sort(); // ISO date in the name → lexical order is chronological
|
|
125
|
+
for (const name of names.reverse()) {
|
|
126
|
+
try {
|
|
127
|
+
const parsed = BenchResult.safeParse(JSON.parse(await readFile(join(resultsDir, name), 'utf8')));
|
|
128
|
+
if (parsed.success && parsed.data.suite === suite && parsed.data.set === set) {
|
|
129
|
+
return { path: join(resultsDir, name), prior: parsed.data };
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
/* skip an unparseable/partial file, try the next-most-recent */
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return { path: dated };
|
|
137
|
+
}
|
|
138
|
+
/** Map of `case::arm` → the prior ArmScore for pairs already `scored` (resume reuses these; error/skipped are retried). */
|
|
139
|
+
function priorScored(prior) {
|
|
140
|
+
const done = new Map();
|
|
141
|
+
for (const pc of prior?.cases ?? []) {
|
|
142
|
+
for (const a of pc.arms)
|
|
143
|
+
if (a.status === 'scored')
|
|
144
|
+
done.set(`${pc.case}::${a.arm}`, a);
|
|
145
|
+
}
|
|
146
|
+
return done;
|
|
147
|
+
}
|
|
148
|
+
/** Dry-run: resolve what a `runBench` with these options would execute + its ≈Opus cost. No model calls. */
|
|
149
|
+
export async function planBench(opts = {}) {
|
|
150
|
+
const suite = opts.suite ?? 'code-review';
|
|
151
|
+
const set = opts.set ?? 'build';
|
|
152
|
+
const arms = opts.arms ?? [...ARM_IDS];
|
|
153
|
+
const root = opts.root ?? process.cwd();
|
|
154
|
+
const handles = opts.handles ?? (await setupProviders());
|
|
155
|
+
const available = handles.map((h) => h.id);
|
|
156
|
+
const cases = await loadCases(suite, set, root);
|
|
157
|
+
const resultsDir = join(root, 'bench', 'results');
|
|
158
|
+
const { path, prior } = await resolveCampaign(resultsDir, suite, set, !!opts.resume);
|
|
159
|
+
const done = priorScored(prior);
|
|
160
|
+
const toRun = [];
|
|
161
|
+
let skipCompleted = 0;
|
|
162
|
+
let skipUnavailable = 0;
|
|
163
|
+
for (const c of cases) {
|
|
164
|
+
for (const arm of arms) {
|
|
165
|
+
if (opts.resume && done.has(`${c.name}::${arm}`))
|
|
166
|
+
skipCompleted++;
|
|
167
|
+
else if (!armAvailable(arm, available))
|
|
168
|
+
skipUnavailable++;
|
|
169
|
+
else
|
|
170
|
+
toRun.push({ case: c.name, arm });
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
suite,
|
|
175
|
+
set,
|
|
176
|
+
arms,
|
|
177
|
+
cases: cases.map((c) => c.name),
|
|
178
|
+
toRun,
|
|
179
|
+
skipCompleted,
|
|
180
|
+
skipUnavailable,
|
|
181
|
+
estClaudeCalls: estimateClaudeCalls(toRun),
|
|
182
|
+
resultsPath: path,
|
|
183
|
+
resumedFrom: prior ? path : undefined,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
export async function runBench(opts = {}) {
|
|
187
|
+
const suite = opts.suite ?? 'code-review';
|
|
188
|
+
const set = opts.set ?? 'build';
|
|
189
|
+
const arms = opts.arms ?? [...ARM_IDS];
|
|
190
|
+
const root = opts.root ?? process.cwd();
|
|
191
|
+
const handles = opts.handles ?? (await setupProviders());
|
|
192
|
+
const available = handles.map((h) => h.id);
|
|
193
|
+
const cases = await loadCases(suite, set, root);
|
|
194
|
+
const resultsDir = join(root, 'bench', 'results');
|
|
195
|
+
await mkdir(resultsDir, { recursive: true });
|
|
196
|
+
const { path, prior } = await resolveCampaign(resultsDir, suite, set, !!opts.resume);
|
|
197
|
+
const done = opts.resume ? priorScored(prior) : new Map();
|
|
198
|
+
const result = { suite, set, at: new Date().toISOString(), arms, cases: [], summary: [] };
|
|
199
|
+
for (const c of cases) {
|
|
200
|
+
const caseResult = { case: c.name, seeded: c.bugs.length, arms: [] };
|
|
201
|
+
for (const arm of arms) {
|
|
202
|
+
const kept = done.get(`${c.name}::${arm}`);
|
|
203
|
+
if (kept) {
|
|
204
|
+
caseResult.arms.push(kept); // resume: keep already-scored work across quota windows
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
if (!armAvailable(arm, available)) {
|
|
208
|
+
caseResult.arms.push({ arm, status: 'skipped', reason: `required provider(s) unavailable` });
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
caseResult.arms.push(await runArmOnCase(arm, c, handles, available, root));
|
|
212
|
+
}
|
|
213
|
+
// Carry forward prior scored pairs for arms NOT requested this run — else a narrower `--arms`
|
|
214
|
+
// re-run (e.g. C-only after B,D) would rewrite the campaign file without the paid-for B/D data.
|
|
215
|
+
const priorCase = prior?.cases.find((pc) => pc.case === c.name);
|
|
216
|
+
for (const a of priorCase?.arms ?? []) {
|
|
217
|
+
if (a.status === 'scored' && !arms.includes(a.arm))
|
|
218
|
+
caseResult.arms.push(a);
|
|
219
|
+
}
|
|
220
|
+
result.cases.push(caseResult);
|
|
221
|
+
result.summary = summarize(result.cases, arms);
|
|
222
|
+
// Survive a mid-run stop: write processed cases + the prior file's not-yet-reprocessed cases
|
|
223
|
+
// (a kill mid-loop must not drop prior scored data for later cases from the campaign file).
|
|
224
|
+
const processed = new Set(result.cases.map((x) => x.case));
|
|
225
|
+
const pending = (prior?.cases ?? []).filter((pc) => !processed.has(pc.case));
|
|
226
|
+
await writeIncremental(path, { ...result, cases: [...result.cases, ...pending] });
|
|
227
|
+
}
|
|
228
|
+
return result;
|
|
229
|
+
}
|
|
230
|
+
/** Atomic incremental write (temp + rename). */
|
|
231
|
+
async function writeIncremental(path, result) {
|
|
232
|
+
const tmp = `${path}.tmp`;
|
|
233
|
+
await writeFile(tmp, JSON.stringify(result, null, 2), 'utf8');
|
|
234
|
+
await rename(tmp, path);
|
|
235
|
+
}
|
|
236
|
+
/** Render the per-arm summary table (recall micro/macro, precision, calls, wall-clock). */
|
|
237
|
+
export function renderTable(result) {
|
|
238
|
+
const L = [];
|
|
239
|
+
L.push(`bench ${result.suite} — set ${result.set} — ${result.cases.length} case(s)`);
|
|
240
|
+
L.push('');
|
|
241
|
+
L.push('| Arm | Recall (micro) | Recall (macro) | Matched/Seeded | Reported | Unmatched(FP?) | Precision | Calls | Wall(s) |');
|
|
242
|
+
L.push('|---|---|---|---|---|---|---|---|---|');
|
|
243
|
+
for (const r of result.summary) {
|
|
244
|
+
const pct = (n) => `${(n * 100).toFixed(0)}%`;
|
|
245
|
+
const prec = r.precision === null ? '—' : pct(r.precision);
|
|
246
|
+
L.push(`| ${r.arm} | ${pct(r.recall)} | ${pct(r.recallMacro)} | ${r.matched}/${r.seeded} | ${r.reported} | ${r.unmatched} | ${prec} | ${r.calls} | ${(r.wallMs / 1000).toFixed(1)} |`);
|
|
247
|
+
}
|
|
248
|
+
L.push('');
|
|
249
|
+
L.push('Precision "—" = not yet adjudicated (label false positives with `aiki resolve <run>`).');
|
|
250
|
+
return L.join('\n');
|
|
251
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// Bench result schema (§17, BENCHMARK.md §5, T11). Per-case per-arm scores + a summary, written to
|
|
2
|
+
// bench/results/<suite>-<date>.json. Precision is nullable — it needs FP adjudication (resolve-CR), which
|
|
3
|
+
// happens after the run; recall/calls/wall-clock are automatic. Aggregate recall is MICRO (grill 2026-07-04).
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
export const ArmScore = z.object({
|
|
6
|
+
arm: z.enum(['A', 'B', 'C', 'D', 'E']),
|
|
7
|
+
status: z.enum(['scored', 'skipped', 'error']),
|
|
8
|
+
reason: z.string().optional(), // why skipped / the error message
|
|
9
|
+
runId: z.string().optional(),
|
|
10
|
+
seeded: z.number().int().nonnegative().optional(),
|
|
11
|
+
matched: z.number().int().nonnegative().optional(),
|
|
12
|
+
recall: z.number().optional(),
|
|
13
|
+
reported: z.number().int().nonnegative().optional(),
|
|
14
|
+
unmatched: z.number().int().nonnegative().optional(), // candidate FPs, UNADJUDICATED
|
|
15
|
+
precision: z.number().nullable().optional(), // null until FP-labelled via resolve
|
|
16
|
+
calls: z.number().int().nonnegative().optional(),
|
|
17
|
+
wallMs: z.number().nonnegative().optional(),
|
|
18
|
+
});
|
|
19
|
+
export const CaseResult = z.object({
|
|
20
|
+
case: z.string(),
|
|
21
|
+
seeded: z.number().int().nonnegative(),
|
|
22
|
+
arms: z.array(ArmScore),
|
|
23
|
+
});
|
|
24
|
+
/** Per-arm rollup across all cases. `recall` = micro (total matched / total seeded); `recallMacro` = mean of per-case. */
|
|
25
|
+
export const SummaryRow = z.object({
|
|
26
|
+
arm: z.enum(['A', 'B', 'C', 'D', 'E']),
|
|
27
|
+
cases: z.number().int().nonnegative(), // scored cases
|
|
28
|
+
seeded: z.number().int().nonnegative(),
|
|
29
|
+
matched: z.number().int().nonnegative(),
|
|
30
|
+
recall: z.number(), // micro
|
|
31
|
+
recallMacro: z.number(),
|
|
32
|
+
reported: z.number().int().nonnegative(),
|
|
33
|
+
unmatched: z.number().int().nonnegative(),
|
|
34
|
+
precision: z.number().nullable(), // micro precision if any labels, else null
|
|
35
|
+
calls: z.number().int().nonnegative(),
|
|
36
|
+
wallMs: z.number().nonnegative(),
|
|
37
|
+
});
|
|
38
|
+
export const BenchResult = z.object({
|
|
39
|
+
suite: z.string(),
|
|
40
|
+
set: z.string(),
|
|
41
|
+
at: z.string(),
|
|
42
|
+
arms: z.array(z.enum(['A', 'B', 'C', 'D', 'E'])),
|
|
43
|
+
cases: z.array(CaseResult),
|
|
44
|
+
summary: z.array(SummaryRow),
|
|
45
|
+
});
|
|
46
|
+
/** Roll per-case arm scores up into the per-arm summary (micro recall, macro shown alongside). */
|
|
47
|
+
export function summarize(cases, arms) {
|
|
48
|
+
return arms.map((arm) => {
|
|
49
|
+
const scored = cases.flatMap((c) => c.arms.filter((a) => a.arm === arm && a.status === 'scored'));
|
|
50
|
+
const sum = (pick) => scored.reduce((s, a) => s + pick(a), 0);
|
|
51
|
+
const seeded = sum((a) => a.seeded ?? 0);
|
|
52
|
+
const matched = sum((a) => a.matched ?? 0);
|
|
53
|
+
const reported = sum((a) => a.reported ?? 0);
|
|
54
|
+
const recallMacro = scored.length ? scored.reduce((s, a) => s + (a.recall ?? 0), 0) / scored.length : 0;
|
|
55
|
+
const precisions = scored.map((a) => a.precision).filter((p) => typeof p === 'number');
|
|
56
|
+
return {
|
|
57
|
+
arm,
|
|
58
|
+
cases: scored.length,
|
|
59
|
+
seeded,
|
|
60
|
+
matched,
|
|
61
|
+
recall: seeded ? matched / seeded : 0,
|
|
62
|
+
recallMacro,
|
|
63
|
+
reported,
|
|
64
|
+
unmatched: sum((a) => a.unmatched ?? 0),
|
|
65
|
+
precision: precisions.length ? precisions.reduce((s, p) => s + p, 0) / precisions.length : null,
|
|
66
|
+
calls: sum((a) => a.calls ?? 0),
|
|
67
|
+
wallMs: sum((a) => a.wallMs ?? 0),
|
|
68
|
+
};
|
|
69
|
+
});
|
|
70
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// Seeded-bug scoring for code-review (§17, BENCHMARK.md §3, T11). A seeded bug is ground truth: a
|
|
2
|
+
// known defect at a file + line range + category. A reported finding counts as FOUND iff it matches on
|
|
3
|
+
// same file + overlapping lines + same category — which is exactly `sameFinding` (the §487 matcher).
|
|
4
|
+
// Recall is fully automatic; precision needs FP adjudication (resolve-CR), so it lives elsewhere.
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
import { FindingCategory } from '../../schemas/index.js';
|
|
7
|
+
import { sameFinding } from '../../orchestration/stages/cr-map.js';
|
|
8
|
+
/** One ground-truth seeded bug. `category` is the frozen match key; `class` is a fine, doc-only label. */
|
|
9
|
+
export const SeededBug = z.object({
|
|
10
|
+
id: z.string().min(1),
|
|
11
|
+
file: z.string().min(1),
|
|
12
|
+
line_start: z.number().int().nonnegative(),
|
|
13
|
+
line_end: z.number().int().nonnegative(),
|
|
14
|
+
category: FindingCategory,
|
|
15
|
+
class: z.string().optional(), // "off-by-one" | "race" | ... (documentation only)
|
|
16
|
+
});
|
|
17
|
+
/** The bugs.json in each build case: the seeded defects for its diff. */
|
|
18
|
+
export const BugManifest = z.object({ bugs: z.array(SeededBug) });
|
|
19
|
+
/** Score one arm's findings against a case's seeded bugs (BENCHMARK.md §3). Pure. */
|
|
20
|
+
export function scoreRun(findings, bugs) {
|
|
21
|
+
const matchedBugIds = bugs.filter((bug) => findings.some((f) => sameFinding(f, bug))).map((b) => b.id);
|
|
22
|
+
const unmatched = findings.filter((f) => !bugs.some((bug) => sameFinding(f, bug))).length;
|
|
23
|
+
return {
|
|
24
|
+
seeded: bugs.length,
|
|
25
|
+
matched: matchedBugIds.length,
|
|
26
|
+
recall: bugs.length === 0 ? 0 : matchedBugIds.length / bugs.length,
|
|
27
|
+
reported: findings.length,
|
|
28
|
+
unmatched,
|
|
29
|
+
matchedBugIds,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// `aiki bench <workflow> [--arms A,B,C,D] [--set build] [--resume] [--yes]` (§5, §17, §19) — run the
|
|
2
|
+
// benchmark arms on a task set, write bench/results/<suite>-<date>.json, print the per-arm summary table.
|
|
3
|
+
// Without --yes it prints a pre-run Opus-call estimate and exits (so a big sweep never silently drains the
|
|
4
|
+
// quota). --resume continues the latest results file, keeping already-scored case×arm pairs.
|
|
5
|
+
import { planBench, renderTable, runBench } from '../bench/harness.js';
|
|
6
|
+
import { setupProviders } from '../orchestration/context.js';
|
|
7
|
+
const VALID_ARMS = ['A', 'B', 'C', 'D', 'E'];
|
|
8
|
+
/** One-block pre-run summary: what will run + the ≈Opus cost, so the user commits knowingly (§19). */
|
|
9
|
+
function renderPlan(plan) {
|
|
10
|
+
const L = [`bench ${plan.suite} — set ${plan.set} — ${plan.cases.length} case(s) × arms ${plan.arms.join(',')}`];
|
|
11
|
+
if (plan.resumedFrom)
|
|
12
|
+
L.push(`resume: continuing ${plan.resumedFrom} — ${plan.skipCompleted} case×arm already scored (kept)`);
|
|
13
|
+
const unavail = plan.skipUnavailable ? ` · ${plan.skipUnavailable} skipped (provider unavailable)` : '';
|
|
14
|
+
L.push(`to run: ${plan.toRun.length} case×arm pair(s) → ≈${plan.estClaudeCalls} claude/Opus call(s)${unavail}`);
|
|
15
|
+
return L.join('\n');
|
|
16
|
+
}
|
|
17
|
+
export async function benchCommand(workflow, opts = {}) {
|
|
18
|
+
if (workflow !== 'code-review') {
|
|
19
|
+
process.stderr.write(`bench supports only "code-review" in v1 (got "${workflow}")\n`);
|
|
20
|
+
return 1;
|
|
21
|
+
}
|
|
22
|
+
const arms = (opts.arms ?? 'A,B,C,D')
|
|
23
|
+
.split(',')
|
|
24
|
+
.map((s) => s.trim().toUpperCase())
|
|
25
|
+
.filter((a) => VALID_ARMS.includes(a));
|
|
26
|
+
if (arms.length === 0) {
|
|
27
|
+
process.stderr.write(`no valid arms in "${opts.arms}". Valid: A,B,C,D\n`);
|
|
28
|
+
return 1;
|
|
29
|
+
}
|
|
30
|
+
const set = opts.set ?? 'build';
|
|
31
|
+
const handles = await setupProviders(); // detection only (no model calls); shared by plan + run
|
|
32
|
+
const plan = await planBench({ suite: 'code-review', set, arms, resume: opts.resume, handles });
|
|
33
|
+
if (plan.cases.length === 0) {
|
|
34
|
+
process.stderr.write(`no cases found in bench/sets/code-review/${set}/ — create <name>/{diff.patch,bugs.json} case dirs first\n`);
|
|
35
|
+
return 1;
|
|
36
|
+
}
|
|
37
|
+
process.stdout.write(`\n${renderPlan(plan)}\n`);
|
|
38
|
+
if (plan.toRun.length === 0) {
|
|
39
|
+
process.stdout.write(`\nnothing to run — every requested arm is already scored on every case.\n\n`);
|
|
40
|
+
return 0;
|
|
41
|
+
}
|
|
42
|
+
if (!opts.yes) {
|
|
43
|
+
const resumeHint = opts.resume ? '' : ' (add --resume to continue a partial run across quota windows)';
|
|
44
|
+
process.stdout.write(`\nThis makes ≈${plan.estClaudeCalls} claude/Opus call(s). Re-run with --yes to execute${resumeHint}.\n\n`);
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
const result = await runBench({ suite: 'code-review', set, arms, resume: opts.resume, handles });
|
|
48
|
+
process.stdout.write(`\n${renderTable(result)}\n\n results: ${plan.resultsPath}\n\n`);
|
|
49
|
+
return 0;
|
|
50
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// `aiki config` (§5/§128) — print the effective config (defaults merged with .aiki/config.json);
|
|
2
|
+
// `--edit` opens .aiki/config.json in $VISUAL/$EDITOR, creating it with `{}` if missing.
|
|
3
|
+
import { spawn } from 'node:child_process';
|
|
4
|
+
import { access, mkdir, writeFile } from 'node:fs/promises';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { ConfigError, effectiveConfig, loadLayeredConfig } from '../config/config.js';
|
|
7
|
+
const ROOT = '.aiki';
|
|
8
|
+
const CONFIG_PATH = join(ROOT, 'config.json');
|
|
9
|
+
/** Open the config file in the user's editor, creating an empty `{}` scaffold first if it's missing. */
|
|
10
|
+
async function editConfig() {
|
|
11
|
+
await mkdir(ROOT, { recursive: true });
|
|
12
|
+
try {
|
|
13
|
+
await access(CONFIG_PATH);
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
await writeFile(CONFIG_PATH, '{}\n', 'utf8');
|
|
17
|
+
}
|
|
18
|
+
const editor = process.env.VISUAL || process.env.EDITOR;
|
|
19
|
+
if (!editor) {
|
|
20
|
+
process.stdout.write(`no $VISUAL/$EDITOR set — edit ${CONFIG_PATH} manually.\n`);
|
|
21
|
+
return 0;
|
|
22
|
+
}
|
|
23
|
+
const parts = editor.split(/\s+/);
|
|
24
|
+
const cmd = parts[0];
|
|
25
|
+
const args = parts.slice(1);
|
|
26
|
+
return new Promise((res) => {
|
|
27
|
+
const child = spawn(cmd, [...args, CONFIG_PATH], { stdio: 'inherit' });
|
|
28
|
+
child.on('exit', (code) => res(code ?? 0));
|
|
29
|
+
child.on('error', () => {
|
|
30
|
+
process.stderr.write(`failed to launch editor "${editor}" — edit ${CONFIG_PATH} manually.\n`);
|
|
31
|
+
res(1);
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
export async function config(opts = {}) {
|
|
36
|
+
if (opts.edit)
|
|
37
|
+
return editConfig();
|
|
38
|
+
try {
|
|
39
|
+
const eff = effectiveConfig(await loadLayeredConfig()); // global ~/.aiki base + this project's .aiki
|
|
40
|
+
process.stdout.write(`${JSON.stringify(eff, null, 2)}\n`);
|
|
41
|
+
process.stderr.write('(roles shown are config pins; the actual per-run assignment also depends on which providers are available)\n');
|
|
42
|
+
return 0;
|
|
43
|
+
}
|
|
44
|
+
catch (e) {
|
|
45
|
+
if (e instanceof ConfigError) {
|
|
46
|
+
process.stderr.write(`${e.message}\n`);
|
|
47
|
+
return 1;
|
|
48
|
+
}
|
|
49
|
+
throw e;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { DISPLAY_NAME, PROVIDER_IDS } from '../providers/types.js';
|
|
2
|
+
import { detect } from '../providers/detect.js';
|
|
3
|
+
import { probeFlags } from '../providers/probe.js';
|
|
4
|
+
import { smokeTest } from '../providers/smoke.js';
|
|
5
|
+
import { entryToSmoke, isFresh, readSmokeCache, toEntry, writeSmokeCache } from '../config/smoke-cache.js';
|
|
6
|
+
// Default role labels for the status panel (§4.1 / §10). Assignment logic is T5+.
|
|
7
|
+
// NOTE: agy = Gemini 3.1 Pro (strong + metered), not the old cheap/free gemini — §10 role
|
|
8
|
+
// rationale should be revisited at T5.
|
|
9
|
+
const ROLE_LABEL = {
|
|
10
|
+
claude: 'judge',
|
|
11
|
+
codex: 'critic/verifier',
|
|
12
|
+
agy: 'analyst/prompt-builder',
|
|
13
|
+
};
|
|
14
|
+
const READONLY_LABEL = {
|
|
15
|
+
plan: 'plan',
|
|
16
|
+
sandbox: 'sandbox',
|
|
17
|
+
none: '⚠ none',
|
|
18
|
+
};
|
|
19
|
+
const pad = (s, w) => s.padEnd(w);
|
|
20
|
+
/**
|
|
21
|
+
* `aiki doctor` — detection + flag probe + (optional) smoke test, table output, actionable
|
|
22
|
+
* fixes. Exit 0 iff ≥2 providers are "ready" (§5, §8 quorum). With smoke on, ready = smoke
|
|
23
|
+
* passed; with --no-smoke, ready = detected.
|
|
24
|
+
*
|
|
25
|
+
* Smoke results are cached 6h in `.aiki/smoke-cache.json` (§242); `--fresh` bypasses the cache.
|
|
26
|
+
* A cache entry is reused only when its provider version still matches (an upgrade re-smokes).
|
|
27
|
+
*/
|
|
28
|
+
export async function doctor(opts = {}) {
|
|
29
|
+
const runSmoke = opts.smoke !== false;
|
|
30
|
+
const cache = runSmoke ? await readSmokeCache() : {};
|
|
31
|
+
const now = Date.now();
|
|
32
|
+
const updates = {};
|
|
33
|
+
const rows = await Promise.all(PROVIDER_IDS.map(async (id) => {
|
|
34
|
+
const det = await detect(id);
|
|
35
|
+
if (det.status !== 'READY')
|
|
36
|
+
return { det };
|
|
37
|
+
const flags = await probeFlags(id);
|
|
38
|
+
if (!runSmoke)
|
|
39
|
+
return { det, flags };
|
|
40
|
+
const cached = opts.fresh ? undefined : cache[id];
|
|
41
|
+
if (cached && isFresh(cached, det.version ?? null, now)) {
|
|
42
|
+
return { det, flags, smoke: entryToSmoke(cached), cached: true };
|
|
43
|
+
}
|
|
44
|
+
const smoke = await smokeTest(id, flags);
|
|
45
|
+
updates[id] = toEntry(smoke, det.version ?? null, new Date(now));
|
|
46
|
+
return { det, flags, smoke };
|
|
47
|
+
}));
|
|
48
|
+
if (Object.keys(updates).length)
|
|
49
|
+
await writeSmokeCache({ ...cache, ...updates });
|
|
50
|
+
const lines = [];
|
|
51
|
+
lines.push('');
|
|
52
|
+
lines.push(` aiki doctor — provider status${runSmoke ? '' : ' (smoke skipped)'}`);
|
|
53
|
+
lines.push('');
|
|
54
|
+
lines.push(` ${pad('PROVIDER', 10)}${pad('VERSION', 11)}${pad('STATUS', 15)}${pad('JSON', 6)}${pad('READ-ONLY', 11)}${pad('SMOKE', 14)}ROLE`);
|
|
55
|
+
lines.push(` ${'─'.repeat(82)}`);
|
|
56
|
+
let ready = 0;
|
|
57
|
+
const fixes = [];
|
|
58
|
+
for (const row of rows) {
|
|
59
|
+
lines.push(renderRow(row, runSmoke));
|
|
60
|
+
if (isReady(row, runSmoke))
|
|
61
|
+
ready++;
|
|
62
|
+
fixes.push(...fixLines(row));
|
|
63
|
+
}
|
|
64
|
+
lines.push('');
|
|
65
|
+
lines.push(` ${ready}/3 providers ready. Engine minimum quorum: 2.`);
|
|
66
|
+
if (rows.some((r) => r.cached))
|
|
67
|
+
lines.push(' (smoke: cached ≤6h — `aiki doctor --fresh` re-runs)');
|
|
68
|
+
if (fixes.length) {
|
|
69
|
+
lines.push('');
|
|
70
|
+
lines.push(' Fixes:');
|
|
71
|
+
lines.push(...fixes);
|
|
72
|
+
}
|
|
73
|
+
lines.push('');
|
|
74
|
+
process.stdout.write(lines.join('\n') + '\n');
|
|
75
|
+
return ready >= 2 ? 0 : 1;
|
|
76
|
+
}
|
|
77
|
+
function isReady(row, runSmoke) {
|
|
78
|
+
if (row.det.status !== 'READY')
|
|
79
|
+
return false;
|
|
80
|
+
return runSmoke ? row.smoke?.ok === true : true;
|
|
81
|
+
}
|
|
82
|
+
function renderRow(row, runSmoke) {
|
|
83
|
+
const { det, flags, smoke } = row;
|
|
84
|
+
const status = det.status === 'READY' ? '✔ ready' : '✖ NOT_INSTALLED';
|
|
85
|
+
const version = det.version ?? '—';
|
|
86
|
+
const json = flags ? (flags.jsonOutput ? 'yes' : 'no') : '—';
|
|
87
|
+
const ro = flags ? READONLY_LABEL[flags.readOnlyFlag] : '—';
|
|
88
|
+
const smokeCell = renderSmoke(det.status === 'READY', runSmoke, smoke);
|
|
89
|
+
return ` ${pad(DISPLAY_NAME[det.id], 10)}${pad(version, 11)}${pad(status, 15)}${pad(json, 6)}${pad(ro, 11)}${pad(smokeCell, 14)}${ROLE_LABEL[det.id]}`;
|
|
90
|
+
}
|
|
91
|
+
function renderSmoke(detected, runSmoke, smoke) {
|
|
92
|
+
if (!detected || !runSmoke)
|
|
93
|
+
return '—';
|
|
94
|
+
if (!smoke)
|
|
95
|
+
return '—';
|
|
96
|
+
if (smoke.ok)
|
|
97
|
+
return `✔ ${(smoke.durationMs / 1000).toFixed(1)}s`;
|
|
98
|
+
return `✖ ${smoke.error ?? 'FAIL'}`;
|
|
99
|
+
}
|
|
100
|
+
function fixLines(row) {
|
|
101
|
+
const { det, smoke } = row;
|
|
102
|
+
const name = DISPLAY_NAME[det.id];
|
|
103
|
+
// Fixes are user-facing (show display name) but reference the real binary for commands.
|
|
104
|
+
if (det.status !== 'READY' && det.hint)
|
|
105
|
+
return [` ${name}: ${det.hint}`];
|
|
106
|
+
if (smoke && !smoke.ok) {
|
|
107
|
+
const fix = smoke.error === 'AUTH'
|
|
108
|
+
? `run \`${det.id}\` once to log in`
|
|
109
|
+
: smoke.error === 'QUOTA'
|
|
110
|
+
? 'provider quota/rate limited — retry later'
|
|
111
|
+
: (smoke.detail ?? 'smoke failed');
|
|
112
|
+
return [` ${name}: ${fix}`];
|
|
113
|
+
}
|
|
114
|
+
return [];
|
|
115
|
+
}
|