aiki-cli 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/LICENSE +21 -0
- package/README.md +275 -0
- package/dist/bench/arms.js +104 -0
- package/dist/bench/harness.js +251 -0
- package/dist/bench/results.js +70 -0
- package/dist/bench/scoring/seeded-bugs.js +31 -0
- package/dist/cli/bench.js +50 -0
- package/dist/cli/config.js +51 -0
- package/dist/cli/doctor.js +115 -0
- package/dist/cli/index.js +129 -0
- package/dist/cli/models.js +51 -0
- package/dist/cli/providers.js +31 -0
- package/dist/cli/resolve.js +159 -0
- package/dist/cli/resume.js +94 -0
- package/dist/cli/run.js +155 -0
- package/dist/cli/sessions.js +35 -0
- package/dist/cli/show.js +73 -0
- package/dist/config/config.js +102 -0
- package/dist/config/smoke-cache.js +65 -0
- package/dist/council/open.js +26 -0
- package/dist/council/view.js +873 -0
- package/dist/orchestration/cluster.js +83 -0
- package/dist/orchestration/context.js +277 -0
- package/dist/orchestration/engine.js +92 -0
- package/dist/orchestration/git.js +133 -0
- package/dist/orchestration/jsonStage.js +32 -0
- package/dist/orchestration/skills.js +39 -0
- package/dist/orchestration/stages/cr-ladder.js +63 -0
- package/dist/orchestration/stages/cr-map.js +62 -0
- package/dist/orchestration/stages/cr-report.js +83 -0
- package/dist/orchestration/stages/cr-s4-review.js +69 -0
- package/dist/orchestration/stages/cr-s8-crossexam.js +104 -0
- package/dist/orchestration/stages/cr-s9-judge.js +89 -0
- package/dist/orchestration/stages/s0-grill.js +79 -0
- package/dist/orchestration/stages/s1-intent.js +25 -0
- package/dist/orchestration/stages/s10-render.js +198 -0
- package/dist/orchestration/stages/s2-misread.js +76 -0
- package/dist/orchestration/stages/s3-prompts.js +55 -0
- package/dist/orchestration/stages/s4-analyze.js +50 -0
- package/dist/orchestration/stages/s5-drift.js +40 -0
- package/dist/orchestration/stages/s6-claims.js +56 -0
- package/dist/orchestration/stages/s7-disagreement.js +134 -0
- package/dist/orchestration/stages/s8-verify.js +56 -0
- package/dist/orchestration/stages/s9-judge.js +152 -0
- package/dist/orchestration/stages/s9b-plan.js +192 -0
- package/dist/providers/adapter-core.js +131 -0
- package/dist/providers/adapters.js +9 -0
- package/dist/providers/agy.js +29 -0
- package/dist/providers/claude.js +56 -0
- package/dist/providers/codex.js +35 -0
- package/dist/providers/detect.js +21 -0
- package/dist/providers/probe.js +43 -0
- package/dist/providers/profiles.js +38 -0
- package/dist/providers/profiles.json +5 -0
- package/dist/providers/smoke.js +26 -0
- package/dist/providers/spawn.js +152 -0
- package/dist/providers/types.js +17 -0
- package/dist/schemas/index.js +374 -0
- package/dist/skills/.gitkeep +0 -0
- package/dist/skills/code-review/judge.md +23 -0
- package/dist/skills/code-review/reviewer.md +38 -0
- package/dist/skills/idea-refinement/analyst.md +45 -0
- package/dist/skills/idea-refinement/planner.md +25 -0
- package/dist/storage/feedback.js +111 -0
- package/dist/storage/paths.js +20 -0
- package/dist/storage/replay.js +0 -0
- package/dist/storage/runs-read.js +95 -0
- package/dist/storage/runs.js +129 -0
- package/dist/storage/sessions.js +71 -0
- package/dist/tui/app.js +444 -0
- package/dist/tui/format.js +27 -0
- package/dist/tui/index.js +8 -0
- package/dist/tui/smart-entry.js +106 -0
- package/dist/tui/timeline.js +91 -0
- package/dist/workflows/code-review.js +76 -0
- package/dist/workflows/idea-refinement.js +105 -0
- package/package.json +64 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- **Contextual intent preflight** — idea-refinement now starts with an S0 run brief: the analyst generates
|
|
7
|
+
3-4 context-specific questions, the TUI asks them before the main council work, and the answers are
|
|
8
|
+
persisted in `00b-run-brief.json` and included in downstream prompts.
|
|
9
|
+
- **Idea report v3** — idea-refinement reports now emit an explicit BLUF recommendation
|
|
10
|
+
(`PROCEED`, `PROCEED_WITH_CONDITIONS`, `PIVOT`, `STOP`), conditions when needed, a best-effort
|
|
11
|
+
12-dimension scorecard, assumption audit table in HTML, deterministic debate narrative, anchored
|
|
12
|
+
validation plan with kill signals (`09b-action-plan.json`), open questions, red-team note, and a
|
|
13
|
+
call/provider receipt. The Markdown copy button includes the expanded brief.
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
- Idea-refinement run estimate is now ~12 provider calls / ~4 Claude-Opus calls because S0 writes the
|
|
17
|
+
intent preflight and the judge seat also writes the validation plan. The default budget is now 13 so
|
|
18
|
+
a normal run still has room for one repair without skipping the validation plan.
|
|
19
|
+
|
|
20
|
+
## 0.2.0 — 2026-07-06 — v2 product round
|
|
21
|
+
|
|
22
|
+
### Added
|
|
23
|
+
- **Council View + HTML export** — `aiki show <run> --html [--open]` renders a plain-language decision
|
|
24
|
+
brief (verdict, risks that held up, blind spots, recommended next steps; raw per-model analysis in a
|
|
25
|
+
collapsible section). Renderer-only; artifacts/schemas unchanged.
|
|
26
|
+
- **Slash-command home screen** — the TUI opens on a command home: `/idea <text>`, `/review [--branch]`,
|
|
27
|
+
`/resume <id>`, `/sessions`, `/models`, `/config`, `/help`. Plain (non-slash) text still routes to the
|
|
28
|
+
idea flow, or is redirected if it's a general question / code paste. It's a fixed parser, not chat.
|
|
29
|
+
- **Run from anywhere** — hybrid storage: runs live in the project's `.aiki/` when inside a git repo, else
|
|
30
|
+
in `~/.aiki/`. `$AIKI_HOME` overrides the global home.
|
|
31
|
+
- **Sessions + resume** — global registry (`~/.aiki/sessions.jsonl`); `aiki sessions` lists runs across
|
|
32
|
+
locations; `aiki resume <id>` (or `/resume`) continues a killed/timed-out run by **replaying** every
|
|
33
|
+
completed provider call from disk, so only the failed stage onward spends a real call.
|
|
34
|
+
- **Per-provider models** — `aiki models` (Gemini enumerates via `agy models`; Claude/Codex take any id).
|
|
35
|
+
Pin a model per provider in `.aiki/config.json` or global `~/.aiki/config.json`. No hardcoded versions.
|
|
36
|
+
- **Richer intent clarify** — the misunderstanding guard now offers pick-one / combine-all / type-your-own,
|
|
37
|
+
and merges same-meaning readings more reliably (stopword-stripped content overlap; threshold unchanged).
|
|
38
|
+
- **Run-cost preview** — `aiki run` prints a call estimate and confirms before spending (skip with `--yes`
|
|
39
|
+
or in a non-interactive shell).
|
|
40
|
+
|
|
41
|
+
### Changed
|
|
42
|
+
- Per-call timeout 180→300s and wall-clock deadline 10→20min (a real Opus judge exceeded both).
|
|
43
|
+
- Config is now layered: global `~/.aiki/config.json` (base) + project `.aiki/config.json` (override);
|
|
44
|
+
`aiki config` shows the merged effective config.
|
|
45
|
+
|
|
46
|
+
### Safety (unchanged)
|
|
47
|
+
- Read-only orchestration, no credential handling, no API keys, no write/exec tools, no chat. aiki writes
|
|
48
|
+
only under `.aiki/` / `~/.aiki/`.
|
|
49
|
+
|
|
50
|
+
## 0.1.0 — v1
|
|
51
|
+
|
|
52
|
+
- Local multi-model orchestration binding installed AI CLIs into schema-validated workflows
|
|
53
|
+
(idea-refinement + code-review) with a pre-registered benchmark harness. Thesis proven (RESULTS.md):
|
|
54
|
+
cross-provider structured review caught every planted bug where the best single model missed ~1 in 4, at
|
|
55
|
+
equal precision, on a 10-case held-out set.
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gaurav Palaspagar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="docs/One.png" alt="aiki — a local model council" width="820">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">aiki</h1>
|
|
6
|
+
|
|
7
|
+
<p align="center"><em>A local <strong>model council</strong> for code review and idea stress-testing — driven by the AI CLIs you already have.</em></p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-blue.svg">
|
|
11
|
+
<img alt="Node ≥ 20" src="https://img.shields.io/badge/node-%E2%89%A5%2020-brightgreen.svg">
|
|
12
|
+
<img alt="Local-first, no API keys" src="https://img.shields.io/badge/local--first-no%20API%20keys-informational.svg">
|
|
13
|
+
<img alt="Read-only orchestration" src="https://img.shields.io/badge/orchestration-read--only-success.svg">
|
|
14
|
+
<img alt="Tests" src="https://img.shields.io/badge/tests-261%20passing-success.svg">
|
|
15
|
+
</p>
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
**aiki** runs the AI coding CLIs you already have installed and logged in (Claude Code, Codex, Antigravity/Gemini)
|
|
20
|
+
as a **panel that can genuinely disagree** — they review independently, cross-examine each other, a judge
|
|
21
|
+
adjudicates the disputes, and you get a clear decision brief.
|
|
22
|
+
|
|
23
|
+
It does two jobs, well:
|
|
24
|
+
|
|
25
|
+
- **Code review** — catch the bugs a single model misses.
|
|
26
|
+
- **Idea stress-testing** — pressure-test a plan before you build it.
|
|
27
|
+
|
|
28
|
+
aiki is **not** a general assistant. Trivia and chat are routed away, not answered — a council adds cost, not
|
|
29
|
+
accuracy, when there's one right answer.
|
|
30
|
+
|
|
31
|
+
**Jump to:** [Why](#why) · [Benchmark](#benchmark) · [Requirements](#requirements) · [Install](#install) · [Quickstart](#quickstart) · [The two workflows](#the-two-workflows) · [Example](#example-a-real-idea-run) · [Configuration](#configuration) · [Sessions & resume](#sessions--resume) · [Safety](#safety-model) · [Costs & limits](#costs--limits) · [How it works](#how-it-works)
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## Why
|
|
36
|
+
|
|
37
|
+
On a code review or an "is this idea sound?" call, a single model has one blind spot. Two or three *different*
|
|
38
|
+
models — each analyzing independently, then cross-examining — catch what any one misses. aiki orchestrates
|
|
39
|
+
that locally: **no API keys, no new subscriptions**. It uses the CLIs and logins you already pay for, so you
|
|
40
|
+
stop copy-pasting between them by hand.
|
|
41
|
+
|
|
42
|
+
<p align="center">
|
|
43
|
+
<img src="docs/Three.png" alt="One model's field of view lets bugs slip past; three overlapping fields catch them all" width="820">
|
|
44
|
+
</p>
|
|
45
|
+
|
|
46
|
+
## Benchmark
|
|
47
|
+
|
|
48
|
+
On a **pre-registered, 10-case held-out** code-review benchmark (frozen before the run so it couldn't be
|
|
49
|
+
tuned post-hoc — see [BENCHMARK.md](BENCHMARK.md) and [RESULTS.md](RESULTS.md)):
|
|
50
|
+
|
|
51
|
+
> **The cross-provider council caught _every_ planted bug where the best single model missed ~1 in 4 — at
|
|
52
|
+
> equal precision, zero false positives.**
|
|
53
|
+
|
|
54
|
+
| Arm | What it is | Seeded-bug recall | Precision | Provider calls |
|
|
55
|
+
|---|---|---|---|---|
|
|
56
|
+
| **B** | best single model — structured, self-adversarial review | 77% (33/43) | 1.00 | 10 |
|
|
57
|
+
| **D** | **cross-provider council** — Claude + Codex review, Gemini judges | **100% (43/43)** | **1.00** | 44 |
|
|
58
|
+
|
|
59
|
+
**→ 1.30× the recall at identical precision** (0 false positives across 59 adjudicated unmatched findings).
|
|
60
|
+
|
|
61
|
+
<details>
|
|
62
|
+
<summary><strong>Per-case results (every case, no cherry-picking)</strong></summary>
|
|
63
|
+
|
|
64
|
+
| Case | Seeded bugs | B (single) | D (council) |
|
|
65
|
+
|---|---|---|---|
|
|
66
|
+
| 01 payments | 4 | 4/4 | 4/4 |
|
|
67
|
+
| 02 inventory | 4 | 4/4 | 4/4 |
|
|
68
|
+
| 03 comments | 4 | **1/4** | 4/4 |
|
|
69
|
+
| 04 search | 5 | **2/5** | 5/5 |
|
|
70
|
+
| 05 notifications | 4 | **2/4** | 4/4 |
|
|
71
|
+
| 06 profile | 4 | 4/4 | 4/4 |
|
|
72
|
+
| 07 dashboard | 5 | **3/5** | 5/5 |
|
|
73
|
+
| 08 upload | 4 | 4/4 | 4/4 |
|
|
74
|
+
| 09 sessions | 4 | 4/4 | 4/4 |
|
|
75
|
+
| 10 analytics | 5 | 5/5 | 5/5 |
|
|
76
|
+
| **Total** | **43** | **33/43 (77%)** | **43/43 (100%)** |
|
|
77
|
+
|
|
78
|
+
The single model's misses cluster on 4 cases (03/04/05/07); the council caught all of them.
|
|
79
|
+
</details>
|
|
80
|
+
|
|
81
|
+
**Reproduce it yourself:** `aiki bench code-review --arms B,D --set holdout --yes`
|
|
82
|
+
|
|
83
|
+
**Honest caveats (in full).** The win is a **recall** win — precision was non-discriminating on this bug-dense
|
|
84
|
+
set. It is **not** a claim of beating cheap self-consistency (that comparison is deferred, not evaluated).
|
|
85
|
+
n = 10 cases, single run per arm — directional, not a p-value. Full method and every number in
|
|
86
|
+
[RESULTS.md §7](RESULTS.md).
|
|
87
|
+
|
|
88
|
+
## Requirements
|
|
89
|
+
|
|
90
|
+
> ⚠️ **aiki drives your existing CLIs — it does not ship or host any model.** You must have the provider CLIs
|
|
91
|
+
> **installed and already logged in.** aiki never sees, stores, or transmits your credentials.
|
|
92
|
+
|
|
93
|
+
- **Node ≥ 20.** (Node 16/18 will crash at startup — this is a hard requirement.)
|
|
94
|
+
- **macOS or Linux** (WSL2 works).
|
|
95
|
+
- The provider CLIs on your `PATH`, **each already authenticated**:
|
|
96
|
+
| CLI | Command | Shown in aiki as |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| Claude Code | `claude` | Claude |
|
|
99
|
+
| Codex | `codex` | Codex |
|
|
100
|
+
| Antigravity | `agy` | Gemini |
|
|
101
|
+
- **At least 2 of the 3** must be ready (a council needs a panel). Check anytime with `aiki doctor`.
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
aiki doctor # lists each provider: version, ready/not, read-only mode
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Install
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
npm install -g aiki-cli
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
From source:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
git clone https://github.com/kratos619/Aiki.git
|
|
117
|
+
cd Aiki
|
|
118
|
+
npm install
|
|
119
|
+
npm run build
|
|
120
|
+
npm link # puts `aiki` on your PATH (or run directly: node dist/cli/index.js)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Quickstart
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
aiki # opens the interactive home screen
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Type a command, or just describe an idea and press Enter:
|
|
130
|
+
|
|
131
|
+
| Command | What it does |
|
|
132
|
+
|---|---|
|
|
133
|
+
| `/idea <text>` | stress-test an idea with the council |
|
|
134
|
+
| `/review [--branch]` | review your working-tree changes (or this branch vs its base) |
|
|
135
|
+
| `/resume <id>` | continue a killed/timed-out run — replays finished work, only redoes the rest |
|
|
136
|
+
| `/sessions` | list past runs (newest first) |
|
|
137
|
+
| `/models` | show / choose the model each provider uses |
|
|
138
|
+
| `/config` | show the effective config |
|
|
139
|
+
| `/help` | the command list |
|
|
140
|
+
|
|
141
|
+
The command palette filters as you type (`/mo` → `/models`); **Tab** completes, **↑/↓** pick, **Enter** runs.
|
|
142
|
+
Plain text is never charged silently — you get a confirm step before any run spends model calls.
|
|
143
|
+
|
|
144
|
+
**Headless (scriptable):**
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
aiki run idea-refinement "a fridge-photo-to-recipe app for busy parents"
|
|
148
|
+
aiki run idea-refinement ./idea.md
|
|
149
|
+
aiki run code-review --base main # review this branch vs main
|
|
150
|
+
aiki run code-review --diff ./changes.patch # review a patch file
|
|
151
|
+
aiki run code-review --cheap # Gemini+Codex review, Claude judges only disputes (~⅓ the Opus)
|
|
152
|
+
aiki show <run-id> --html --open # open the shareable decision brief in your browser
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
An idea run **auto-opens** its report in your browser when it finishes.
|
|
156
|
+
|
|
157
|
+
## The two workflows
|
|
158
|
+
|
|
159
|
+
<p align="center">
|
|
160
|
+
<img src="docs/Four.png" alt="Three reviewers → cross-examination → a judge → a decision brief" width="820">
|
|
161
|
+
</p>
|
|
162
|
+
|
|
163
|
+
**Code review** — parallel blind review → deterministic file:line validation (every finding must point at a
|
|
164
|
+
real line in the diff) → mutual adversarial cross-examination → consensus/dispute map → the judge adjudicates
|
|
165
|
+
only the disputes → report.
|
|
166
|
+
|
|
167
|
+
**Idea refinement** — a contextual preflight (a few sharp questions to pin down what you actually mean) →
|
|
168
|
+
intent contract → misunderstanding guard → parallel adversarial analysis → disagreement map → verifier →
|
|
169
|
+
judge → a validation planner. The report is a **decision brief**, not an essay:
|
|
170
|
+
|
|
171
|
+
- a **BLUF recommendation** — `PROCEED` / `PROCEED WITH CONDITIONS` / `PIVOT` / `STOP`
|
|
172
|
+
- the **chairman's reasoning** (what decided it, where the models split, whose side the judge took)
|
|
173
|
+
- a **dimension scorecard** (which of 12 angles were examined, contested, or missed)
|
|
174
|
+
- an **assumption audit** (held / failed / unverified, with confidence)
|
|
175
|
+
- **the debate** (who argued what, who won)
|
|
176
|
+
- an **anchored validation plan** — concrete next actions, each with an effort estimate and a *kill signal*
|
|
177
|
+
- a **cost receipt** (calls per provider)
|
|
178
|
+
|
|
179
|
+
## Example: a real idea run
|
|
180
|
+
|
|
181
|
+
<p align="center">
|
|
182
|
+
<img src="docs/Two.png" alt="Three models deliberate and cross-examine; the judge reaches a verdict" width="760">
|
|
183
|
+
</p>
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
aiki run idea-refinement "Users keep asking for more features, so we should add a plugin marketplace to boost retention."
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
The council's verdict on that one:
|
|
190
|
+
|
|
191
|
+
> **Recommendation: STOP.** The premise (churn is caused by missing features) is unproven; a marketplace adds
|
|
192
|
+
> a security-vs-capability dilemma and a developer cold-start problem, and simpler alternatives dominate
|
|
193
|
+
> (build the top-3 requested features natively, or integrate an existing automation platform).
|
|
194
|
+
|
|
195
|
+
…followed by **7 anchored validation actions**, e.g. *"Pull the last 90 days of churned users and tag each
|
|
196
|
+
with their primary churn reason"* (effort S, kill signal: churn isn't feature-driven). The full brief opens in
|
|
197
|
+
your browser and has a **Copy report (Markdown)** button so you can paste it straight into your coding assistant.
|
|
198
|
+
|
|
199
|
+
## Configuration
|
|
200
|
+
|
|
201
|
+
Each provider runs its own model families — pick one per provider. Nothing is hardcoded, so a new model works
|
|
202
|
+
the day it ships.
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
aiki models # lists what each CLI offers
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Set models, roles, and budget in `.aiki/config.json` (per-project) or `~/.aiki/config.json` (global default):
|
|
209
|
+
|
|
210
|
+
```json
|
|
211
|
+
{
|
|
212
|
+
"models": { "agy": "Gemini 3.1 Pro (High)", "claude": "opus", "codex": "gpt-5-codex" },
|
|
213
|
+
"roles": { "judge": "claude" },
|
|
214
|
+
"budget": 18
|
|
215
|
+
}
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
`roles` pins which provider judges and which review; `budget` is the max provider calls per run (a guard
|
|
219
|
+
against repair storms — most runs use far fewer).
|
|
220
|
+
|
|
221
|
+
## Sessions & resume
|
|
222
|
+
|
|
223
|
+
Every run is recorded in a global registry (`~/.aiki/sessions.jsonl`), so you can find and continue runs from
|
|
224
|
+
anywhere:
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
aiki sessions # all runs, newest first, resumable ones flagged
|
|
228
|
+
aiki resume <session-id> # continue a killed/timed-out run (or /resume in the TUI)
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Resume re-runs the pipeline but **replays** every step that already finished from the saved outputs on disk —
|
|
232
|
+
so only the failed step onward spends a real model call. (A step that failed on bad *content*, not a crash,
|
|
233
|
+
will replay that content; resume is for timeouts, crashes, and Ctrl-C.)
|
|
234
|
+
|
|
235
|
+
## Safety model
|
|
236
|
+
|
|
237
|
+
This is the part that makes aiki trustworthy to point at a real repo:
|
|
238
|
+
|
|
239
|
+
- **Read-only orchestration.** Providers run with their read-only flags (`claude --permission-mode plan`,
|
|
240
|
+
`codex --sandbox read-only`, `agy --sandbox`). aiki never uses `--dangerously-skip-permissions` or any
|
|
241
|
+
edit/exec mode.
|
|
242
|
+
- **No credentials, ever.** aiki never reads credential directories and filters `KEY|TOKEN|SECRET` out of the
|
|
243
|
+
environment before spawning a provider.
|
|
244
|
+
- **Writes only under `.aiki/` / `~/.aiki/`.** Nothing else on your disk is touched.
|
|
245
|
+
- **No API keys, no chat UI, no write/exec tools, no "learned" routing** — by design.
|
|
246
|
+
- Skill playbooks load **only** from the repo and are scanned for exfiltration patterns before use.
|
|
247
|
+
|
|
248
|
+
## Costs & limits
|
|
249
|
+
|
|
250
|
+
- **Runs cost real model calls** against your existing CLI subscriptions/quota. Idea refinement is about
|
|
251
|
+
**12 provider calls** (~4 on Claude/Opus); code review is about **5**. `aiki run` shows an estimate and asks
|
|
252
|
+
to confirm (skip with `--yes`).
|
|
253
|
+
- **Not a general assistant.** Questions and "explore my whole codebase" requests are redirected, not answered
|
|
254
|
+
— aiki reviews a *diff* and vets a *stated idea*.
|
|
255
|
+
- **Analysis, not advice.** Every report is a decision aid. Verify before acting.
|
|
256
|
+
- Every run leaves a full audit trail (each stage's prompt + raw output + intermediate artifacts) under its
|
|
257
|
+
run directory, and every stage's output is schema-validated before the next stage sees it — free-form prose
|
|
258
|
+
never crosses a stage boundary.
|
|
259
|
+
|
|
260
|
+
## Where files live
|
|
261
|
+
|
|
262
|
+
- Inside a git repo → the project's `.aiki/` (runs stay with the project).
|
|
263
|
+
- Anywhere else → `~/.aiki/`. `$AIKI_HOME` overrides the global home.
|
|
264
|
+
- The global session registry always lives in `~/.aiki/`.
|
|
265
|
+
|
|
266
|
+
## How it works
|
|
267
|
+
|
|
268
|
+
Every stage is a small, independently-testable unit with a zod-validated output contract. Model text lives in
|
|
269
|
+
bounded, capped fields slotted into deterministic report structure — so the output is a briefing, not a chat
|
|
270
|
+
transcript. The council's disagreement is the signal: consensus is trusted, disputes are adjudicated, and
|
|
271
|
+
what nobody examined is flagged as a blind spot.
|
|
272
|
+
|
|
273
|
+
## License
|
|
274
|
+
|
|
275
|
+
MIT — see [LICENSE](LICENSE). Analysis, not advice — verify before acting.
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// Benchmark arms A–D for code-review (§17, BENCHMARK.md §1, T11). Each arm is an engine composition:
|
|
2
|
+
// the harness runs it via `executeRun` (→ a full .aiki/runs record) and it returns its final Finding[]
|
|
3
|
+
// for scoring. A/B are single claude calls (the baselines); C is claude sampled 3× + self-consistency
|
|
4
|
+
// synthesis; D is the full cross-provider pipeline. A/B/C use claude (the fixed "single best model").
|
|
5
|
+
import { readFile } from 'node:fs/promises';
|
|
6
|
+
import { resolve } from 'node:path';
|
|
7
|
+
import { CodeReviewRoleOutputModel } from '../schemas/index.js';
|
|
8
|
+
import { jsonCall } from '../orchestration/jsonStage.js';
|
|
9
|
+
import { parseDiffFiles } from '../orchestration/git.js';
|
|
10
|
+
import { countLines, filterValidFindings } from '../orchestration/stages/cr-s4-review.js';
|
|
11
|
+
import { sameFinding } from '../orchestration/stages/cr-map.js';
|
|
12
|
+
import { s9ReviewJudge } from '../orchestration/stages/cr-s9-judge.js';
|
|
13
|
+
import { scoreFindings } from '../orchestration/stages/cr-report.js';
|
|
14
|
+
import { runCodeReview } from '../workflows/code-review.js';
|
|
15
|
+
export const ARM_IDS = ['A', 'B', 'C', 'D', 'E'];
|
|
16
|
+
const SCHEMA_LINE = 'findings: [{id "F1"..., file, line_start, line_end, severity P0|P1|P2|P3, category CORRECTNESS|SECURITY|CONCURRENCY|ERROR_HANDLING|PERF|MAINTAINABILITY, claim, evidence, suggested_fix, self_confidence 0-1}]';
|
|
17
|
+
const A_PROMPT = `Review this code diff and report the bugs you find. The diff is at {{DIFF_PATH}} (repo root = your cwd).
|
|
18
|
+
Output ONLY JSON: {task_echo (≤2 sentences), ${SCHEMA_LINE}}. JSON only.`;
|
|
19
|
+
// Arm B — the real opponent: a single strong structured-adversarial prompt (analyze → self-attack → re-answer).
|
|
20
|
+
const B_PROMPT = `You are a rigorous senior code reviewer. Review the diff at {{DIFF_PATH}} (repo root = your cwd) in three
|
|
21
|
+
internal passes, then output ONLY the final JSON — do not show your working:
|
|
22
|
+
1. ANALYZE: list every plausible defect (correctness, security, concurrency, error handling, performance).
|
|
23
|
+
2. SELF-ATTACK: for each candidate, argue the strongest case that it is a FALSE POSITIVE; discard the ones that don't survive.
|
|
24
|
+
3. RE-ANSWER: report only the surviving, defensible findings, each citing a file:line you verified exists.
|
|
25
|
+
Output ONLY JSON: {task_echo (≤2 sentences), ${SCHEMA_LINE}}. severity P0 = correctness/security/data-loss. JSON only.`;
|
|
26
|
+
/** Persist the diff + filled prompt, and return the reviewer-facing artifacts a single call needs. */
|
|
27
|
+
async function setup(ctx, diff, template) {
|
|
28
|
+
await ctx.writer.writeInput('diff.patch', diff);
|
|
29
|
+
const files = parseDiffFiles(diff);
|
|
30
|
+
const prompt = template.replace('{{DIFF_PATH}}', resolve(ctx.writer.dir, 'inputs', 'diff.patch'));
|
|
31
|
+
await ctx.writer.writePrompt('reviewer.md', prompt);
|
|
32
|
+
return { prompt, files, diffSet: new Set(files), lineCounts: await countLines(ctx.cwd, files) };
|
|
33
|
+
}
|
|
34
|
+
/** A/B — one claude call, file:line-validated. `label` names the stage + role-output artifact. */
|
|
35
|
+
async function singleCallArm(ctx, diff, template, label) {
|
|
36
|
+
const { prompt, diffSet, lineCounts } = await setup(ctx, diff, template);
|
|
37
|
+
const model = await jsonCall(ctx, ctx.handle('claude'), label, prompt, CodeReviewRoleOutputModel);
|
|
38
|
+
await ctx.writer.writeRoleOutput('claude', { workflow: 'code-review', ...model });
|
|
39
|
+
return filterValidFindings(model.findings, diffSet, lineCounts).valid;
|
|
40
|
+
}
|
|
41
|
+
export const armA = (ctx, diff) => singleCallArm(ctx, diff, A_PROMPT, 'A');
|
|
42
|
+
export const armB = (ctx, diff) => singleCallArm(ctx, diff, B_PROMPT, 'B');
|
|
43
|
+
/** Cluster findings across samples by the §487 matcher. Support = how many samples raised it. */
|
|
44
|
+
function mergeSamples(samples) {
|
|
45
|
+
const clusters = [];
|
|
46
|
+
const rank = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
47
|
+
for (const sample of samples) {
|
|
48
|
+
const claimed = new Set();
|
|
49
|
+
for (const f of sample) {
|
|
50
|
+
const c = clusters.find((cl) => !claimed.has(cl) && sameFinding(cl.rep, f));
|
|
51
|
+
if (c) {
|
|
52
|
+
claimed.add(c);
|
|
53
|
+
c.support += 1;
|
|
54
|
+
if (rank[f.severity] < rank[c.rep.severity])
|
|
55
|
+
c.rep = f; // keep highest severity
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
const nc = { rep: f, support: 1 };
|
|
59
|
+
clusters.push(nc);
|
|
60
|
+
claimed.add(nc);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// ≥2 samples → self-consistent consensus; exactly 1 → disputed (the judge decides keep/drop).
|
|
65
|
+
const claude = ['claude'];
|
|
66
|
+
const consensus = clusters.filter((c) => c.support >= 2).map((c) => ({ finding: c.rep, reviewers: [...claude], cross_verdict: 'NONE' }));
|
|
67
|
+
const disputed = clusters.filter((c) => c.support === 1).map((c) => ({ finding: c.rep, reviewers: [...claude], cross_verdict: 'REFUTE', refutation: 'appeared in only 1 of 3 samples' }));
|
|
68
|
+
let n = 0;
|
|
69
|
+
const reindex = (arr) => arr.map((a) => ({ ...a, finding: { ...a.finding, id: `G${++n}` } }));
|
|
70
|
+
return {
|
|
71
|
+
consensus: reindex(consensus),
|
|
72
|
+
disputed: reindex(disputed),
|
|
73
|
+
single_reviewer: [],
|
|
74
|
+
per_reviewer: [{ provider: 'claude', raised: samples.reduce((s, x) => s + x.length, 0), kept: clusters.length, dropped: 0 }],
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
/** Arm C — claude sampled 3× (sample-keyed self-consistency) + the code-review judge on singletons. */
|
|
78
|
+
export const armC = async (ctx, diff) => {
|
|
79
|
+
const { prompt, diffSet, lineCounts } = await setup(ctx, diff, B_PROMPT);
|
|
80
|
+
const samples = [];
|
|
81
|
+
for (let i = 0; i < 3; i++) {
|
|
82
|
+
const model = await jsonCall(ctx, ctx.handle('claude'), `C-s${i + 1}`, prompt, CodeReviewRoleOutputModel);
|
|
83
|
+
await ctx.writer.writeRoleOutput(`sample-${i + 1}`, { workflow: 'code-review', ...model });
|
|
84
|
+
samples.push(filterValidFindings(model.findings, diffSet, lineCounts).valid);
|
|
85
|
+
}
|
|
86
|
+
const map = mergeSamples(samples);
|
|
87
|
+
await ctx.writer.writeJson('review-map', map);
|
|
88
|
+
const judge = await s9ReviewJudge(ctx, map); // ctx.roles.judge = claude for arm C (harness sets it)
|
|
89
|
+
return scoreFindings(map, judge).filter((s) => s.disposition === 'kept').map((s) => s.finding);
|
|
90
|
+
};
|
|
91
|
+
/** Arms D and E — the full product pipeline (`runCodeReview`) → read back its kept findings for scoring.
|
|
92
|
+
* SAME pipeline; the harness injects their differing roles: D = claude+codex reviewers + agy judge;
|
|
93
|
+
* E = agy+codex reviewers + claude judge (the Opus-thrift variant — claude fires only on disputes). */
|
|
94
|
+
const productPipeline = async (ctx, diff) => {
|
|
95
|
+
await runCodeReview(ctx, diff);
|
|
96
|
+
const [map, judge] = await Promise.all([
|
|
97
|
+
readFile(resolve(ctx.writer.dir, '07-review-map.json'), 'utf8').then(JSON.parse),
|
|
98
|
+
readFile(resolve(ctx.writer.dir, '09-judge-report.json'), 'utf8').then(JSON.parse),
|
|
99
|
+
]);
|
|
100
|
+
return scoreFindings(map, judge).filter((s) => s.disposition === 'kept').map((s) => s.finding);
|
|
101
|
+
};
|
|
102
|
+
export const armD = productPipeline;
|
|
103
|
+
export const armE = productPipeline;
|
|
104
|
+
export const ARMS = { A: armA, B: armB, C: armC, D: armD, E: armE };
|