aiki-cli 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +55 -0
  2. package/LICENSE +21 -0
  3. package/README.md +275 -0
  4. package/dist/bench/arms.js +104 -0
  5. package/dist/bench/harness.js +251 -0
  6. package/dist/bench/results.js +70 -0
  7. package/dist/bench/scoring/seeded-bugs.js +31 -0
  8. package/dist/cli/bench.js +50 -0
  9. package/dist/cli/config.js +51 -0
  10. package/dist/cli/doctor.js +115 -0
  11. package/dist/cli/index.js +129 -0
  12. package/dist/cli/models.js +51 -0
  13. package/dist/cli/providers.js +31 -0
  14. package/dist/cli/resolve.js +159 -0
  15. package/dist/cli/resume.js +94 -0
  16. package/dist/cli/run.js +155 -0
  17. package/dist/cli/sessions.js +35 -0
  18. package/dist/cli/show.js +73 -0
  19. package/dist/config/config.js +102 -0
  20. package/dist/config/smoke-cache.js +65 -0
  21. package/dist/council/open.js +26 -0
  22. package/dist/council/view.js +873 -0
  23. package/dist/orchestration/cluster.js +83 -0
  24. package/dist/orchestration/context.js +277 -0
  25. package/dist/orchestration/engine.js +92 -0
  26. package/dist/orchestration/git.js +133 -0
  27. package/dist/orchestration/jsonStage.js +32 -0
  28. package/dist/orchestration/skills.js +39 -0
  29. package/dist/orchestration/stages/cr-ladder.js +63 -0
  30. package/dist/orchestration/stages/cr-map.js +62 -0
  31. package/dist/orchestration/stages/cr-report.js +83 -0
  32. package/dist/orchestration/stages/cr-s4-review.js +69 -0
  33. package/dist/orchestration/stages/cr-s8-crossexam.js +104 -0
  34. package/dist/orchestration/stages/cr-s9-judge.js +89 -0
  35. package/dist/orchestration/stages/s0-grill.js +79 -0
  36. package/dist/orchestration/stages/s1-intent.js +25 -0
  37. package/dist/orchestration/stages/s10-render.js +198 -0
  38. package/dist/orchestration/stages/s2-misread.js +76 -0
  39. package/dist/orchestration/stages/s3-prompts.js +55 -0
  40. package/dist/orchestration/stages/s4-analyze.js +50 -0
  41. package/dist/orchestration/stages/s5-drift.js +40 -0
  42. package/dist/orchestration/stages/s6-claims.js +56 -0
  43. package/dist/orchestration/stages/s7-disagreement.js +134 -0
  44. package/dist/orchestration/stages/s8-verify.js +56 -0
  45. package/dist/orchestration/stages/s9-judge.js +152 -0
  46. package/dist/orchestration/stages/s9b-plan.js +192 -0
  47. package/dist/providers/adapter-core.js +131 -0
  48. package/dist/providers/adapters.js +9 -0
  49. package/dist/providers/agy.js +29 -0
  50. package/dist/providers/claude.js +56 -0
  51. package/dist/providers/codex.js +35 -0
  52. package/dist/providers/detect.js +21 -0
  53. package/dist/providers/probe.js +43 -0
  54. package/dist/providers/profiles.js +38 -0
  55. package/dist/providers/profiles.json +5 -0
  56. package/dist/providers/smoke.js +26 -0
  57. package/dist/providers/spawn.js +152 -0
  58. package/dist/providers/types.js +17 -0
  59. package/dist/schemas/index.js +374 -0
  60. package/dist/skills/.gitkeep +0 -0
  61. package/dist/skills/code-review/judge.md +23 -0
  62. package/dist/skills/code-review/reviewer.md +38 -0
  63. package/dist/skills/idea-refinement/analyst.md +45 -0
  64. package/dist/skills/idea-refinement/planner.md +25 -0
  65. package/dist/storage/feedback.js +111 -0
  66. package/dist/storage/paths.js +20 -0
  67. package/dist/storage/replay.js +0 -0
  68. package/dist/storage/runs-read.js +95 -0
  69. package/dist/storage/runs.js +129 -0
  70. package/dist/storage/sessions.js +71 -0
  71. package/dist/tui/app.js +444 -0
  72. package/dist/tui/format.js +27 -0
  73. package/dist/tui/index.js +8 -0
  74. package/dist/tui/smart-entry.js +106 -0
  75. package/dist/tui/timeline.js +91 -0
  76. package/dist/workflows/code-review.js +76 -0
  77. package/dist/workflows/idea-refinement.js +105 -0
  78. package/package.json +64 -0
package/CHANGELOG.md ADDED
@@ -0,0 +1,55 @@
1
+ # Changelog
2
+
3
+ ## Unreleased
4
+
5
+ ### Added
6
+ - **Contextual intent preflight** — idea-refinement now starts with an S0 run brief: the analyst generates
7
+ 3-4 context-specific questions, the TUI asks them before the main council work, and the answers are
8
+ persisted in `00b-run-brief.json` and included in downstream prompts.
9
+ - **Idea report v3** — idea-refinement reports now emit an explicit BLUF recommendation
10
+ (`PROCEED`, `PROCEED_WITH_CONDITIONS`, `PIVOT`, `STOP`), conditions when needed, a best-effort
11
+ 12-dimension scorecard, assumption audit table in HTML, deterministic debate narrative, anchored
12
+ validation plan with kill signals (`09b-action-plan.json`), open questions, red-team note, and a
13
+ call/provider receipt. The Markdown copy button includes the expanded brief.
14
+
15
+ ### Changed
16
+ - Idea-refinement run estimate is now ~12 provider calls / ~4 Claude-Opus calls because S0 writes the
17
+ intent preflight and the judge seat also writes the validation plan. The default budget is now 13 so
18
+ a normal run still has room for one repair without skipping the validation plan.
19
+
20
+ ## 0.2.0 — 2026-07-06 — v2 product round
21
+
22
+ ### Added
23
+ - **Council View + HTML export** — `aiki show <run> --html [--open]` renders a plain-language decision
24
+ brief (verdict, risks that held up, blind spots, recommended next steps; raw per-model analysis in a
25
+ collapsible section). Renderer-only; artifacts/schemas unchanged.
26
+ - **Slash-command home screen** — the TUI opens on a command home: `/idea <text>`, `/review [--branch]`,
27
+ `/resume <id>`, `/sessions`, `/models`, `/config`, `/help`. Plain (non-slash) text still routes to the
28
+ idea flow, or is redirected if it's a general question / code paste. It's a fixed parser, not chat.
29
+ - **Run from anywhere** — hybrid storage: runs live in the project's `.aiki/` when inside a git repo, else
30
+ in `~/.aiki/`. `$AIKI_HOME` overrides the global home.
31
+ - **Sessions + resume** — global registry (`~/.aiki/sessions.jsonl`); `aiki sessions` lists runs across
32
+ locations; `aiki resume <id>` (or `/resume`) continues a killed/timed-out run by **replaying** every
33
+ completed provider call from disk, so only the failed stage onward spends a real call.
34
+ - **Per-provider models** — `aiki models` (Gemini enumerates via `agy models`; Claude/Codex take any id).
35
+ Pin a model per provider in `.aiki/config.json` or global `~/.aiki/config.json`. No hardcoded versions.
36
+ - **Richer intent clarify** — the misunderstanding guard now offers pick-one / combine-all / type-your-own,
37
+ and merges same-meaning readings more reliably (stopword-stripped content overlap; threshold unchanged).
38
+ - **Run-cost preview** — `aiki run` prints a call estimate and confirms before spending (skip with `--yes`
39
+ or in a non-interactive shell).
40
+
41
+ ### Changed
42
+ - Per-call timeout 180→300s and wall-clock deadline 10→20min (a real Opus judge exceeded both).
43
+ - Config is now layered: global `~/.aiki/config.json` (base) + project `.aiki/config.json` (override);
44
+ `aiki config` shows the merged effective config.
45
+
46
+ ### Safety (unchanged)
47
+ - Read-only orchestration, no credential handling, no API keys, no write/exec tools, no chat. aiki writes
48
+ only under `.aiki/` / `~/.aiki/`.
49
+
50
+ ## 0.1.0 — v1
51
+
52
+ - Local multi-model orchestration binding installed AI CLIs into schema-validated workflows
53
+ (idea-refinement + code-review) with a pre-registered benchmark harness. Thesis proven (RESULTS.md):
54
+ cross-provider structured review caught every planted bug where the best single model missed ~1 in 4, at
55
+ equal precision, on a 10-case held-out set.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Gaurav Palaspagar
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,275 @@
1
+ <p align="center">
2
+ <img src="docs/One.png" alt="aiki — a local model council" width="820">
3
+ </p>
4
+
5
+ <h1 align="center">aiki</h1>
6
+
7
+ <p align="center"><em>A local <strong>model council</strong> for code review and idea stress-testing — driven by the AI CLIs you already have.</em></p>
8
+
9
+ <p align="center">
10
+ <img alt="License: MIT" src="https://img.shields.io/badge/license-MIT-blue.svg">
11
+ <img alt="Node ≥ 20" src="https://img.shields.io/badge/node-%E2%89%A5%2020-brightgreen.svg">
12
+ <img alt="Local-first, no API keys" src="https://img.shields.io/badge/local--first-no%20API%20keys-informational.svg">
13
+ <img alt="Read-only orchestration" src="https://img.shields.io/badge/orchestration-read--only-success.svg">
14
+ <img alt="Tests" src="https://img.shields.io/badge/tests-261%20passing-success.svg">
15
+ </p>
16
+
17
+ ---
18
+
19
+ **aiki** runs the AI coding CLIs you already have installed and logged in (Claude Code, Codex, Antigravity/Gemini)
20
+ as a **panel that can genuinely disagree** — they review independently, cross-examine each other, a judge
21
+ adjudicates the disputes, and you get a clear decision brief.
22
+
23
+ It does two jobs, well:
24
+
25
+ - **Code review** — catch the bugs a single model misses.
26
+ - **Idea stress-testing** — pressure-test a plan before you build it.
27
+
28
+ aiki is **not** a general assistant. Trivia and chat are routed away, not answered — a council adds cost, not
29
+ accuracy, when there's one right answer.
30
+
31
+ **Jump to:** [Why](#why) · [Benchmark](#benchmark) · [Requirements](#requirements) · [Install](#install) · [Quickstart](#quickstart) · [The two workflows](#the-two-workflows) · [Example](#example-a-real-idea-run) · [Configuration](#configuration) · [Sessions & resume](#sessions--resume) · [Safety](#safety-model) · [Costs & limits](#costs--limits) · [How it works](#how-it-works)
32
+
33
+ ---
34
+
35
+ ## Why
36
+
37
+ On a code review or an "is this idea sound?" call, a single model has one blind spot. Two or three *different*
38
+ models — each analyzing independently, then cross-examining — catch what any one misses. aiki orchestrates
39
+ that locally: **no API keys, no new subscriptions**. It uses the CLIs and logins you already pay for, so you
40
+ stop copy-pasting between them by hand.
41
+
42
+ <p align="center">
43
+ <img src="docs/Three.png" alt="One model's field of view lets bugs slip past; three overlapping fields catch them all" width="820">
44
+ </p>
45
+
46
+ ## Benchmark
47
+
48
+ On a **pre-registered, 10-case held-out** code-review benchmark (frozen before the run so it couldn't be
49
+ tuned post-hoc — see [BENCHMARK.md](BENCHMARK.md) and [RESULTS.md](RESULTS.md)):
50
+
51
+ > **The cross-provider council caught _every_ planted bug where the best single model missed ~1 in 4 — at
52
+ > equal precision, zero false positives.**
53
+
54
+ | Arm | What it is | Seeded-bug recall | Precision | Provider calls |
55
+ |---|---|---|---|---|
56
+ | **B** | best single model — structured, self-adversarial review | 77% (33/43) | 1.00 | 10 |
57
+ | **D** | **cross-provider council** — Claude + Codex review, Gemini judges | **100% (43/43)** | **1.00** | 44 |
58
+
59
+ **→ 1.30× the recall at identical precision** (0 false positives across 59 adjudicated unmatched findings).
60
+
61
+ <details>
62
+ <summary><strong>Per-case results (every case, no cherry-picking)</strong></summary>
63
+
64
+ | Case | Seeded bugs | B (single) | D (council) |
65
+ |---|---|---|---|
66
+ | 01 payments | 4 | 4/4 | 4/4 |
67
+ | 02 inventory | 4 | 4/4 | 4/4 |
68
+ | 03 comments | 4 | **1/4** | 4/4 |
69
+ | 04 search | 5 | **2/5** | 5/5 |
70
+ | 05 notifications | 4 | **2/4** | 4/4 |
71
+ | 06 profile | 4 | 4/4 | 4/4 |
72
+ | 07 dashboard | 5 | **3/5** | 5/5 |
73
+ | 08 upload | 4 | 4/4 | 4/4 |
74
+ | 09 sessions | 4 | 4/4 | 4/4 |
75
+ | 10 analytics | 5 | 5/5 | 5/5 |
76
+ | **Total** | **43** | **33/43 (77%)** | **43/43 (100%)** |
77
+
78
+ The single model's misses cluster on 4 cases (03/04/05/07); the council caught all of them.
79
+ </details>
80
+
81
+ **Reproduce it yourself:** `aiki bench code-review --arms B,D --set holdout --yes`
82
+
83
+ **Honest caveats (in full).** The win is a **recall** win — precision was non-discriminating on this bug-dense
84
+ set. It is **not** a claim of beating cheap self-consistency (that comparison is deferred, not evaluated).
85
+ n = 10 cases, single run per arm — directional, not a p-value. Full method and every number in
86
+ [RESULTS.md §7](RESULTS.md).
87
+
88
+ ## Requirements
89
+
90
+ > ⚠️ **aiki drives your existing CLIs — it does not ship or host any model.** You must have the provider CLIs
91
+ > **installed and already logged in.** aiki never sees, stores, or transmits your credentials.
92
+
93
+ - **Node ≥ 20.** (Node 16/18 will crash at startup — this is a hard requirement.)
94
+ - **macOS or Linux** (WSL2 works).
95
+ - The provider CLIs on your `PATH`, **each already authenticated**:
96
+ | CLI | Command | Shown in aiki as |
97
+ |---|---|---|
98
+ | Claude Code | `claude` | Claude |
99
+ | Codex | `codex` | Codex |
100
+ | Antigravity | `agy` | Gemini |
101
+ - **At least 2 of the 3** must be ready (a council needs a panel). Check anytime with `aiki doctor`.
102
+
103
+ ```bash
104
+ aiki doctor # lists each provider: version, ready/not, read-only mode
105
+ ```
106
+
107
+ ## Install
108
+
109
+ ```bash
110
+ npm install -g aiki-cli
111
+ ```
112
+
113
+ From source:
114
+
115
+ ```bash
116
+ git clone https://github.com/kratos619/Aiki.git
117
+ cd Aiki
118
+ npm install
119
+ npm run build
120
+ npm link # puts `aiki` on your PATH (or run directly: node dist/cli/index.js)
121
+ ```
122
+
123
+ ## Quickstart
124
+
125
+ ```bash
126
+ aiki # opens the interactive home screen
127
+ ```
128
+
129
+ Type a command, or just describe an idea and press Enter:
130
+
131
+ | Command | What it does |
132
+ |---|---|
133
+ | `/idea <text>` | stress-test an idea with the council |
134
+ | `/review [--branch]` | review your working-tree changes (or this branch vs its base) |
135
+ | `/resume <id>` | continue a killed/timed-out run — replays finished work, only redoes the rest |
136
+ | `/sessions` | list past runs (newest first) |
137
+ | `/models` | show / choose the model each provider uses |
138
+ | `/config` | show the effective config |
139
+ | `/help` | the command list |
140
+
141
+ The command palette filters as you type (`/mo` → `/models`); **Tab** completes, **↑/↓** pick, **Enter** runs.
142
+ Plain text is never charged silently — you get a confirm step before any run spends model calls.
143
+
144
+ **Headless (scriptable):**
145
+
146
+ ```bash
147
+ aiki run idea-refinement "a fridge-photo-to-recipe app for busy parents"
148
+ aiki run idea-refinement ./idea.md
149
+ aiki run code-review --base main # review this branch vs main
150
+ aiki run code-review --diff ./changes.patch # review a patch file
151
+ aiki run code-review --cheap # Gemini+Codex review, Claude judges only disputes (~⅓ the Opus)
152
+ aiki show <run-id> --html --open # open the shareable decision brief in your browser
153
+ ```
154
+
155
+ An idea run **auto-opens** its report in your browser when it finishes.
156
+
157
+ ## The two workflows
158
+
159
+ <p align="center">
160
+ <img src="docs/Four.png" alt="Three reviewers → cross-examination → a judge → a decision brief" width="820">
161
+ </p>
162
+
163
+ **Code review** — parallel blind review → deterministic file:line validation (every finding must point at a
164
+ real line in the diff) → mutual adversarial cross-examination → consensus/dispute map → the judge adjudicates
165
+ only the disputes → report.
166
+
167
+ **Idea refinement** — a contextual preflight (a few sharp questions to pin down what you actually mean) →
168
+ intent contract → misunderstanding guard → parallel adversarial analysis → disagreement map → verifier →
169
+ judge → a validation planner. The report is a **decision brief**, not an essay:
170
+
171
+ - a **BLUF recommendation** — `PROCEED` / `PROCEED WITH CONDITIONS` / `PIVOT` / `STOP`
172
+ - the **chairman's reasoning** (what decided it, where the models split, whose side the judge took)
173
+ - a **dimension scorecard** (which of 12 angles were examined, contested, or missed)
174
+ - an **assumption audit** (held / failed / unverified, with confidence)
175
+ - **the debate** (who argued what, who won)
176
+ - an **anchored validation plan** — concrete next actions, each with an effort estimate and a *kill signal*
177
+ - a **cost receipt** (calls per provider)
178
+
179
+ ## Example: a real idea run
180
+
181
+ <p align="center">
182
+ <img src="docs/Two.png" alt="Three models deliberate and cross-examine; the judge reaches a verdict" width="760">
183
+ </p>
184
+
185
+ ```bash
186
+ aiki run idea-refinement "Users keep asking for more features, so we should add a plugin marketplace to boost retention."
187
+ ```
188
+
189
+ The council's verdict on that one:
190
+
191
+ > **Recommendation: STOP.** The premise (churn is caused by missing features) is unproven; a marketplace adds
192
+ > a security-vs-capability dilemma and a developer cold-start problem, and simpler alternatives dominate
193
+ > (build the top-3 requested features natively, or integrate an existing automation platform).
194
+
195
+ …followed by **7 anchored validation actions**, e.g. *"Pull the last 90 days of churned users and tag each
196
+ with their primary churn reason"* (effort S, kill signal: churn isn't feature-driven). The full brief opens in
197
+ your browser and has a **Copy report (Markdown)** button so you can paste it straight into your coding assistant.
198
+
199
+ ## Configuration
200
+
201
+ Each provider runs its own model families — pick one per provider. Nothing is hardcoded, so a new model works
202
+ the day it ships.
203
+
204
+ ```bash
205
+ aiki models # lists what each CLI offers
206
+ ```
207
+
208
+ Set models, roles, and budget in `.aiki/config.json` (per-project) or `~/.aiki/config.json` (global default):
209
+
210
+ ```json
211
+ {
212
+ "models": { "agy": "Gemini 3.1 Pro (High)", "claude": "opus", "codex": "gpt-5-codex" },
213
+ "roles": { "judge": "claude" },
214
+ "budget": 18
215
+ }
216
+ ```
217
+
218
+ `roles` pins which provider judges and which review; `budget` is the max provider calls per run (a guard
219
+ against repair storms — most runs use far fewer).
220
+
221
+ ## Sessions & resume
222
+
223
+ Every run is recorded in a global registry (`~/.aiki/sessions.jsonl`), so you can find and continue runs from
224
+ anywhere:
225
+
226
+ ```bash
227
+ aiki sessions # all runs, newest first, resumable ones flagged
228
+ aiki resume <session-id> # continue a killed/timed-out run (or /resume in the TUI)
229
+ ```
230
+
231
+ Resume re-runs the pipeline but **replays** every step that already finished from the saved outputs on disk —
232
+ so only the failed step onward spends a real model call. (A step that failed on bad *content*, not a crash,
233
+ will replay that content; resume is for timeouts, crashes, and Ctrl-C.)
234
+
235
+ ## Safety model
236
+
237
+ This is the part that makes aiki trustworthy to point at a real repo:
238
+
239
+ - **Read-only orchestration.** Providers run with their read-only flags (`claude --permission-mode plan`,
240
+ `codex --sandbox read-only`, `agy --sandbox`). aiki never uses `--dangerously-skip-permissions` or any
241
+ edit/exec mode.
242
+ - **No credentials, ever.** aiki never reads credential directories and filters `KEY|TOKEN|SECRET` out of the
243
+ environment before spawning a provider.
244
+ - **Writes only under `.aiki/` / `~/.aiki/`.** Nothing else on your disk is touched.
245
+ - **No API keys, no chat UI, no write/exec tools, no "learned" routing** — by design.
246
+ - Skill playbooks load **only** from the repo and are scanned for exfiltration patterns before use.
247
+
248
+ ## Costs & limits
249
+
250
+ - **Runs cost real model calls** against your existing CLI subscriptions/quota. Idea refinement is about
251
+ **12 provider calls** (~4 on Claude/Opus); code review is about **5**. `aiki run` shows an estimate and asks
252
+ to confirm (skip with `--yes`).
253
+ - **Not a general assistant.** Questions and "explore my whole codebase" requests are redirected, not answered
254
+ — aiki reviews a *diff* and vets a *stated idea*.
255
+ - **Analysis, not advice.** Every report is a decision aid. Verify before acting.
256
+ - Every run leaves a full audit trail (each stage's prompt + raw output + intermediate artifacts) under its
257
+ run directory, and every stage's output is schema-validated before the next stage sees it — free-form prose
258
+ never crosses a stage boundary.
259
+
260
+ ## Where files live
261
+
262
+ - Inside a git repo → the project's `.aiki/` (runs stay with the project).
263
+ - Anywhere else → `~/.aiki/`. `$AIKI_HOME` overrides the global home.
264
+ - The global session registry always lives in `~/.aiki/`.
265
+
266
+ ## How it works
267
+
268
+ Every stage is a small, independently-testable unit with a zod-validated output contract. Model text lives in
269
+ bounded, capped fields slotted into deterministic report structure — so the output is a briefing, not a chat
270
+ transcript. The council's disagreement is the signal: consensus is trusted, disputes are adjudicated, and
271
+ what nobody examined is flagged as a blind spot.
272
+
273
+ ## License
274
+
275
+ MIT — see [LICENSE](LICENSE). Analysis, not advice — verify before acting.
@@ -0,0 +1,104 @@
1
+ // Benchmark arms A–D for code-review (§17, BENCHMARK.md §1, T11). Each arm is an engine composition:
2
+ // the harness runs it via `executeRun` (→ a full .aiki/runs record) and it returns its final Finding[]
3
+ // for scoring. A/B are single claude calls (the baselines); C is claude sampled 3× + self-consistency
4
+ // synthesis; D is the full cross-provider pipeline. A/B/C use claude (the fixed "single best model").
5
+ import { readFile } from 'node:fs/promises';
6
+ import { resolve } from 'node:path';
7
+ import { CodeReviewRoleOutputModel } from '../schemas/index.js';
8
+ import { jsonCall } from '../orchestration/jsonStage.js';
9
+ import { parseDiffFiles } from '../orchestration/git.js';
10
+ import { countLines, filterValidFindings } from '../orchestration/stages/cr-s4-review.js';
11
+ import { sameFinding } from '../orchestration/stages/cr-map.js';
12
+ import { s9ReviewJudge } from '../orchestration/stages/cr-s9-judge.js';
13
+ import { scoreFindings } from '../orchestration/stages/cr-report.js';
14
+ import { runCodeReview } from '../workflows/code-review.js';
15
+ export const ARM_IDS = ['A', 'B', 'C', 'D', 'E'];
16
+ const SCHEMA_LINE = 'findings: [{id "F1"..., file, line_start, line_end, severity P0|P1|P2|P3, category CORRECTNESS|SECURITY|CONCURRENCY|ERROR_HANDLING|PERF|MAINTAINABILITY, claim, evidence, suggested_fix, self_confidence 0-1}]';
17
+ const A_PROMPT = `Review this code diff and report the bugs you find. The diff is at {{DIFF_PATH}} (repo root = your cwd).
18
+ Output ONLY JSON: {task_echo (≤2 sentences), ${SCHEMA_LINE}}. JSON only.`;
19
+ // Arm B — the real opponent: a single strong structured-adversarial prompt (analyze → self-attack → re-answer).
20
+ const B_PROMPT = `You are a rigorous senior code reviewer. Review the diff at {{DIFF_PATH}} (repo root = your cwd) in three
21
+ internal passes, then output ONLY the final JSON — do not show your working:
22
+ 1. ANALYZE: list every plausible defect (correctness, security, concurrency, error handling, performance).
23
+ 2. SELF-ATTACK: for each candidate, argue the strongest case that it is a FALSE POSITIVE; discard the ones that don't survive.
24
+ 3. RE-ANSWER: report only the surviving, defensible findings, each citing a file:line you verified exists.
25
+ Output ONLY JSON: {task_echo (≤2 sentences), ${SCHEMA_LINE}}. severity P0 = correctness/security/data-loss. JSON only.`;
26
+ /** Persist the diff + filled prompt, and return the reviewer-facing artifacts a single call needs. */
27
+ async function setup(ctx, diff, template) {
28
+ await ctx.writer.writeInput('diff.patch', diff);
29
+ const files = parseDiffFiles(diff);
30
+ const prompt = template.replace('{{DIFF_PATH}}', resolve(ctx.writer.dir, 'inputs', 'diff.patch'));
31
+ await ctx.writer.writePrompt('reviewer.md', prompt);
32
+ return { prompt, files, diffSet: new Set(files), lineCounts: await countLines(ctx.cwd, files) };
33
+ }
34
+ /** A/B — one claude call, file:line-validated. `label` names the stage + role-output artifact. */
35
+ async function singleCallArm(ctx, diff, template, label) {
36
+ const { prompt, diffSet, lineCounts } = await setup(ctx, diff, template);
37
+ const model = await jsonCall(ctx, ctx.handle('claude'), label, prompt, CodeReviewRoleOutputModel);
38
+ await ctx.writer.writeRoleOutput('claude', { workflow: 'code-review', ...model });
39
+ return filterValidFindings(model.findings, diffSet, lineCounts).valid;
40
+ }
41
+ export const armA = (ctx, diff) => singleCallArm(ctx, diff, A_PROMPT, 'A');
42
+ export const armB = (ctx, diff) => singleCallArm(ctx, diff, B_PROMPT, 'B');
43
+ /** Cluster findings across samples by the §487 matcher. Support = how many samples raised it. */
44
+ function mergeSamples(samples) {
45
+ const clusters = [];
46
+ const rank = { P0: 0, P1: 1, P2: 2, P3: 3 };
47
+ for (const sample of samples) {
48
+ const claimed = new Set();
49
+ for (const f of sample) {
50
+ const c = clusters.find((cl) => !claimed.has(cl) && sameFinding(cl.rep, f));
51
+ if (c) {
52
+ claimed.add(c);
53
+ c.support += 1;
54
+ if (rank[f.severity] < rank[c.rep.severity])
55
+ c.rep = f; // keep highest severity
56
+ }
57
+ else {
58
+ const nc = { rep: f, support: 1 };
59
+ clusters.push(nc);
60
+ claimed.add(nc);
61
+ }
62
+ }
63
+ }
64
+ // ≥2 samples → self-consistent consensus; exactly 1 → disputed (the judge decides keep/drop).
65
+ const claude = ['claude'];
66
+ const consensus = clusters.filter((c) => c.support >= 2).map((c) => ({ finding: c.rep, reviewers: [...claude], cross_verdict: 'NONE' }));
67
+ const disputed = clusters.filter((c) => c.support === 1).map((c) => ({ finding: c.rep, reviewers: [...claude], cross_verdict: 'REFUTE', refutation: 'appeared in only 1 of 3 samples' }));
68
+ let n = 0;
69
+ const reindex = (arr) => arr.map((a) => ({ ...a, finding: { ...a.finding, id: `G${++n}` } }));
70
+ return {
71
+ consensus: reindex(consensus),
72
+ disputed: reindex(disputed),
73
+ single_reviewer: [],
74
+ per_reviewer: [{ provider: 'claude', raised: samples.reduce((s, x) => s + x.length, 0), kept: clusters.length, dropped: 0 }],
75
+ };
76
+ }
77
+ /** Arm C — claude sampled 3× (sample-keyed self-consistency) + the code-review judge on singletons. */
78
+ export const armC = async (ctx, diff) => {
79
+ const { prompt, diffSet, lineCounts } = await setup(ctx, diff, B_PROMPT);
80
+ const samples = [];
81
+ for (let i = 0; i < 3; i++) {
82
+ const model = await jsonCall(ctx, ctx.handle('claude'), `C-s${i + 1}`, prompt, CodeReviewRoleOutputModel);
83
+ await ctx.writer.writeRoleOutput(`sample-${i + 1}`, { workflow: 'code-review', ...model });
84
+ samples.push(filterValidFindings(model.findings, diffSet, lineCounts).valid);
85
+ }
86
+ const map = mergeSamples(samples);
87
+ await ctx.writer.writeJson('review-map', map);
88
+ const judge = await s9ReviewJudge(ctx, map); // ctx.roles.judge = claude for arm C (harness sets it)
89
+ return scoreFindings(map, judge).filter((s) => s.disposition === 'kept').map((s) => s.finding);
90
+ };
91
+ /** Arms D and E — the full product pipeline (`runCodeReview`) → read back its kept findings for scoring.
92
+ * SAME pipeline; the harness injects their differing roles: D = claude+codex reviewers + agy judge;
93
+ * E = agy+codex reviewers + claude judge (the Opus-thrift variant — claude fires only on disputes). */
94
+ const productPipeline = async (ctx, diff) => {
95
+ await runCodeReview(ctx, diff);
96
+ const [map, judge] = await Promise.all([
97
+ readFile(resolve(ctx.writer.dir, '07-review-map.json'), 'utf8').then(JSON.parse),
98
+ readFile(resolve(ctx.writer.dir, '09-judge-report.json'), 'utf8').then(JSON.parse),
99
+ ]);
100
+ return scoreFindings(map, judge).filter((s) => s.disposition === 'kept').map((s) => s.finding);
101
+ };
102
+ export const armD = productPipeline;
103
+ export const armE = productPipeline;
104
+ export const ARMS = { A: armA, B: armB, C: armC, D: armD, E: armE };