@agentsdance/codejury 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +246 -0
- package/bin/jury.js +1152 -0
- package/jury.config.example.json +43 -0
- package/lib/agents.js +295 -0
- package/lib/config.js +157 -0
- package/lib/findings.js +112 -0
- package/lib/loop.js +400 -0
- package/lib/prompt.js +67 -0
- package/lib/reply.js +134 -0
- package/lib/repository.js +185 -0
- package/lib/server.js +224 -0
- package/lib/store.js +316 -0
- package/lib/style.js +90 -0
- package/lib/triage.js +144 -0
- package/package.json +44 -0
- package/prompts/feedback.md +54 -0
- package/prompts/review-round.md +52 -0
- package/web/index.html +1699 -0
package/lib/store.js
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
// Where a run lives on disk.
|
|
2
|
+
//
|
|
3
|
+
// runs/<slug>/events.ndjson append-only; two writers are safe (the CLI appends
|
|
4
|
+
// agent events, the main agent appends verdicts)
|
|
5
|
+
// runs/<slug>/run.json the folded view the console reads
|
|
6
|
+
import { mkdir, readFile, writeFile, appendFile, readdir, open } from "node:fs/promises";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Where a run lives.
|
|
11
|
+
*
|
|
12
|
+
* `attempt` separates one invocation from another. Without it every
|
|
13
|
+
* `jury agent <same-pr>` appended to the same directory, so three separate
|
|
14
|
+
* reviews merged into rounds 1-5 of a single run — and a killed run's open
|
|
15
|
+
* rounds interleaved with the next one's. Rounds within ONE invocation still
|
|
16
|
+
* belong together; two invocations do not.
|
|
17
|
+
*/
|
|
18
|
+
export function slugFor(target) {
|
|
19
|
+
const repo = (target.repo ?? "repo").replace(/[^\w.-]+/g, "-");
|
|
20
|
+
const id = String(target.id ?? "0").replace(/[^\w.-]+/g, "");
|
|
21
|
+
const base = `${repo}-${id}`;
|
|
22
|
+
const attempt = target.attempt ? String(target.attempt).replace(/[^\w.-]+/g, "") : "";
|
|
23
|
+
return attempt ? `${base}-${attempt}` : base;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* A short, sortable stamp identifying one invocation: YYYYMMDD-HHMM.
|
|
28
|
+
* Sortable so the directory listing reads chronologically, and minute
|
|
29
|
+
* granularity because two runs of the same PR in the same minute are the same
|
|
30
|
+
* mistake as running it twice by accident.
|
|
31
|
+
*/
|
|
32
|
+
export function attemptStamp(now = new Date()) {
|
|
33
|
+
const p = (n) => String(n).padStart(2, "0");
|
|
34
|
+
return `${now.getFullYear()}${p(now.getMonth() + 1)}${p(now.getDate())}`
|
|
35
|
+
+ `-${p(now.getHours())}${p(now.getMinutes())}`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function runsDir(cwd = process.cwd()) {
|
|
39
|
+
return path.join(cwd, "runs");
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function appendEvent(dir, event) {
|
|
43
|
+
await mkdir(dir, { recursive: true });
|
|
44
|
+
const line = JSON.stringify({ ts: new Date().toISOString(), ...event });
|
|
45
|
+
await appendFile(path.join(dir, "events.ndjson"), line + "\n", "utf8");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function readEvents(dir) {
|
|
49
|
+
try {
|
|
50
|
+
const raw = await readFile(path.join(dir, "events.ndjson"), "utf8");
|
|
51
|
+
return raw
|
|
52
|
+
.split("\n")
|
|
53
|
+
.filter(Boolean)
|
|
54
|
+
.map((l) => {
|
|
55
|
+
try {
|
|
56
|
+
return JSON.parse(l);
|
|
57
|
+
} catch {
|
|
58
|
+
return null; // a torn line must not blank the run
|
|
59
|
+
}
|
|
60
|
+
})
|
|
61
|
+
.filter(Boolean);
|
|
62
|
+
} catch (err) {
|
|
63
|
+
if (err.code === "ENOENT") return [];
|
|
64
|
+
throw err;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Verbatim text kept beside the event log — the prompt a reviewer was given and
|
|
70
|
+
* the raw stream it produced. These are the evidence that the review happened
|
|
71
|
+
* and are often far too large for the event log, so they live in their own files.
|
|
72
|
+
*/
|
|
73
|
+
export async function writeArtifact(dir, name, text) {
|
|
74
|
+
await mkdir(dir, { recursive: true });
|
|
75
|
+
const safe = name.replace(/[^\w.-]+/g, "_");
|
|
76
|
+
await writeFile(path.join(dir, safe), text ?? "", "utf8");
|
|
77
|
+
return safe;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* An artifact opened for incremental writing, so a reviewer's stream reaches
|
|
82
|
+
* disk while it is still talking rather than only when it exits. Writes are
|
|
83
|
+
* chained through one promise: fs handles do not queue concurrent writes, and
|
|
84
|
+
* chunks arriving faster than the disk can take them would otherwise interleave.
|
|
85
|
+
*/
|
|
86
|
+
export async function openArtifact(dir, name) {
|
|
87
|
+
await mkdir(dir, { recursive: true });
|
|
88
|
+
const safe = path.basename(name);
|
|
89
|
+
const fh = await open(path.join(dir, safe), "w");
|
|
90
|
+
let chain = Promise.resolve();
|
|
91
|
+
let closed = false;
|
|
92
|
+
return {
|
|
93
|
+
name: safe,
|
|
94
|
+
write(text) {
|
|
95
|
+
if (closed) return chain;
|
|
96
|
+
// Swallowed on purpose: a failed tail-write must not kill a review that
|
|
97
|
+
// is otherwise fine, and r.raw still holds the authoritative copy.
|
|
98
|
+
chain = chain.then(() => fh.write(text)).catch(() => {});
|
|
99
|
+
return chain;
|
|
100
|
+
},
|
|
101
|
+
async close() {
|
|
102
|
+
if (closed) return;
|
|
103
|
+
closed = true;
|
|
104
|
+
await chain;
|
|
105
|
+
await fh.close().catch(() => {});
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export async function readArtifact(dir, name) {
|
|
111
|
+
// Never let a caller-supplied name walk out of the run directory.
|
|
112
|
+
const safe = path.basename(name);
|
|
113
|
+
return readFile(path.join(dir, safe), "utf8");
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export async function writeRun(dir, run) {
|
|
117
|
+
await mkdir(dir, { recursive: true });
|
|
118
|
+
await writeFile(path.join(dir, "run.json"), JSON.stringify(run, null, 2) + "\n", "utf8");
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export async function readRun(dir) {
|
|
122
|
+
const raw = await readFile(path.join(dir, "run.json"), "utf8");
|
|
123
|
+
return JSON.parse(raw);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Every run under runs/. A malformed one is skipped and reported, never fatal —
|
|
128
|
+
* one bad file must not blank the whole console.
|
|
129
|
+
*/
|
|
130
|
+
export async function listRuns(base) {
|
|
131
|
+
let entries;
|
|
132
|
+
try {
|
|
133
|
+
entries = await readdir(base, { withFileTypes: true });
|
|
134
|
+
} catch (err) {
|
|
135
|
+
if (err.code === "ENOENT") return { runs: [], skipped: [] };
|
|
136
|
+
throw err;
|
|
137
|
+
}
|
|
138
|
+
const runs = [];
|
|
139
|
+
const skipped = [];
|
|
140
|
+
for (const e of entries) {
|
|
141
|
+
if (!e.isDirectory()) continue;
|
|
142
|
+
try {
|
|
143
|
+
// The directory name is how the console asks for this run's artifacts.
|
|
144
|
+
runs.push({ slug: e.name, ...(await readRun(path.join(base, e.name))) });
|
|
145
|
+
} catch (err) {
|
|
146
|
+
skipped.push({ dir: e.name, reason: err.code === "ENOENT" ? "no run.json" : err.message });
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
const rank = { human: 0, review: 1, queued: 2, converged: 3, merged: 4 };
|
|
150
|
+
// Newest first within a state. The id used to be the only tiebreak, which
|
|
151
|
+
// ordered runs alphabetically: an old review of #3 outranked a fresh one of
|
|
152
|
+
// #5, and the console's bare fallback (runs[0]) opened the stale one.
|
|
153
|
+
// `attempt` is the run's own stamp; runs made before it existed have none and
|
|
154
|
+
// sort last among their peers rather than jumping the queue. The id stays on
|
|
155
|
+
// as the final tiebreak so equal-attempt runs keep a stable order instead of
|
|
156
|
+
// falling back to whatever readdir returned.
|
|
157
|
+
const when = (r) => String(r.target?.attempt ?? "");
|
|
158
|
+
runs.sort((a, b) => {
|
|
159
|
+
const ra = rank[a.target?.state] ?? 9;
|
|
160
|
+
const rb = rank[b.target?.state] ?? 9;
|
|
161
|
+
return ra - rb
|
|
162
|
+
|| when(b).localeCompare(when(a))
|
|
163
|
+
|| String(a.target?.id).localeCompare(String(b.target?.id));
|
|
164
|
+
});
|
|
165
|
+
return { runs, skipped };
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Fold the event log into the shape the console renders. */
|
|
169
|
+
export function foldEvents(events, seed = {}) {
|
|
170
|
+
const run = {
|
|
171
|
+
target: seed.target ?? {},
|
|
172
|
+
rounds: [],
|
|
173
|
+
exchanges: [],
|
|
174
|
+
replies: [],
|
|
175
|
+
lanes: [],
|
|
176
|
+
marks: [],
|
|
177
|
+
ship: [],
|
|
178
|
+
tlfoot: "",
|
|
179
|
+
foot: "",
|
|
180
|
+
...seed,
|
|
181
|
+
};
|
|
182
|
+
const findings = new Map();
|
|
183
|
+
const lanes = new Map(); // agent -> lane, kept in launch order
|
|
184
|
+
let t0 = null;
|
|
185
|
+
let lastMin = 0;
|
|
186
|
+
|
|
187
|
+
const laneFor = (agent) => {
|
|
188
|
+
let lane = lanes.get(agent);
|
|
189
|
+
if (!lane) lanes.set(agent, (lane = { who: agent, label: agent, segs: [] }));
|
|
190
|
+
return lane;
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
for (const e of events) {
|
|
194
|
+
const at = e.ts ? Date.parse(e.ts) : null;
|
|
195
|
+
if (at && t0 === null) t0 = at;
|
|
196
|
+
const min = at && t0 !== null ? +((at - t0) / 60000).toFixed(1) : 0;
|
|
197
|
+
if (min > lastMin) lastMin = min;
|
|
198
|
+
|
|
199
|
+
switch (e.t) {
|
|
200
|
+
case "round.start":
|
|
201
|
+
run.rounds.push({
|
|
202
|
+
n: e.n, sha: e.sha ?? "", verdicts: [], startMin: min,
|
|
203
|
+
// Wall-clock, not just the offset: the console sorts runs by recency
|
|
204
|
+
// and a round that has not reported yet still happened at a time.
|
|
205
|
+
startedAt: e.ts ?? "",
|
|
206
|
+
// The exact text the reviewers were given, so a verdict can be read
|
|
207
|
+
// against the question that produced it.
|
|
208
|
+
prompt: e.prompt ?? "", promptFile: e.promptFile ?? "",
|
|
209
|
+
});
|
|
210
|
+
run.marks.push({ at: min, l: `round ${e.n}` });
|
|
211
|
+
break;
|
|
212
|
+
case "agent.launch":
|
|
213
|
+
laneFor(e.agent).segs.push({ r: e.round, s: min });
|
|
214
|
+
break;
|
|
215
|
+
case "agent.report": {
|
|
216
|
+
const r = run.rounds.find((x) => x.n === e.round);
|
|
217
|
+
r?.verdicts.push({
|
|
218
|
+
a: e.agent, s: e.verdict, t: e.summary ?? e.verdict, seconds: e.seconds,
|
|
219
|
+
// Full text, not just the first line: a clean verdict still carries
|
|
220
|
+
// what the reviewer checked and what it could not check.
|
|
221
|
+
report: e.report ?? "", rawFile: e.rawFile ?? "", rawBytes: e.rawBytes ?? 0,
|
|
222
|
+
at: e.ts ?? "", atMin: min,
|
|
223
|
+
});
|
|
224
|
+
// Closes the span its own launch opened, so two rounds stay two spans.
|
|
225
|
+
const seg = lanes.get(e.agent)?.segs.find((s) => s.r === e.round && s.e === undefined);
|
|
226
|
+
if (seg) seg.e = min;
|
|
227
|
+
break;
|
|
228
|
+
}
|
|
229
|
+
case "finding.raised":
|
|
230
|
+
findings.set(e.id, {
|
|
231
|
+
id: e.id, round: e.round, with: e.agent, claim: e.claim,
|
|
232
|
+
loc: e.loc ?? "", res: "open", turns: [],
|
|
233
|
+
});
|
|
234
|
+
break;
|
|
235
|
+
case "finding.reproduced": {
|
|
236
|
+
const f = findings.get(e.id);
|
|
237
|
+
if (f) f.reproduced = e.evidence ?? true;
|
|
238
|
+
break;
|
|
239
|
+
}
|
|
240
|
+
case "finding.resolved": {
|
|
241
|
+
const f = findings.get(e.id);
|
|
242
|
+
if (f) { f.res = e.verdict; f.outcome = e.reason ?? ""; }
|
|
243
|
+
break;
|
|
244
|
+
}
|
|
245
|
+
// A reply is a turn in one reviewer's conversation, not a round of its
|
|
246
|
+
// own: it concerns only that reviewer's findings, so it hangs off them
|
|
247
|
+
// rather than appearing as a fourth round nobody ran.
|
|
248
|
+
case "reply.sent":
|
|
249
|
+
for (const f of findings.values()) {
|
|
250
|
+
if (f.with === e.agent && f.res !== "open") {
|
|
251
|
+
f.turns.push({ who: "claude", kind: "reply", resumed: e.resumed, at: min });
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
break;
|
|
255
|
+
case "reply.answered": {
|
|
256
|
+
const lane = laneFor(e.agent);
|
|
257
|
+
lane.segs.push({ r: "reply", s: min, e: min, t: `reply · ${e.seconds}s` });
|
|
258
|
+
for (const f of findings.values()) {
|
|
259
|
+
if (f.with === e.agent && f.res !== "open") {
|
|
260
|
+
f.turns.push({ who: e.agent, kind: "answer", seconds: e.seconds, at: min });
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
run.replies = [
|
|
264
|
+
...(run.replies ?? []),
|
|
265
|
+
{ agent: e.agent, verdict: e.verdict, seconds: e.seconds, report: e.report ?? "" },
|
|
266
|
+
];
|
|
267
|
+
break;
|
|
268
|
+
}
|
|
269
|
+
case "target":
|
|
270
|
+
run.target = { ...run.target, ...e.target };
|
|
271
|
+
break;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// A launch with no report — the agent was killed, or the round is still
|
|
276
|
+
// running. Only the LAST round can still be running: once a later round has
|
|
277
|
+
// started, an unfinished launch from an earlier one is abandoned, not live.
|
|
278
|
+
// Running those to the newest event drew a span across every round that
|
|
279
|
+
// followed — a killed reviewer showed as "963.7 min · unfinished" stretched
|
|
280
|
+
// over the whole timeline, burying the rounds that actually happened.
|
|
281
|
+
const lastRound = Math.max(0, ...run.rounds.map((r) => r.n));
|
|
282
|
+
// Where each round ended, so an abandoned span stops at its own round rather
|
|
283
|
+
// than at the present moment.
|
|
284
|
+
const endOf = new Map();
|
|
285
|
+
for (const r of run.rounds) {
|
|
286
|
+
const done = r.verdicts.map((v) => v.atMin).filter((m) => typeof m === "number");
|
|
287
|
+
if (done.length) endOf.set(r.n, Math.max(...done));
|
|
288
|
+
}
|
|
289
|
+
for (const lane of lanes.values()) {
|
|
290
|
+
for (const s of lane.segs) {
|
|
291
|
+
if (s.e !== undefined) continue;
|
|
292
|
+
const live = s.r === lastRound || s.r === "reply";
|
|
293
|
+
if (live) {
|
|
294
|
+
s.e = Math.max(s.s, lastMin);
|
|
295
|
+
s.open = true;
|
|
296
|
+
s.t = `round ${s.r} · ${+(s.e - s.s).toFixed(1)} min · still running`;
|
|
297
|
+
} else {
|
|
298
|
+
// Abandoned: end it where its own round ended, or at its start if
|
|
299
|
+
// nothing in that round ever reported.
|
|
300
|
+
s.e = Math.max(s.s, endOf.get(s.r) ?? s.s);
|
|
301
|
+
s.abandoned = true;
|
|
302
|
+
s.t = `round ${s.r} · never finished`;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
run.lanes = [...lanes.values()];
|
|
308
|
+
run.exchanges = [...findings.values()];
|
|
309
|
+
// Spans count too: one ending past the last mark would overflow the timeline.
|
|
310
|
+
run.totalMin = Math.max(
|
|
311
|
+
1,
|
|
312
|
+
...run.marks.map((m) => m.at),
|
|
313
|
+
...run.lanes.flatMap((l) => l.segs.map((s) => s.e)),
|
|
314
|
+
);
|
|
315
|
+
return run;
|
|
316
|
+
}
|
package/lib/style.js
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// Colour and shape for the CLI's own output.
|
|
2
|
+
//
|
|
3
|
+
// A round prints for pages, and every line used to arrive at the same weight:
|
|
4
|
+
// a round boundary, a reviewer's raw report and a triage verdict were
|
|
5
|
+
// indistinguishable while scrolling. Colour is used only where it carries
|
|
6
|
+
// meaning — whose turn it is, and how a finding was settled — and never as
|
|
7
|
+
// decoration.
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Colour is off unless stdout is a terminal that wants it.
|
|
11
|
+
*
|
|
12
|
+
* `NO_COLOR` is honoured for any value at all, per no-color.org: the variable
|
|
13
|
+
* existing is the signal. A pipe or a file gets plain text, so `jury … > log`
|
|
14
|
+
* and `| grep` stay readable.
|
|
15
|
+
*/
|
|
16
|
+
const ON = process.env.NO_COLOR === undefined
|
|
17
|
+
&& process.env.TERM !== "dumb"
|
|
18
|
+
&& process.stdout.isTTY === true;
|
|
19
|
+
|
|
20
|
+
const ESC = "\x1b[";
|
|
21
|
+
const wrap = (open, close) => (s) => (ON ? `${ESC}${open}m${s}${ESC}${close}m` : String(s));
|
|
22
|
+
|
|
23
|
+
export const bold = wrap(1, 22);
|
|
24
|
+
export const dim = wrap(2, 22);
|
|
25
|
+
export const under = wrap(4, 24);
|
|
26
|
+
|
|
27
|
+
// 256-colour, so the palette matches the console's hues rather than whatever
|
|
28
|
+
// the terminal theme maps the 8 basic colours to.
|
|
29
|
+
const fg = (n) => wrap(`38;5;${n}`, 39);
|
|
30
|
+
|
|
31
|
+
export const ok = fg(71); // green — accepted, converged
|
|
32
|
+
export const bad = fg(167); // red — rejected, failed to run
|
|
33
|
+
export const warn = fg(179); // amber — deferred, still open
|
|
34
|
+
export const info = fg(74); // blue — headings, structural
|
|
35
|
+
export const muted = fg(245); // grey — timings, paths, detail
|
|
36
|
+
|
|
37
|
+
/** The console assigns each agent a hue; the CLI uses the same ones. */
|
|
38
|
+
const AGENT_HUE = {
|
|
39
|
+
claude: 176, codex: 250, grok: 66, agy: 74, droid: 209,
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/** Stable per name, so an unconfigured agent still reads as itself. */
|
|
43
|
+
export function agent(name) {
|
|
44
|
+
if (!ON) return String(name);
|
|
45
|
+
let n = AGENT_HUE[name];
|
|
46
|
+
if (n === undefined) {
|
|
47
|
+
let h = 0;
|
|
48
|
+
for (let i = 0; i < name.length; i++) h = (h * 31 + name.charCodeAt(i)) | 0;
|
|
49
|
+
// 22..219 avoids the near-black and near-white ends of the cube, both of
|
|
50
|
+
// which vanish into one terminal background or the other.
|
|
51
|
+
n = 22 + (Math.abs(h) % 198);
|
|
52
|
+
}
|
|
53
|
+
return fg(n)(name);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const VERDICT_COLOUR = { accepted: ok, rejected: bad, deferred: warn, superseded: muted };
|
|
57
|
+
|
|
58
|
+
/** accepted / rejected / deferred in the three colours the web view uses. */
|
|
59
|
+
export function verdict(v) {
|
|
60
|
+
return (VERDICT_COLOUR[v] ?? muted)(String(v).toUpperCase());
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* "1 finding" / "2 findings", rather than the "(s)" placeholder that leaked
|
|
65
|
+
* into the output and stayed there.
|
|
66
|
+
*/
|
|
67
|
+
export function count(n, one, many = `${one}s`) {
|
|
68
|
+
return `${n} ${n === 1 ? one : many}`;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** A labelled line in the run header: dim key, plain value. */
|
|
72
|
+
export function field(key, value) {
|
|
73
|
+
return `${dim(key.padEnd(8))} ${value}`;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* A rule that says what follows it. Reviewer reports are dumped whole, and
|
|
78
|
+
* without a marked boundary one runs straight into the triage below it.
|
|
79
|
+
*/
|
|
80
|
+
export function rule(label, width = 64) {
|
|
81
|
+
const text = label ? ` ${label} ` : "";
|
|
82
|
+
const dashes = Math.max(0, width - text.length);
|
|
83
|
+
const left = Math.min(3, dashes);
|
|
84
|
+
return dim("─".repeat(left)) + bold(text) + dim("─".repeat(dashes - left));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Indent every line of a block, so a pasted report is visibly not our voice. */
|
|
88
|
+
export function indent(text, prefix = " ") {
|
|
89
|
+
return String(text).replace(/^/gm, prefix);
|
|
90
|
+
}
|
package/lib/triage.js
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
// Triage: the judgement half of the loop.
|
|
2
|
+
//
|
|
3
|
+
// DESIGN.md left this to the operator on purpose — "deciding a finding
|
|
4
|
+
// reproduces is the calling agent's job, and roughly a third of suggestions do
|
|
5
|
+
// not survive that step". With nobody in that seat the loop reviewed the same
|
|
6
|
+
// commit ten times and called it convergence.
|
|
7
|
+
//
|
|
8
|
+
// The main agent fills the seat now. It is spawned exactly like a reviewer, but
|
|
9
|
+
// it is the only one allowed to touch the tree: it reproduces a claim before
|
|
10
|
+
// accepting it, writes the regression test, and answers the ones it rejects
|
|
11
|
+
// with the evidence that settled them.
|
|
12
|
+
import { runAgent } from "./agents.js";
|
|
13
|
+
|
|
14
|
+
/** What the main agent must answer for one finding. */
|
|
15
|
+
export const VERDICT_SCHEMA = `{
|
|
16
|
+
"reproduced": "<what you observed, or null if it does not reproduce>",
|
|
17
|
+
"verdict": "accepted | rejected | deferred",
|
|
18
|
+
"reason": "<why, in one or two sentences>",
|
|
19
|
+
"test": "<the regression test you added and watched fail without the fix, or null>"
|
|
20
|
+
}`;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The prompt for one finding.
|
|
24
|
+
*
|
|
25
|
+
* Deliberately adversarial about acceptance: a reviewer's claim is a hypothesis,
|
|
26
|
+
* and the whole value of the gate is that a third of them do not survive being
|
|
27
|
+
* checked. Being asked to demonstrate it first is what makes "accepted" mean
|
|
28
|
+
* something.
|
|
29
|
+
*/
|
|
30
|
+
export function triagePrompt({ finding, trunk, stopToken }) {
|
|
31
|
+
return `A code reviewer raised this finding against the current worktree.
|
|
32
|
+
|
|
33
|
+
FINDING: ${finding.claim}
|
|
34
|
+
${finding.loc ? `WHERE: ${finding.loc}` : ""}
|
|
35
|
+
|
|
36
|
+
${finding.body || "(no further detail given)"}
|
|
37
|
+
|
|
38
|
+
Your job is to decide whether it is real, and to act on it.
|
|
39
|
+
|
|
40
|
+
1. Read the code at that location. Run \`git diff $(git merge-base HEAD origin/${trunk}) HEAD\`
|
|
41
|
+
if you need to see what changed.
|
|
42
|
+
2. Try to REPRODUCE it — a failing test, an observed wrong value, a traced path.
|
|
43
|
+
A claim you cannot demonstrate is not accepted, however plausible it sounds.
|
|
44
|
+
Roughly a third of review findings do not survive this step; rejecting one
|
|
45
|
+
with evidence is as valuable as fixing one.
|
|
46
|
+
3. If it reproduces: fix it, and add a regression test. Run the test with the
|
|
47
|
+
fix reverted and confirm it FAILS — a test that passes either way is
|
|
48
|
+
decoration. Then run the full suite.
|
|
49
|
+
4. If it does not: say what you checked and why the claim is wrong.
|
|
50
|
+
5. Deferring means you did NOT fix it. Use it only when the problem is
|
|
51
|
+
pre-existing and not made materially more likely by this change, and leave
|
|
52
|
+
the code alone. If you fixed it, the verdict is "accepted" — a fix plus a
|
|
53
|
+
proven test recorded as "deferred" tells the reviewer its finding was
|
|
54
|
+
waved through, and leaves the loop believing there is still work to do.
|
|
55
|
+
|
|
56
|
+
Do not fix anything the finding did not raise. Do not commit — the loop commits.
|
|
57
|
+
|
|
58
|
+
Answer with ONLY a JSON object on the last line of your output, no fence:
|
|
59
|
+
|
|
60
|
+
${VERDICT_SCHEMA}
|
|
61
|
+
|
|
62
|
+
Choosing the verdict:
|
|
63
|
+
|
|
64
|
+
accepted you demonstrated it AND fixed it AND have a test that failed
|
|
65
|
+
without the fix. Requires non-null "reproduced" and "test" — the
|
|
66
|
+
CLI refuses an acceptance carrying neither.
|
|
67
|
+
rejected you checked and the claim is wrong. Say what you checked.
|
|
68
|
+
deferred real, pre-existing, out of scope, and you changed nothing.
|
|
69
|
+
|
|
70
|
+
If you edited a file, the verdict is "accepted" or you should revert the edit.`;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Parse the verdict out of the main agent's output.
|
|
75
|
+
*
|
|
76
|
+
* Last JSON object wins: agents narrate before answering, and an example echoed
|
|
77
|
+
* from the prompt earlier in the transcript must not beat the real answer at
|
|
78
|
+
* the end.
|
|
79
|
+
*/
|
|
80
|
+
export function parseVerdict(text) {
|
|
81
|
+
const s = String(text ?? "");
|
|
82
|
+
let found = null;
|
|
83
|
+
// Scan for balanced top-level objects rather than regex-matching braces,
|
|
84
|
+
// which breaks on any nested object in the reasoning above it.
|
|
85
|
+
for (let i = 0; i < s.length; i++) {
|
|
86
|
+
if (s[i] !== "{") continue;
|
|
87
|
+
let depth = 0, inStr = false, esc = false;
|
|
88
|
+
for (let j = i; j < s.length; j++) {
|
|
89
|
+
const c = s[j];
|
|
90
|
+
if (esc) { esc = false; continue; }
|
|
91
|
+
if (c === "\\") { esc = true; continue; }
|
|
92
|
+
if (c === '"') { inStr = !inStr; continue; }
|
|
93
|
+
if (inStr) continue;
|
|
94
|
+
if (c === "{") depth++;
|
|
95
|
+
else if (c === "}" && --depth === 0) {
|
|
96
|
+
try {
|
|
97
|
+
const o = JSON.parse(s.slice(i, j + 1));
|
|
98
|
+
if (o && typeof o === "object" && "verdict" in o) found = o;
|
|
99
|
+
} catch { /* not the object we want */ }
|
|
100
|
+
i = j;
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (!found) return null;
|
|
106
|
+
const verdict = String(found.verdict ?? "").toLowerCase().trim();
|
|
107
|
+
const clean = (v) => {
|
|
108
|
+
const t = typeof v === "string" ? v.trim() : v;
|
|
109
|
+
// Agents write "null" and "none" as strings when they mean nothing.
|
|
110
|
+
return !t || /^(null|none|n\/a)$/i.test(String(t)) ? null : String(t);
|
|
111
|
+
};
|
|
112
|
+
return {
|
|
113
|
+
verdict: ["accepted", "rejected", "deferred", "superseded"].includes(verdict) ? verdict : null,
|
|
114
|
+
reproduced: clean(found.reproduced),
|
|
115
|
+
reason: clean(found.reason) ?? "",
|
|
116
|
+
test: clean(found.test),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Put one finding to the main agent and return its verdict.
|
|
122
|
+
*
|
|
123
|
+
* Never throws: a main agent that dies on one finding must not lose the round's
|
|
124
|
+
* other verdicts, so a failure is a null verdict and the finding stays open for
|
|
125
|
+
* the next round to raise again.
|
|
126
|
+
*/
|
|
127
|
+
export async function triageOne(main, finding, { worktree, trunk, stopToken, dryRun, onLog, onChunk }) {
|
|
128
|
+
const prompt = triagePrompt({ finding, trunk, stopToken });
|
|
129
|
+
if (dryRun) {
|
|
130
|
+
return { verdict: "rejected", reproduced: null, reason: "dry run — not triaged", test: null, seconds: 0 };
|
|
131
|
+
}
|
|
132
|
+
const r = await runAgent(main, {
|
|
133
|
+
worktree, prompt, stopToken, onLog, onChunk,
|
|
134
|
+
// Triage is slower than review: it reads, reproduces, edits and runs tests.
|
|
135
|
+
// Floor as well as multiple: a misconfigured 0 must not mean 'kill it now'.
|
|
136
|
+
timeoutSeconds: Math.max(900, (main.expectSeconds || 900) * 3),
|
|
137
|
+
});
|
|
138
|
+
if (!r.ok) return { verdict: null, failed: true, report: r.report, seconds: r.seconds };
|
|
139
|
+
const v = parseVerdict(r.report) ?? parseVerdict(r.raw);
|
|
140
|
+
if (!v?.verdict) {
|
|
141
|
+
return { verdict: null, unparsed: true, report: r.report, seconds: r.seconds };
|
|
142
|
+
}
|
|
143
|
+
return { ...v, report: r.report, seconds: r.seconds };
|
|
144
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@agentsdance/codejury",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Independent AI reviewers that iterate until your pull request is clean.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"code-review",
|
|
7
|
+
"ai",
|
|
8
|
+
"agents",
|
|
9
|
+
"pull-request",
|
|
10
|
+
"claude",
|
|
11
|
+
"codex"
|
|
12
|
+
],
|
|
13
|
+
"homepage": "https://github.com/agentsdance/codejury#readme",
|
|
14
|
+
"bugs": "https://github.com/agentsdance/codejury/issues",
|
|
15
|
+
"repository": {
|
|
16
|
+
"type": "git",
|
|
17
|
+
"url": "git+https://github.com/agentsdance/codejury.git"
|
|
18
|
+
},
|
|
19
|
+
"type": "module",
|
|
20
|
+
"bin": {
|
|
21
|
+
"jury": "bin/jury.js",
|
|
22
|
+
"codejury": "bin/jury.js",
|
|
23
|
+
"cr": "bin/jury.js"
|
|
24
|
+
},
|
|
25
|
+
"engines": {
|
|
26
|
+
"node": ">=20"
|
|
27
|
+
},
|
|
28
|
+
"files": [
|
|
29
|
+
"bin",
|
|
30
|
+
"lib",
|
|
31
|
+
"web/index.html",
|
|
32
|
+
"prompts",
|
|
33
|
+
"jury.config.example.json"
|
|
34
|
+
],
|
|
35
|
+
"publishConfig": {
|
|
36
|
+
"access": "public"
|
|
37
|
+
},
|
|
38
|
+
"scripts": {
|
|
39
|
+
"prepublishOnly": "npm test",
|
|
40
|
+
"start": "node bin/jury.js web --open",
|
|
41
|
+
"test": "node --test \"test/*.test.js\""
|
|
42
|
+
},
|
|
43
|
+
"license": "MIT"
|
|
44
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# Feedback prompt template
|
|
2
|
+
|
|
3
|
+
Sent back to an agent after acting on its review. Purpose: close the loop on disagreements explicitly,
|
|
4
|
+
rather than silently ignoring findings you chose not to act on.
|
|
5
|
+
|
|
6
|
+
Delivery differs per agent:
|
|
7
|
+
|
|
8
|
+
- **codex** — `codex exec resume --last "$(cat feedback.md)"`, prior context retained.
|
|
9
|
+
- **droid** — no session id in `exec` text output; re-run fresh and quote its prior findings into the
|
|
10
|
+
prompt so it knows what it said.
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
Thanks — I acted on your review. Here is what I did with each finding, including where I disagree. The
|
|
15
|
+
final state is at {{worktree path}} (detached at {{sha}}). Please verify my claims and tell me where you
|
|
16
|
+
still disagree.
|
|
17
|
+
|
|
18
|
+
**{{N}}. {{finding title}} — ACCEPTED, fixed.**
|
|
19
|
+
|
|
20
|
+
{{What changed, with the code if short. If the regression test needed a non-obvious shape, say why —
|
|
21
|
+
e.g. "a single run only catches this 50% of the time because the select is random, so the test repeats
|
|
22
|
+
20 runs".}}
|
|
23
|
+
|
|
24
|
+
**{{N}}. {{finding title}} — AGREE it is real, NOT fixing here.**
|
|
25
|
+
|
|
26
|
+
{{Why it is out of scope: untouched by this change, pre-existing, needs its own tests.}}
|
|
27
|
+
|
|
28
|
+
Question: do you agree it is (a) pre-existing and (b) not made materially more likely by this change?
|
|
29
|
+
If you think otherwise, say so and I will reconsider.
|
|
30
|
+
|
|
31
|
+
**{{N}}. {{finding title}} — REJECTED.**
|
|
32
|
+
|
|
33
|
+
{{The evidence. Prefer something reproducible: a command, an error message, a counter-example.}}
|
|
34
|
+
|
|
35
|
+
**{{N}}. {{finding title}} — PARTIAL pushback.**
|
|
36
|
+
|
|
37
|
+
{{Which half you accept and which you dispute, and what you changed as a result.}}
|
|
38
|
+
|
|
39
|
+
Question: do you accept that framing, or is a different definition actually better?
|
|
40
|
+
|
|
41
|
+
Please review the final state and answer: any remaining correctness problem, and do you accept my
|
|
42
|
+
positions on {{list}}? Do not edit files.
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Notes
|
|
47
|
+
|
|
48
|
+
- Ending each disputed item with a **direct question** is what makes this a conversation. In the
|
|
49
|
+
reference run it converted two "you should fix this" items into explicit agreement to defer.
|
|
50
|
+
- State rejections with evidence, not authority. One rejection was settled by writing a three-line
|
|
51
|
+
throwaway test and pasting the panic message.
|
|
52
|
+
- Expect to be wrong sometimes. The most valuable single finding of the reference run — a real
|
|
53
|
+
cancellation bug — came back in a reply to feedback, not in the first review.
|
|
54
|
+
- Tell an agent when its finding was caused by *your* prompt (e.g. a wrong diff base). It is your bug.
|