@niwa-labs/flow-code 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +26 -2
- package/package.json +16 -4
- package/src/build.ts +162 -0
- package/src/index.ts +13 -0
- package/src/loops.ts +215 -0
- package/src/review.ts +141 -0
- package/src/worktree.ts +177 -0
- package/test/build.test.ts +236 -0
- package/test/code.test.ts +395 -0
- package/test/example.test.ts +86 -0
- package/test/support.ts +187 -0
- package/tsconfig.json +6 -0
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
// The repository helpers in a real script, on a real git repository: a worktree left clean is
|
|
2
|
+
// removed and one with commits is kept; a fork's losers are removed; review and fix stops once the
|
|
3
|
+
// reviewers pass; test first needs a test that fails before it passes; debugging and coverage stop
|
|
4
|
+
// when their command says so.
|
|
5
|
+
|
|
6
|
+
import { afterEach, expect, test } from "bun:test";
|
|
7
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
|
|
10
|
+
import { selfTestVerdict } from "../src/index.ts";
|
|
11
|
+
import { codeHost, commitFile, gitIn, resultOf } from "./support.ts";
|
|
12
|
+
import type { CodeHost, Reply } from "./support.ts";
|
|
13
|
+
|
|
14
|
+
const hosts: CodeHost[] = [];
|
|
15
|
+
afterEach(async () => {
|
|
16
|
+
for (const one of hosts.splice(0)) await one.close();
|
|
17
|
+
}, 30_000);
|
|
18
|
+
|
|
19
|
+
async function host(agents: Record<string, Reply> = {}): Promise<CodeHost> {
|
|
20
|
+
const made = await codeHost(agents);
|
|
21
|
+
hosts.push(made);
|
|
22
|
+
return made;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** The directory an agent call worked in. */
|
|
26
|
+
const dirOf = (call: { cwd?: string }): string => {
|
|
27
|
+
if (call.cwd === undefined) throw new Error("the agent was not started in a directory");
|
|
28
|
+
return call.cwd;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
test("a worktree left clean is removed, and one with a commit is kept with its branch", async () => {
|
|
32
|
+
const flow = await host({ author: (call) => ({ sha: commitFile(dirOf(call), "work.txt", "done\n") }) });
|
|
33
|
+
const ended = await flow.run(`
|
|
34
|
+
import { inWorktree } from "@niwa-labs/flow-code";
|
|
35
|
+
export default workflow({ output: z.json() }, async (): Promise<any> => {
|
|
36
|
+
const clean = await inWorktree("look", { repo: "$REPO" }, async (wt) => wt.path);
|
|
37
|
+
const kept = await inWorktree("work", { repo: "$REPO" }, async (wt) => {
|
|
38
|
+
const done = await ask({ harness: "author", cwd: wt.path, schema: z.object({ sha: z.string() }) }, "Do the work");
|
|
39
|
+
return { path: wt.path, branch: wt.branch, sha: done.sha };
|
|
40
|
+
});
|
|
41
|
+
return { clean, kept };
|
|
42
|
+
});`);
|
|
43
|
+
const out = resultOf(ended) as { clean: string; kept: { path: string; branch: string; sha: string } };
|
|
44
|
+
expect(existsSync(out.clean)).toBe(false);
|
|
45
|
+
expect(readFileSync(join(out.kept.path, "work.txt"), "utf8")).toBe("done\n");
|
|
46
|
+
expect(gitIn(flow.repo, "rev-parse", out.kept.branch)).toBe(out.kept.sha);
|
|
47
|
+
}, 30_000);
|
|
48
|
+
|
|
49
|
+
test("forkSelect keeps the picked fork and removes the others with their commits", async () => {
|
|
50
|
+
const build: Reply = (call) => ({ sha: commitFile(dirOf(call), "fork.txt", `${String(call.harness)}\n`) });
|
|
51
|
+
const flow = await host({ a: build, b: build });
|
|
52
|
+
const ended = await flow.run(`
|
|
53
|
+
import { commitBy, forkSelect } from "@niwa-labs/flow-code";
|
|
54
|
+
export default workflow({ output: z.json() }, async (): Promise<any> => {
|
|
55
|
+
const picked = await forkSelect("race", {
|
|
56
|
+
seats: ["a", "b"],
|
|
57
|
+
repo: "$REPO",
|
|
58
|
+
build: async (seat, wt) => ({ ...(await commitBy(seat, wt, "Build it")), path: wt.path }),
|
|
59
|
+
pick: async () => "b",
|
|
60
|
+
});
|
|
61
|
+
return { winner: picked.winner.id, path: picked.winner.worktree.path, rejected: picked.rejected };
|
|
62
|
+
});`);
|
|
63
|
+
const out = resultOf(ended) as { winner: string; path: string; rejected: string[] };
|
|
64
|
+
expect(out.winner).toBe("b");
|
|
65
|
+
expect(out.rejected).toEqual(["a"]);
|
|
66
|
+
expect(existsSync(out.path)).toBe(true);
|
|
67
|
+
const losers = flow.calls.a?.map(dirOf) ?? [];
|
|
68
|
+
expect(losers).toHaveLength(1);
|
|
69
|
+
expect(existsSync(losers[0] as string)).toBe(false);
|
|
70
|
+
}, 30_000);
|
|
71
|
+
|
|
72
|
+
test("forkSelect refuses a pick that names no build, a name every object has among them", async () => {
|
|
73
|
+
const flow = await host({ a: (call) => ({ sha: commitFile(dirOf(call), "fork.txt", "a\n") }) });
|
|
74
|
+
const ended = await flow.run(`
|
|
75
|
+
import { commitBy, forkSelect } from "@niwa-labs/flow-code";
|
|
76
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
77
|
+
forkSelect("race", { seats: ["a"], repo: "$REPO", build: (seat, wt) => commitBy(seat, wt, "Build it"), pick: async () => "constructor" }).then(
|
|
78
|
+
(picked) => "picked " + typeof picked.winner.value,
|
|
79
|
+
(error: Error) => error.message,
|
|
80
|
+
));`);
|
|
81
|
+
expect(resultOf(ended)).toBe('forkSelect "race": "constructor" is not a build that succeeded');
|
|
82
|
+
}, 30_000);
|
|
83
|
+
|
|
84
|
+
test("forkSelect names why each build failed when none succeeded", async () => {
|
|
85
|
+
const flow = await host({ a: () => ({ sha: "unused" }) });
|
|
86
|
+
const ended = await flow.run(`
|
|
87
|
+
import { forkSelect } from "@niwa-labs/flow-code";
|
|
88
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
89
|
+
forkSelect("race", { seats: ["a"], repo: "$REPO", build: async () => { throw new Error("the build broke"); }, pick: async () => "a" }).then(
|
|
90
|
+
() => "picked",
|
|
91
|
+
(error: Error) => error.message,
|
|
92
|
+
));`);
|
|
93
|
+
expect(resultOf(ended)).toBe('forkSelect "race": no build succeeded (a: the build broke)');
|
|
94
|
+
}, 30_000);
|
|
95
|
+
|
|
96
|
+
test("review and fix sends blocking findings to the author and stops once the reviewers pass", async () => {
|
|
97
|
+
let reviews = 0;
|
|
98
|
+
const flow = await host({
|
|
99
|
+
author: (call) => ({ sha: commitFile(dirOf(call), "fix.txt", "fixed\n") }),
|
|
100
|
+
reviewer: () => {
|
|
101
|
+
reviews += 1;
|
|
102
|
+
return reviews === 1 ? { verdict: "FAIL", findings: [{ title: "no error handling", evidence: "main.ts:3", severity: "blocks" }] } : { verdict: "PASS", findings: [] };
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
const ended = await flow.run(`
|
|
106
|
+
import { inWorktree, reviewFix } from "@niwa-labs/flow-code";
|
|
107
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
108
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
109
|
+
const out = await reviewFix("review", { wt, base: "main", author: "author", reviewers: ["reviewer"] });
|
|
110
|
+
return { sha: out.value.sha, rounds: out.rounds, stopped: out.stopped, history: out.history.map((j) => j.verdict) };
|
|
111
|
+
}));`);
|
|
112
|
+
const out = resultOf(ended) as { sha: string; rounds: number; stopped: string; history: string[] };
|
|
113
|
+
expect(out).toMatchObject({ rounds: 2, stopped: "accepted", history: ["FAIL", "PASS"] });
|
|
114
|
+
expect(flow.calls.author?.[0]?.prompt).toContain("no error handling");
|
|
115
|
+
expect(gitIn(dirOf(flow.calls.author?.[0] ?? {}), "rev-parse", "HEAD")).toBe(out.sha);
|
|
116
|
+
}, 30_000);
|
|
117
|
+
|
|
118
|
+
test("the commit a review round reports is the one the worktree is at, not the one the author names", async () => {
|
|
119
|
+
const flow = await host({
|
|
120
|
+
author: (call) => {
|
|
121
|
+
commitFile(dirOf(call), "fix.txt", "fixed\n");
|
|
122
|
+
// The author names a commit that is not the one it made.
|
|
123
|
+
return { sha: "0".repeat(40) };
|
|
124
|
+
},
|
|
125
|
+
reviewer: (call) => (call.prompt.includes(`to the commit ${"0".repeat(40)}`) ? { verdict: "FAIL", findings: [{ title: "no such commit", evidence: "x", severity: "blocks" }] } : { verdict: "PASS", findings: [] }),
|
|
126
|
+
});
|
|
127
|
+
const ended = await flow.run(`
|
|
128
|
+
import { commitBy, inWorktree, reviewFix } from "@niwa-labs/flow-code";
|
|
129
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
130
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
131
|
+
const built = await commitBy("author", wt, "Fix it");
|
|
132
|
+
const out = await reviewFix("review", { wt, base: "main", author: "author", reviewers: ["reviewer"] });
|
|
133
|
+
return { built, sha: out.value.sha, verdict: out.judged.verdict };
|
|
134
|
+
}));`);
|
|
135
|
+
const out = resultOf(ended) as { built: { sha: string; dirty: boolean }; sha: string; verdict: string };
|
|
136
|
+
const worktree = dirOf(flow.calls.author?.[0] ?? {});
|
|
137
|
+
expect(out.built).toEqual({ sha: gitIn(worktree, "rev-parse", "HEAD"), dirty: false });
|
|
138
|
+
expect(out.built.sha).not.toBe("0".repeat(40));
|
|
139
|
+
expect(out.verdict).toBe("PASS");
|
|
140
|
+
}, 30_000);
|
|
141
|
+
|
|
142
|
+
test("a fix left uncommitted is sent back to the author without being reviewed", async () => {
|
|
143
|
+
let fixes = 0;
|
|
144
|
+
let reviews = 0;
|
|
145
|
+
const flow = await host({
|
|
146
|
+
author: (call) => {
|
|
147
|
+
fixes += 1;
|
|
148
|
+
// The first fix is written and not committed: the review would not see it.
|
|
149
|
+
if (fixes === 1) writeFileSync(join(dirOf(call), "fix.txt"), "fixed\n");
|
|
150
|
+
else commitFile(dirOf(call), "fix.txt", "fixed\n");
|
|
151
|
+
return { summary: "done" };
|
|
152
|
+
},
|
|
153
|
+
reviewer: () => {
|
|
154
|
+
reviews += 1;
|
|
155
|
+
return reviews === 1 ? { verdict: "FAIL", findings: [{ title: "not handled", evidence: "main.ts", severity: "blocks" }] } : { verdict: "PASS", findings: [] };
|
|
156
|
+
},
|
|
157
|
+
});
|
|
158
|
+
const ended = await flow.run(`
|
|
159
|
+
import { inWorktree, reviewFix } from "@niwa-labs/flow-code";
|
|
160
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
161
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
162
|
+
const out = await reviewFix("review", { wt, base: "main", author: "author", reviewers: ["reviewer"] });
|
|
163
|
+
return { history: out.history.map((j) => j.verdict), titles: out.history.map((j) => j.findings.map((f) => f.title)) };
|
|
164
|
+
}));`);
|
|
165
|
+
expect(resultOf(ended)).toEqual({ history: ["FAIL", "FAIL", "PASS"], titles: [["not handled"], ["the work was left uncommitted"], []] });
|
|
166
|
+
expect(reviews).toBe(2);
|
|
167
|
+
}, 30_000);
|
|
168
|
+
|
|
169
|
+
test("a passing check that changes the worktree cannot clear a review round", async () => {
|
|
170
|
+
const flow = await host({ reviewer: () => ({ verdict: "PASS", findings: [] }) });
|
|
171
|
+
const ended = await flow.run(`
|
|
172
|
+
import { inWorktree, reviewFix } from "@niwa-labs/flow-code";
|
|
173
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
174
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
175
|
+
const out = await reviewFix("review", {
|
|
176
|
+
wt, base: "main", author: "reviewer", reviewers: ["reviewer"], max: 1,
|
|
177
|
+
test: { cmd: ["sh", "-c", "touch uncommitted.txt"], cwd: wt.path },
|
|
178
|
+
});
|
|
179
|
+
return { verdict: out.judged.verdict, check: out.value.check?.digest, findings: out.judged.findings };
|
|
180
|
+
}));`);
|
|
181
|
+
expect(resultOf(ended)).toMatchObject({
|
|
182
|
+
verdict: "FAIL",
|
|
183
|
+
check: expect.stringMatching(/^sha256:/),
|
|
184
|
+
findings: [{ title: "the work was left uncommitted" }],
|
|
185
|
+
});
|
|
186
|
+
expect(flow.calls.reviewer).toEqual([]);
|
|
187
|
+
}, 30_000);
|
|
188
|
+
|
|
189
|
+
test("a review panel names the reviewer that could not look, apart from a clean review", async () => {
|
|
190
|
+
const flow = await host({
|
|
191
|
+
author: () => ({ summary: "unused" }),
|
|
192
|
+
careful: () => ({ findings: [{ title: "unchecked input", evidence: "main.ts:9", severity: "blocks" }] }),
|
|
193
|
+
// An answer that is not what was asked for.
|
|
194
|
+
absent: () => ({ findings: "none" }),
|
|
195
|
+
});
|
|
196
|
+
const ended = await flow.run(`
|
|
197
|
+
import { inWorktree, reviewPanel } from "@niwa-labs/flow-code";
|
|
198
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
199
|
+
inWorktree("wt", { repo: "$REPO" }, (wt) => reviewPanel("panel", { wt, base: "main", reviewers: ["careful", "absent"], lenses: ["security", "style"] })));`);
|
|
200
|
+
const out = resultOf(ended) as { findings: Array<{ title: string }>; byLens: Record<string, unknown[]>; failures: Array<{ reviewer: string; lens: string; error: string }> };
|
|
201
|
+
expect(out.findings.map((one) => one.title)).toEqual(["unchecked input"]);
|
|
202
|
+
expect(out.failures.map((one) => `${one.reviewer}/${one.lens}`)).toEqual(["absent/security", "absent/style"]);
|
|
203
|
+
expect(out.failures[0]?.error).not.toBe("");
|
|
204
|
+
}, 30_000);
|
|
205
|
+
|
|
206
|
+
test("test first takes a test that fails before the change, then a change that makes it pass", async () => {
|
|
207
|
+
let reds = 0;
|
|
208
|
+
const flow = await host({
|
|
209
|
+
author: (call) => {
|
|
210
|
+
const dir = dirOf(call);
|
|
211
|
+
if (call.prompt.includes("fails now")) {
|
|
212
|
+
reds += 1;
|
|
213
|
+
// The first test it writes passes already: it is sent back.
|
|
214
|
+
return { sha: commitFile(dir, "check.sh", reds === 1 ? "exit 0\n" : "test -f impl.txt\n") };
|
|
215
|
+
}
|
|
216
|
+
return { sha: commitFile(dir, "impl.txt", "impl\n") };
|
|
217
|
+
},
|
|
218
|
+
});
|
|
219
|
+
const ended = await flow.run(`
|
|
220
|
+
import { inWorktree, tddLoop } from "@niwa-labs/flow-code";
|
|
221
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
222
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
223
|
+
const out = await tddLoop("tdd", { task: "add impl", wt, author: "author", test: { cmd: ["sh", "check.sh"], cwd: wt.path } });
|
|
224
|
+
return { red: out.value.redSha !== "", stopped: out.stopped, history: out.history.map((j) => j.verdict) };
|
|
225
|
+
}));`);
|
|
226
|
+
expect(resultOf(ended)).toEqual({ red: true, stopped: "accepted", history: ["FAIL", "PASS"] });
|
|
227
|
+
expect(reds).toBe(2);
|
|
228
|
+
}, 30_000);
|
|
229
|
+
|
|
230
|
+
test("a red-phase check that changes the worktree cannot establish a failing commit", async () => {
|
|
231
|
+
const flow = await host({ author: (call) => ({ sha: commitFile(dirOf(call), "check.txt", "x\n") }) });
|
|
232
|
+
const ended = await flow.run(`
|
|
233
|
+
import { inWorktree, tddLoop } from "@niwa-labs/flow-code";
|
|
234
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
235
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
236
|
+
const out = await tddLoop("tdd", {
|
|
237
|
+
task: "add a test", wt, author: "author", max: 1,
|
|
238
|
+
test: { cmd: ["sh", "-c", "touch uncommitted.txt; exit 1"], cwd: wt.path },
|
|
239
|
+
});
|
|
240
|
+
return { redSha: out.value.redSha, judged: out.judged };
|
|
241
|
+
}));`);
|
|
242
|
+
expect(resultOf(ended)).toMatchObject({ redSha: "", judged: { verdict: "FAIL", findings: [{ title: "the work was left uncommitted" }] } });
|
|
243
|
+
expect(flow.calls.author).toHaveLength(1);
|
|
244
|
+
}, 30_000);
|
|
245
|
+
|
|
246
|
+
test("debugging needs a failing repro, and fixes until the repro passes", async () => {
|
|
247
|
+
let tries = 0;
|
|
248
|
+
const flow = await host({
|
|
249
|
+
author: (call) => {
|
|
250
|
+
tries += 1;
|
|
251
|
+
const dir = dirOf(call);
|
|
252
|
+
return { sha: commitFile(dir, tries === 1 ? "wrong.txt" : "fixed.txt", "x\n"), cause: tries === 1 ? "a guess" : "the file was missing" };
|
|
253
|
+
},
|
|
254
|
+
});
|
|
255
|
+
const ended = await flow.run(`
|
|
256
|
+
import { debugLoop, inWorktree } from "@niwa-labs/flow-code";
|
|
257
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
258
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
259
|
+
const out = await debugLoop("bug", { symptom: "fixed.txt is missing", wt, author: "author", repro: { cmd: ["test", "-f", "fixed.txt"], cwd: wt.path } });
|
|
260
|
+
const none = await debugLoop("nobug", { symptom: "nothing", wt, author: "author", repro: { cmd: ["true"], cwd: wt.path } });
|
|
261
|
+
return { cause: out.cause, verdict: out.judged.verdict, none: none.judged.verdict, noneSha: none.sha };
|
|
262
|
+
}));`);
|
|
263
|
+
expect(resultOf(ended)).toEqual({ cause: "the file was missing", verdict: "PASS", none: "INCONCLUSIVE", noneSha: "" });
|
|
264
|
+
expect(tries).toBe(2);
|
|
265
|
+
}, 30_000);
|
|
266
|
+
|
|
267
|
+
test("a passing repro that changes the worktree cannot clear a bug fix", async () => {
|
|
268
|
+
const flow = await host({ author: (call) => ({ cause: "missing file", sha: commitFile(dirOf(call), "fixed.txt", "x\n") }) });
|
|
269
|
+
const ended = await flow.run(`
|
|
270
|
+
import { debugLoop, inWorktree } from "@niwa-labs/flow-code";
|
|
271
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
272
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
273
|
+
const out = await debugLoop("bug", {
|
|
274
|
+
wt, symptom: "missing file", author: "author", max: 1,
|
|
275
|
+
repro: { cmd: ["sh", "-c", "test -f fixed.txt && touch uncommitted.txt"], cwd: wt.path },
|
|
276
|
+
});
|
|
277
|
+
return out.judged;
|
|
278
|
+
}));`);
|
|
279
|
+
expect(resultOf(ended)).toMatchObject({ verdict: "FAIL", findings: [{ title: "the work was left uncommitted" }] });
|
|
280
|
+
}, 30_000);
|
|
281
|
+
|
|
282
|
+
test("coverage rises round by round until the measure reports the threshold", async () => {
|
|
283
|
+
const flow = await host({
|
|
284
|
+
author: (call) => {
|
|
285
|
+
const dir = dirOf(call);
|
|
286
|
+
const now = Number(readFileSync(join(dir, "cov.txt"), "utf8"));
|
|
287
|
+
return { sha: commitFile(dir, "cov.txt", String(now + 0.25)) };
|
|
288
|
+
},
|
|
289
|
+
});
|
|
290
|
+
const ended = await flow.run(`
|
|
291
|
+
import { coverageLoop, inWorktree } from "@niwa-labs/flow-code";
|
|
292
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
293
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
294
|
+
await call("proc.run", { cmd: ["sh", "-c", "echo 0.25 > cov.txt"], cwd: wt.path, timeout: 1000 });
|
|
295
|
+
const out = await coverageLoop("cov", { wt, threshold: 0.7, author: "author", measure: { cmd: ["sh", "-c", 'echo "{\\\\"coverage\\\\": $(cat cov.txt)}"'], cwd: wt.path } });
|
|
296
|
+
return { before: out.before, after: out.after, reached: out.reached };
|
|
297
|
+
}));`);
|
|
298
|
+
expect(resultOf(ended)).toEqual({ before: 0.25, after: 0.75, reached: true });
|
|
299
|
+
expect(flow.calls.author).toHaveLength(2);
|
|
300
|
+
}, 30_000);
|
|
301
|
+
|
|
302
|
+
test("a coverage measure that changes a committed round cannot reach the threshold", async () => {
|
|
303
|
+
const flow = await host({ author: (call) => ({ sha: commitFile(dirOf(call), "added.txt", "x\n") }) });
|
|
304
|
+
const ended = await flow.run(`
|
|
305
|
+
import { coverageLoop, inWorktree } from "@niwa-labs/flow-code";
|
|
306
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
307
|
+
inWorktree("wt", { repo: "$REPO" }, (wt) => coverageLoop("cov", {
|
|
308
|
+
wt, threshold: 0.5, author: "author", max: 1,
|
|
309
|
+
measure: {
|
|
310
|
+
cmd: ["sh", "-c", "if test -f added.txt; then touch uncommitted.txt; echo '{\\"coverage\\": 1}'; else echo '{\\"coverage\\": 0}'; fi"],
|
|
311
|
+
cwd: wt.path,
|
|
312
|
+
},
|
|
313
|
+
})));`);
|
|
314
|
+
expect(resultOf(ended)).toMatchObject({ before: 0, after: 0, reached: false, failure: { reason: expect.stringContaining("the work was left uncommitted") } });
|
|
315
|
+
}, 30_000);
|
|
316
|
+
|
|
317
|
+
test("a coverage measure that fails gives no coverage, whatever it printed", async () => {
|
|
318
|
+
const flow = await host({ author: (call) => ({ sha: commitFile(dirOf(call), "t.txt", "x\n") }) });
|
|
319
|
+
const ended = await flow.run(`
|
|
320
|
+
import { coverageLoop, inWorktree } from "@niwa-labs/flow-code";
|
|
321
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
322
|
+
inWorktree("wt", { repo: "$REPO" }, (wt) =>
|
|
323
|
+
coverageLoop("cov", { wt, threshold: 0.5, author: "author", measure: { cmd: ["sh", "-c", "echo '{\\"coverage\\": 1}'; echo tests crashed; exit 1"], cwd: wt.path } }).then(
|
|
324
|
+
() => "measured",
|
|
325
|
+
(error: any) => ({ name: error.name, message: error.message, digest: error.artifact?.digest }),
|
|
326
|
+
)));`);
|
|
327
|
+
const out = resultOf(ended) as { name: string; message: string; digest: string };
|
|
328
|
+
expect(out.name).toBe("MeasureFailed");
|
|
329
|
+
expect(out.message).toContain("tests crashed");
|
|
330
|
+
expect(out.digest).toMatch(/^sha256:/);
|
|
331
|
+
expect(flow.calls.author).toHaveLength(0);
|
|
332
|
+
}, 30_000);
|
|
333
|
+
|
|
334
|
+
test("a coverage measure that fails after a round is sent to the author and is the last word when it stays failed", async () => {
|
|
335
|
+
const flow = await host({
|
|
336
|
+
author: (call) => {
|
|
337
|
+
const dir = dirOf(call);
|
|
338
|
+
if (call.prompt.includes("did not count")) {
|
|
339
|
+
gitIn(dir, "rm", "-q", "broken");
|
|
340
|
+
gitIn(dir, "commit", "-q", "-m", "mend");
|
|
341
|
+
return { sha: gitIn(dir, "rev-parse", "HEAD") };
|
|
342
|
+
}
|
|
343
|
+
return { sha: commitFile(dir, "broken", "x\n") };
|
|
344
|
+
},
|
|
345
|
+
});
|
|
346
|
+
const ended = await flow.run(`
|
|
347
|
+
import { coverageLoop, inWorktree } from "@niwa-labs/flow-code";
|
|
348
|
+
const measure = (cwd: string) => ({ cmd: ["sh", "-c", "if test -f broken; then echo suite crashed; exit 1; fi; echo '{\\"coverage\\": 0.25}'"], cwd });
|
|
349
|
+
export default workflow({ output: z.json() }, async (): Promise<any> => {
|
|
350
|
+
const stuck = await inWorktree("one", { repo: "$REPO" }, (wt) => coverageLoop("cov", { wt, threshold: 0.7, author: "author", max: 1, measure: measure(wt.path) }));
|
|
351
|
+
const mended = await inWorktree("two", { repo: "$REPO" }, (wt) => coverageLoop("cov", { wt, threshold: 0.7, author: "author", max: 2, measure: measure(wt.path) }));
|
|
352
|
+
return { stuck, mended };
|
|
353
|
+
});`);
|
|
354
|
+
const out = resultOf(ended) as { stuck: { after: number; reached: boolean; failure?: { reason: string } }; mended: { reached: boolean; failure?: unknown } };
|
|
355
|
+
expect(out.stuck).toMatchObject({ before: 0.25, after: 0.25, reached: false });
|
|
356
|
+
expect(out.stuck.failure?.reason).toContain("suite crashed");
|
|
357
|
+
expect(out.mended.failure).toBeUndefined();
|
|
358
|
+
expect(flow.calls.author?.[2]?.prompt).toContain("suite crashed");
|
|
359
|
+
}, 30_000);
|
|
360
|
+
|
|
361
|
+
test("a test that times out in the red phase has shown no failure", async () => {
|
|
362
|
+
const flow = await host({ author: (call) => ({ sha: commitFile(dirOf(call), call.prompt.includes("fails now") ? "check.sh" : "impl.txt", "x\n") }) });
|
|
363
|
+
const ended = await flow.run(`
|
|
364
|
+
import { inWorktree, tddLoop } from "@niwa-labs/flow-code";
|
|
365
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
366
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
367
|
+
// It hangs until impl.txt exists: the red phase times out, and the green phase would pass.
|
|
368
|
+
const test = { cmd: ["sh", "-c", "if test -f impl.txt; then exit 0; fi; exec sleep 5"], cwd: wt.path, timeout: 300 };
|
|
369
|
+
const out = await tddLoop("tdd", { task: "add impl", wt, author: "author", test, max: 1 });
|
|
370
|
+
return { stopped: out.stopped, redSha: out.value.redSha, titles: out.judged.findings.map((f) => f.title) };
|
|
371
|
+
}));`);
|
|
372
|
+
expect(resultOf(ended)).toEqual({ stopped: "max-rounds", redSha: "", titles: ["the new test did not finish"] });
|
|
373
|
+
expect(flow.calls.author).toHaveLength(1);
|
|
374
|
+
}, 30_000);
|
|
375
|
+
|
|
376
|
+
test("a repro that times out is no bug to fix", async () => {
|
|
377
|
+
const flow = await host({ author: (call) => ({ sha: commitFile(dirOf(call), "x.txt", "x\n"), cause: "none" }) });
|
|
378
|
+
const ended = await flow.run(`
|
|
379
|
+
import { debugLoop, inWorktree } from "@niwa-labs/flow-code";
|
|
380
|
+
export default workflow({ output: z.json() }, async (): Promise<any> =>
|
|
381
|
+
inWorktree("wt", { repo: "$REPO" }, async (wt) => {
|
|
382
|
+
const out = await debugLoop("bug", { symptom: "it hangs", wt, author: "author", repro: { cmd: ["sh", "-c", "exec sleep 5"], cwd: wt.path, timeout: 300 } });
|
|
383
|
+
return { verdict: out.judged.verdict, sha: out.sha, titles: out.judged.findings.map((f) => f.title) };
|
|
384
|
+
}));`);
|
|
385
|
+
expect(resultOf(ended)).toEqual({ verdict: "INCONCLUSIVE", sha: "", titles: ["the repro did not finish"] });
|
|
386
|
+
expect(flow.calls.author).toHaveLength(0);
|
|
387
|
+
}, 30_000);
|
|
388
|
+
|
|
389
|
+
test("a self-test fails on any failed or broken scenario and is inconclusive with nothing passed", () => {
|
|
390
|
+
expect(selfTestVerdict([{ status: "pass" }, { status: "pass" }])).toBe("PASS");
|
|
391
|
+
expect(selfTestVerdict([{ status: "pass" }, { status: "gated" }])).toBe("PASS+NOTES");
|
|
392
|
+
expect(selfTestVerdict([{ status: "pass" }, { status: "error" }])).toBe("FAIL");
|
|
393
|
+
expect(selfTestVerdict([{ status: "gated" }])).toBe("INCONCLUSIVE");
|
|
394
|
+
expect(selfTestVerdict([])).toBe("INCONCLUSIVE");
|
|
395
|
+
});
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// The example of the flow libraries, run as it is written: a registered script that imports
|
|
2
|
+
// `revise`, `verify` and `Attempt` from flow and `inWorktree` from flow-code. Its copy in the home
|
|
3
|
+
// finds the libraries through the packages niwa runs with. In a worktree of a real repository the
|
|
4
|
+
// author commits, two verifiers judge the commit, and a fail sends the author round again.
|
|
5
|
+
|
|
6
|
+
import { afterEach, expect, test } from "bun:test";
|
|
7
|
+
import { readFileSync, realpathSync } from "node:fs";
|
|
8
|
+
import { symlink, writeFile } from "node:fs/promises";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
|
|
11
|
+
import { codeHost, commitFile, gitIn, resultOf } from "./support.ts";
|
|
12
|
+
import type { CodeHost } from "./support.ts";
|
|
13
|
+
|
|
14
|
+
const hosts: CodeHost[] = [];
|
|
15
|
+
afterEach(async () => {
|
|
16
|
+
for (const one of hosts.splice(0)) await one.close();
|
|
17
|
+
}, 30_000);
|
|
18
|
+
|
|
19
|
+
const EXAMPLE = `
|
|
20
|
+
import { Attempt, revise, verify } from "@niwa-labs/flow";
|
|
21
|
+
import type { Judged } from "@niwa-labs/flow";
|
|
22
|
+
import { inWorktree } from "@niwa-labs/flow-code";
|
|
23
|
+
|
|
24
|
+
const prompt = (task: string, feedback: Judged | null): string =>
|
|
25
|
+
feedback === null ? task : task + "\\n\\nThe last attempt did not pass:\\n" + feedback.findings.map((f) => "- " + f.title + ": " + f.evidence).join("\\n");
|
|
26
|
+
|
|
27
|
+
export default workflow({
|
|
28
|
+
description: "Fix and verify",
|
|
29
|
+
input: z.object({ task: z.string(), repo: z.string() }),
|
|
30
|
+
output: Attempt(z.object({ sha: z.string() })),
|
|
31
|
+
uses: ["session.start.*", "git.worktree.*"],
|
|
32
|
+
phases: [{ title: "Build" }, { title: "Verify" }],
|
|
33
|
+
}, async ({ task, repo }) =>
|
|
34
|
+
inWorktree("wt", { repo }, (wt) =>
|
|
35
|
+
revise("fix", {
|
|
36
|
+
max: 3,
|
|
37
|
+
write: (feedback) => {
|
|
38
|
+
phase("Build");
|
|
39
|
+
return ask({ harness: "author", cwd: wt.path, schema: z.object({ sha: z.string() }) }, prompt(task, feedback));
|
|
40
|
+
},
|
|
41
|
+
judge: ({ sha }) => {
|
|
42
|
+
phase("Verify");
|
|
43
|
+
return verify("check", { subject: \`\${repo}@\${sha}\`, by: ["verifier", "verifier-alt"], require: "all" });
|
|
44
|
+
},
|
|
45
|
+
})));
|
|
46
|
+
`;
|
|
47
|
+
|
|
48
|
+
test("the example fixes, verifies and fixes again until both verifiers pass, from its stored copy", async () => {
|
|
49
|
+
let builds = 0;
|
|
50
|
+
let altChecks = 0;
|
|
51
|
+
const flow = await codeHost({
|
|
52
|
+
author: (call) => {
|
|
53
|
+
builds += 1;
|
|
54
|
+
if (call.cwd === undefined) throw new Error("the author was not started in the worktree");
|
|
55
|
+
return { sha: commitFile(call.cwd, "fix.txt", `attempt ${String(builds)}\n`) };
|
|
56
|
+
},
|
|
57
|
+
verifier: () => ({ verdict: "PASS", findings: [] }),
|
|
58
|
+
"verifier-alt": () => {
|
|
59
|
+
altChecks += 1;
|
|
60
|
+
return altChecks === 1 ? { verdict: "FAIL", findings: [{ title: "the edge case is not handled", evidence: "empty input", severity: "blocks" }] } : { verdict: "PASS+NOTES", findings: [{ title: "naming", evidence: "x", severity: "note" }] };
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
hosts.push(flow);
|
|
64
|
+
const src = join(flow.home, "example");
|
|
65
|
+
await Bun.write(join(src, ".keep"), "");
|
|
66
|
+
await symlink(realpathSync(join(import.meta.dir, "..", "..", "workflow", "node_modules")), join(src, "node_modules"), "dir");
|
|
67
|
+
await writeFile(join(src, "fix-and-verify.ts"), EXAMPLE);
|
|
68
|
+
|
|
69
|
+
const { id, version } = (await flow.host.call("workflow.register" as never, { file: join(src, "fix-and-verify.ts") } as never)) as { id: string; version: string };
|
|
70
|
+
const manifest = JSON.parse(readFileSync(join(flow.home, "workflows", id, `${version}.json`), "utf8")) as { imports: Record<string, string | null> };
|
|
71
|
+
expect(manifest.imports).toEqual({ "@niwa-labs/flow": "0.0.0", "@niwa-labs/flow-code": "0.0.0" });
|
|
72
|
+
|
|
73
|
+
const { run } = (await flow.host.call("workflow.start" as never, { id, args: { task: "Handle empty input", repo: flow.repo }, limits: { deadline: 7_200_000 } } as never)) as { run: string };
|
|
74
|
+
const ended = (await flow.host.call("workflow.wait" as never, { run, timeoutMs: 30_000 } as never)) as never;
|
|
75
|
+
const attempt = resultOf(ended) as { value: { sha: string }; judged: { verdict: string; findings: unknown[] }; rounds: number; history: Array<{ verdict: string }>; stopped: string };
|
|
76
|
+
|
|
77
|
+
expect(attempt.stopped).toBe("accepted");
|
|
78
|
+
expect(attempt.rounds).toBe(2);
|
|
79
|
+
expect(attempt.history.map((one) => one.verdict)).toEqual(["FAIL", "PASS+NOTES"]);
|
|
80
|
+
expect(attempt.judged.findings).toHaveLength(1);
|
|
81
|
+
expect(flow.calls.author?.[1]?.prompt).toContain("the edge case is not handled");
|
|
82
|
+
// The worktree with the commits is kept, on a branch of its own: niwa never merges.
|
|
83
|
+
const worktree = flow.calls.author?.[1]?.cwd as string;
|
|
84
|
+
expect(gitIn(worktree, "rev-parse", "HEAD")).toBe(attempt.value.sha);
|
|
85
|
+
expect(gitIn(flow.repo, "rev-parse", "main")).not.toBe(attempt.value.sha);
|
|
86
|
+
}, 60_000);
|