@niwa-labs/flow-code 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,177 @@
1
+ /**
2
+ * Worktrees for a script: work in one and leave it when it is clean, or build in one per seat and
3
+ * keep the one picked. A worktree is made and removed through the git module's actions; niwa never
4
+ * merges, so the branch of a worktree is the result.
5
+ */
6
+
7
+ import { checkKey, isFatal, map, script, seatKeys, stream } from "@niwa-labs/flow";
8
+ import type { Seat, SeatOptions, Settled } from "@niwa-labs/flow";
9
+
10
+ /** A worktree as the git module made it: a checkout of `repo` on its own branch, in a directory niwa keeps. */
11
+ export type Worktree = { id: string; description: string; repo: string; path: string; branch: string; base: string; baseSha: string };
12
+
13
+ /** A description a worktree takes: a few words, at most 60 characters. */
14
+ function describe(text: string): string {
15
+ const words = text.trim().replace(/\s+/g, " ");
16
+ return (words.length < 3 ? `${words} work` : words).slice(0, 60).trim();
17
+ }
18
+
19
+ /** Make a worktree of `repo` from `base` (its current branch by default). A step of the script. */
20
+ export async function makeWorktree(opts: { repo: string; base?: string; description: string }): Promise<Worktree> {
21
+ if (typeof opts.repo !== "string" || opts.repo === "") throw new TypeError("a worktree is made from a repository: give `repo`, any directory inside it");
22
+ return (await script().call("git.worktree.create", {
23
+ repo: opts.repo,
24
+ ...(opts.base === undefined ? {} : { base: opts.base }),
25
+ description: describe(opts.description),
26
+ })) as Worktree;
27
+ }
28
+
29
+ /**
30
+ * Remove a worktree. Without `force` the git module refuses one that is dirty, has commits its base
31
+ * does not have or a live session in it: the worktree is kept, and the answer is false.
32
+ */
33
+ export async function removeWorktree(wt: Worktree, force = false): Promise<boolean> {
34
+ try {
35
+ await script().call("git.worktree.remove", { id: wt.id, ...(force ? { force: true } : {}) });
36
+ return true;
37
+ } catch (error) {
38
+ if (error instanceof Error && error.name === "WorktreeRefusal") return false;
39
+ throw error;
40
+ }
41
+ }
42
+
43
+ /** A seat that works in a directory: its `ask()` calls start there. */
44
+ export function seatIn(seat: Seat, cwd: string): SeatOptions {
45
+ return { ...(typeof seat === "string" ? { harness: seat } : seat), cwd };
46
+ }
47
+
48
+ /**
49
+ * How a worktree stands now, as the git module reports it (`git.worktree.get`): the commit its
50
+ * branch is at and whether anything in it is not in that commit. The module reads it outside any
51
+ * box: a command run in the worktree is boxed to that directory, and a linked worktree keeps its git
52
+ * directory in the repository it belongs to. A step of the script, under `key` when one is given.
53
+ */
54
+ export async function stateOf(wt: Worktree, key?: string): Promise<{ sha: string; dirty: boolean }> {
55
+ const ask = () => script().call("git.worktree.get", { id: wt.id });
56
+ const got = (await (key === undefined ? ask() : script().scope(key, undefined, ask))) as { head?: string; dirty: boolean };
57
+ if (got.head === undefined) throw new Error(`the head of the worktree ${wt.id} could not be read: its directory is gone`);
58
+ return { sha: got.head, dirty: got.dirty };
59
+ }
60
+
61
+ /** The commit a worktree's branch is at: read under `key` by `stateOf`. */
62
+ export async function headOf(key: string, wt: Worktree): Promise<string> {
63
+ return (await stateOf(wt, key)).sha;
64
+ }
65
+
66
+ /**
67
+ * Work in a new worktree. When `fn` returns, the worktree is removed if it is clean with no new
68
+ * commit, and kept otherwise: the commits are the result. When `fn` throws, the worktree is kept as
69
+ * it is, for whoever looks into why.
70
+ */
71
+ export async function inWorktree<T>(key: string, opts: { repo: string; base?: string; description?: string }, fn: (wt: Worktree) => Promise<T>): Promise<T> {
72
+ checkKey(key, "the key of a worktree");
73
+ return script().scope(key, undefined, async () => {
74
+ const wt = await makeWorktree({ repo: opts.repo, ...(opts.base === undefined ? {} : { base: opts.base }), description: opts.description ?? `work in ${key}` });
75
+ const result = await fn(wt);
76
+ await removeWorktree(wt);
77
+ return result;
78
+ });
79
+ }
80
+
81
+ /**
82
+ * Build the same thing once per seat, each in a worktree of its own, then let `pick` choose among the
83
+ * builds that succeeded. The winner's worktree is kept; the others are removed, commits and all.
84
+ */
85
+ export async function forkSelect<T>(
86
+ key: string,
87
+ opts: { seats: readonly Seat[]; repo: string; base?: string; build: (seat: Seat, wt: Worktree) => Promise<T>; pick: (candidates: Record<string, T>) => Promise<string> },
88
+ ): Promise<{ winner: { id: string; value: T; worktree: Worktree }; rejected: string[] }> {
89
+ checkKey(key, "the key of a fork");
90
+ const keys = seatKeys(opts.seats, "the key of a seat");
91
+ if (keys.length === 0) throw new TypeError(`forkSelect "${key}": give at least one seat`);
92
+ return script().scope(key, undefined, async () => {
93
+ const made = new Map<string, Worktree>();
94
+ const builds = await map(
95
+ "builds",
96
+ opts.seats.map((seat, index) => ({ seat, id: keys[index] as string })),
97
+ { key: (one) => one.id },
98
+ async (one) => {
99
+ const wt = await makeWorktree({ repo: opts.repo, ...(opts.base === undefined ? {} : { base: opts.base }), description: `fork ${one.id}` });
100
+ made.set(one.id, wt);
101
+ return opts.build(one.seat, wt);
102
+ },
103
+ );
104
+ const built = new Map<string, T>();
105
+ builds.forEach((build, index) => {
106
+ if (build.ok) built.set(keys[index] as string, build.value);
107
+ });
108
+ if (built.size === 0) throw new Error(`forkSelect "${key}": no build succeeded (${failures(keys, builds)})`);
109
+ const id = await opts.pick(Object.fromEntries(built));
110
+ if (!built.has(id)) throw new Error(`forkSelect "${key}": "${id}" is not a build that succeeded`);
111
+ const rejected = keys.filter((one) => one !== id);
112
+ for (const one of rejected) {
113
+ const wt = made.get(one);
114
+ if (wt !== undefined) await removeWorktree(wt, true);
115
+ }
116
+ return { winner: { id, value: built.get(id) as T, worktree: made.get(id) as Worktree }, rejected };
117
+ });
118
+ }
119
+
120
+ /** Why builds failed, one `seat: reason` after another. */
121
+ function failures(keys: readonly string[], builds: readonly Settled<unknown>[]): string {
122
+ return builds.flatMap((build, index) => (build.ok ? [] : [`${keys[index] as string}: ${build.error.message}`])).join("; ");
123
+ }
124
+
125
+ /**
126
+ * Build the same thing once per seat, each in a worktree of its own, and take the first build that
127
+ * `passes`, whichever seat is quick: the builds still running are cancelled, and every worktree but
128
+ * the winner's is removed, commits and all. When no build passes, the first seat's build that
129
+ * finished is the winner, with `passed: false`, for the caller to judge by what its value says.
130
+ * A build that threw is no candidate; with none finished the call throws, naming each cause.
131
+ */
132
+ export async function forkRace<T>(
133
+ key: string,
134
+ opts: { seats: readonly Seat[]; repo: string; base?: string; build: (seat: Seat, wt: Worktree) => Promise<T>; passes: (value: T) => boolean },
135
+ ): Promise<{ winner: { id: string; value: T; worktree: Worktree; passed: boolean }; rejected: string[] }> {
136
+ checkKey(key, "the key of a race");
137
+ const keys = seatKeys(opts.seats, "the key of a seat");
138
+ if (keys.length === 0) throw new TypeError(`forkRace "${key}": give at least one seat`);
139
+ const made = new Map<string, Worktree>();
140
+ const branches = Object.fromEntries(
141
+ opts.seats.map((seat, index) => [
142
+ keys[index] as string,
143
+ async () => {
144
+ const wt = await makeWorktree({ repo: opts.repo, ...(opts.base === undefined ? {} : { base: opts.base }), description: `fork ${keys[index] as string}` });
145
+ made.set(keys[index] as string, wt);
146
+ return opts.build(seat, wt);
147
+ },
148
+ ]),
149
+ );
150
+ const outcomes = new Map<string, Settled<T>>();
151
+ let first: string | undefined;
152
+ const race = stream(key, branches);
153
+ try {
154
+ for (let got = await race.next(); got !== null; got = await race.next()) {
155
+ if (!got.result.ok && isFatal(got.result.error)) throw Object.assign(new Error(got.result.error.message), { name: got.result.error.name });
156
+ outcomes.set(got.key, got.result);
157
+ if (got.result.ok && opts.passes(got.result.value)) {
158
+ first = got.key;
159
+ break;
160
+ }
161
+ }
162
+ } finally {
163
+ await race.close();
164
+ }
165
+ const finished = keys.find((one) => outcomes.get(one)?.ok === true);
166
+ const id = first ?? finished;
167
+ if (id === undefined) {
168
+ throw new Error(`forkRace "${key}": no build finished (${failures(keys, keys.map((one) => outcomes.get(one) ?? { ok: false as const, error: { name: "Cancelled", message: "cancelled" } }))})`);
169
+ }
170
+ const rejected = keys.filter((one) => one !== id);
171
+ for (const one of rejected) {
172
+ const wt = made.get(one);
173
+ if (wt !== undefined) await removeWorktree(wt, true);
174
+ }
175
+ const value = (outcomes.get(id) as { ok: true; value: T }).value;
176
+ return { winner: { id, value, worktree: made.get(id) as Worktree, passed: first !== undefined }, rejected };
177
+ }
@@ -0,0 +1,236 @@
1
+ // The whole-task helpers in a real script, on a real git repository: an implementation judged by
2
+ // its test, one the owner did not let start, an integration of branches, a diagnosis that stops at
3
+ // a confirmed cause, and a self-test that fails on a failed scenario.
4
+
5
+ import { afterEach, expect, test } from "bun:test";
6
+ import { writeFileSync } from "node:fs";
7
+ import { join } from "node:path";
8
+
9
+ import { codeHost, commitFile, gitIn, resultOf } from "./support.ts";
10
+ import type { CodeHost, Reply } from "./support.ts";
11
+
12
+ const hosts: CodeHost[] = [];
13
+ afterEach(async () => {
14
+ for (const one of hosts.splice(0)) await one.close();
15
+ }, 30_000);
16
+
17
+ async function host(agents: Record<string, Reply> = {}): Promise<CodeHost> {
18
+ const made = await codeHost(agents);
19
+ hosts.push(made);
20
+ return made;
21
+ }
22
+
23
+ const dirOf = (call: { cwd?: string }): string => {
24
+ if (call.cwd === undefined) throw new Error("the agent was not started in a directory");
25
+ return call.cwd;
26
+ };
27
+
28
+ test("an economy implementation commits in a new worktree and is judged by its test there", async () => {
29
+ const flow = await host({ author: (call) => ({ sha: commitFile(dirOf(call), "work.txt", "done\n") }) });
30
+ const ended = await flow.run(`
31
+ import { implement } from "@niwa-labs/flow-code";
32
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
33
+ const done = await implement("task", {
34
+ task: "Add work.txt",
35
+ repo: "$REPO",
36
+ mode: "economy",
37
+ seats: { author: "author" },
38
+ test: { cmd: ["sh", "-c", "test -f work.txt"], cwd: "$REPO" },
39
+ });
40
+ return { sha: done.sha, path: done.worktree.path, verdict: done.judged.verdict };
41
+ });`);
42
+ const out = resultOf(ended) as { sha: string; path: string; verdict: string };
43
+ expect(out.verdict).toBe("PASS");
44
+ expect(gitIn(out.path, "rev-parse", "HEAD")).toBe(out.sha);
45
+ expect(out.path).not.toBe(flow.repo);
46
+ }, 30_000);
47
+
48
+ test("an economy implementation whose author leaves the work uncommitted is not judged by its test", async () => {
49
+ const flow = await host({
50
+ author: (call) => {
51
+ writeFileSync(join(dirOf(call), "work.txt"), "done\n");
52
+ return { summary: "done" };
53
+ },
54
+ });
55
+ const ended = await flow.run(`
56
+ import { implement } from "@niwa-labs/flow-code";
57
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
58
+ const done = await implement("task", {
59
+ task: "Add work.txt",
60
+ repo: "$REPO",
61
+ mode: "economy",
62
+ seats: { author: "author" },
63
+ test: { cmd: ["sh", "-c", "test -f work.txt"], cwd: "$REPO" },
64
+ });
65
+ return { sha: done.sha, path: done.worktree.path, judged: done.judged };
66
+ });`);
67
+ const out = resultOf(ended) as { sha: string; path: string; judged: { verdict: string; findings: Array<{ title: string }> } };
68
+ expect(out.judged.verdict).toBe("FAIL");
69
+ expect(out.judged.findings.map((one) => one.title)).toEqual(["the work was left uncommitted"]);
70
+ expect(out.sha).toBe(gitIn(out.path, "rev-parse", "HEAD"));
71
+ }, 30_000);
72
+
73
+ test("a passing check that changes the worktree cannot clear an implementation", async () => {
74
+ const build: Reply = (call) => ({ sha: commitFile(dirOf(call), "work.txt", "x\n") });
75
+ const flow = await host({ author: build, reviewer: () => ({ verdict: "PASS", findings: [] }) });
76
+ const ended = await flow.run(`
77
+ import { implement } from "@niwa-labs/flow-code";
78
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
79
+ const verdicts = [];
80
+ for (const mode of ["economy", "race"] as const) {
81
+ const done = await implement(mode, {
82
+ task: "Add work.txt", repo: "$REPO", mode, seats: { author: "author", reviewer: "reviewer" },
83
+ test: { cmd: ["sh", "-c", "touch uncommitted.txt"], cwd: "$REPO" },
84
+ });
85
+ verdicts.push(done.judged.verdict);
86
+ }
87
+ return verdicts;
88
+ });`);
89
+ expect(resultOf(ended)).toEqual(["FAIL", "FAIL"]);
90
+ expect(flow.calls.reviewer).toEqual([]);
91
+ }, 30_000);
92
+
93
+ test("a race is won by the first fork whose test passes, while a slower fork is still working", async () => {
94
+ const build: Reply = (call) => ({ sha: commitFile(dirOf(call), "work.txt", `${String(call.harness)}\n`) });
95
+ // The first seat never answers: the call must not wait for it.
96
+ const flow = await codeHost({ slow: build, quick: build }, ["flow", "flow-code"], ["slow"]);
97
+ hosts.push(flow);
98
+ const ended = await flow.run(`
99
+ import { implement } from "@niwa-labs/flow-code";
100
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
101
+ const done = await implement("task", {
102
+ task: "Add work.txt",
103
+ repo: "$REPO",
104
+ mode: "race",
105
+ seats: { author: "slow", cheap: ["slow", "quick"] },
106
+ test: { cmd: ["sh", "-c", "test -f work.txt"], cwd: "$REPO" },
107
+ });
108
+ return { sha: done.sha, path: done.worktree.path, verdict: done.judged.verdict };
109
+ });`);
110
+ const out = resultOf(ended) as { sha: string; path: string; verdict: string };
111
+ expect(out.verdict).toBe("PASS");
112
+ expect(out.path).toBe(dirOf(flow.calls.quick?.[0] ?? {}));
113
+ expect(gitIn(out.path, "rev-parse", "HEAD")).toBe(out.sha);
114
+ // The fork that lost is removed with its worktree: the repository and the winner are what is left.
115
+ expect(gitIn(flow.repo, "worktree", "list").split("\n")).toHaveLength(2);
116
+ }, 30_000);
117
+
118
+ test("a race that no fork wins keeps the failure its test showed, whatever the reviewer says", async () => {
119
+ const build: Reply = (call) => ({ sha: commitFile(dirOf(call), "work.txt", "x\n") });
120
+ const flow = await host({ a: build, b: build, reviewer: () => ({ verdict: "PASS", findings: [] }) });
121
+ const ended = await flow.run(`
122
+ import { implement } from "@niwa-labs/flow-code";
123
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
124
+ // The test fails the first time it runs in a fork and would pass the second time.
125
+ const done = await implement("task", {
126
+ task: "Add work.txt",
127
+ repo: "$REPO",
128
+ mode: "race",
129
+ seats: { author: "a", cheap: ["a", "b"], reviewer: "reviewer" },
130
+ test: { cmd: ["sh", "-c", "if test -f ran; then exit 0; fi; touch ran; echo flaky boom; exit 1"], cwd: "$REPO" },
131
+ });
132
+ return done.judged;
133
+ });`);
134
+ const judged = resultOf(ended) as { verdict: string; findings: Array<{ title: string; evidence: string }> };
135
+ expect(judged.verdict).toBe("FAIL");
136
+ expect(judged.findings[0]?.evidence).toContain("flaky boom");
137
+ expect(flow.calls.reviewer ?? []).toEqual([]);
138
+ }, 30_000);
139
+
140
+ test("a safe implementation the owner declines to start stops before any worktree or agent", async () => {
141
+ const flow = await host({ author: (call) => ({ sha: commitFile(dirOf(call), "work.txt", "done\n") }) });
142
+ const run = await flow.start(`
143
+ import { implement } from "@niwa-labs/flow-code";
144
+ export default workflow({ output: z.json() }, async (): Promise<any> =>
145
+ implement("task", { task: "Add work.txt", repo: "$REPO", mode: "safe", seats: { author: "author" } }).then(
146
+ () => "implemented",
147
+ (error: Error) => error.name + ": " + error.message,
148
+ ));`);
149
+ await flow.answer("decline");
150
+ const ended = (await flow.host.call("workflow.wait" as never, { run, timeoutMs: 30_000 } as never)) as never;
151
+ expect(resultOf(ended)).toBe("ImplementStopped: the owner did not let the task start (declined)");
152
+ expect(flow.calls.author).toEqual([]);
153
+ }, 30_000);
154
+
155
+ test("an integration merges the branches into a worktree of the target and runs the test on the result", async () => {
156
+ const flow = await host({
157
+ integrator: (call) => {
158
+ const dir = dirOf(call);
159
+ gitIn(dir, "merge", "--no-edit", "-q", "feature-a", "feature-b");
160
+ return { sha: gitIn(dir, "rev-parse", "HEAD"), conflicts: [] };
161
+ },
162
+ });
163
+ for (const branch of ["feature-a", "feature-b"]) {
164
+ gitIn(flow.repo, "checkout", "-q", "-b", branch, "main");
165
+ commitFile(flow.repo, `${branch}.txt`, `${branch}\n`);
166
+ }
167
+ gitIn(flow.repo, "checkout", "-q", "main");
168
+ const ended = await flow.run(`
169
+ import { integrate } from "@niwa-labs/flow-code";
170
+ export default workflow({ output: z.json() }, async (): Promise<any> =>
171
+ integrate("join", {
172
+ repo: "$REPO",
173
+ into: "main",
174
+ branches: ["feature-a", "feature-b"],
175
+ integrator: "integrator",
176
+ test: { cmd: ["sh", "-c", "test -f feature-a.txt && test -f feature-b.txt"], cwd: "$REPO" },
177
+ }));`);
178
+ const out = resultOf(ended) as { sha: string; conflicts: unknown[]; judged: { verdict: string } };
179
+ expect(out.judged.verdict).toBe("PASS");
180
+ expect(out.conflicts).toEqual([]);
181
+ expect(flow.calls.integrator?.[0]?.prompt).toContain("feature-a, feature-b");
182
+ // niwa never merges into the target itself: main is where it was.
183
+ expect(gitIn(flow.repo, "rev-parse", "main")).not.toBe(out.sha);
184
+ }, 30_000);
185
+
186
+ test("a diagnosis runs the repro each round and stops at the first confirmed cause", async () => {
187
+ let rounds = 0;
188
+ const flow = await host({
189
+ investigator: () => {
190
+ rounds += 1;
191
+ return rounds === 1
192
+ ? { hypotheses: [{ claim: "the cache is stale", status: "open", evidence: "the repro fails after a restart" }] }
193
+ : {
194
+ hypotheses: [
195
+ { claim: "the cache is stale", status: "refuted", evidence: "it fails with the cache off" },
196
+ { claim: "the clock is read twice", status: "confirmed", evidence: "two reads differ by a second" },
197
+ ],
198
+ };
199
+ },
200
+ });
201
+ const ended = await flow.run(`
202
+ import { diagnoseLoop } from "@niwa-labs/flow-code";
203
+ export default workflow({ output: z.json() }, async (): Promise<any> => {
204
+ const wt = { id: "wt-1", description: "here", repo: "$REPO", path: "$REPO", branch: "main", base: "main", baseSha: "" };
205
+ return diagnoseLoop("why", { symptom: "the total is off by one", wt, repro: { cmd: ["sh", "-c", "echo off by one; exit 1"], cwd: "$REPO" }, investigator: "investigator", max: 3 });
206
+ });`);
207
+ const out = resultOf(ended) as { cause: string | null; hypotheses: Array<{ status: string }> };
208
+ expect(out.cause).toBe("the clock is read twice");
209
+ expect(out.hypotheses.map((one) => one.status)).toEqual(["refuted", "confirmed"]);
210
+ expect(flow.calls.investigator).toHaveLength(2);
211
+ expect(flow.calls.investigator?.[1]?.prompt).toContain("[open] the cache is stale");
212
+ expect(flow.calls.investigator?.[0]?.prompt).toContain("off by one");
213
+ }, 30_000);
214
+
215
+ test("a self-test probes every scenario and fails when one of them fails", async () => {
216
+ const flow = await host({
217
+ probe: (call) => (call.prompt.includes("open the page") ? { status: "pass", evidence: "it opened" } : { status: "fail", evidence: "the total was 9" }),
218
+ });
219
+ const ended = await flow.run(`
220
+ import { selfTest } from "@niwa-labs/flow-code";
221
+ export default workflow({ output: z.json() }, async (): Promise<any> =>
222
+ selfTest("probe", {
223
+ probe: "probe",
224
+ scenarios: [
225
+ { id: "open", steps: "open the page", expect: "it opens" },
226
+ { id: "sum", steps: "add two and two", expect: "four" },
227
+ ],
228
+ }));`);
229
+ expect(resultOf(ended)).toEqual({
230
+ verdict: "FAIL",
231
+ rows: [
232
+ { id: "open", status: "pass", evidence: "it opened" },
233
+ { id: "sum", status: "fail", evidence: "the total was 9" },
234
+ ],
235
+ });
236
+ }, 30_000);