@niwa-labs/flow-code 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +26 -2
- package/package.json +16 -4
- package/src/build.ts +162 -0
- package/src/index.ts +13 -0
- package/src/loops.ts +215 -0
- package/src/review.ts +141 -0
- package/src/worktree.ts +177 -0
- package/test/build.test.ts +236 -0
- package/test/code.test.ts +395 -0
- package/test/example.test.ts +86 -0
- package/test/support.ts +187 -0
- package/tsconfig.json +6 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Niwa Labs contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,27 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @niwa-labs/flow-code
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Workflow helpers for work in a repository: worktrees, fork-and-select, review and fix loops, TDD, diagnosis, debugging, coverage, self-tests, implementation and integration.
|
|
4
|
+
|
|
5
|
+
```ts
|
|
6
|
+
import { inWorktree, reviewFix } from "@niwa-labs/flow-code";
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Built on `@niwa-labs/flow` and on the git and proc modules' actions, called by name: worktrees with
|
|
10
|
+
`git.worktree.create`, `git.worktree.get` (the head of a worktree is read by the git module, not by a
|
|
11
|
+
command in the worktree) and `git.worktree.remove`, checks with `runCheck` of `@niwa-labs/proc/flow`
|
|
12
|
+
(re-exported here). A run that uses them needs those actions in its grant.
|
|
13
|
+
|
|
14
|
+
- `inWorktree` removes a worktree its block left clean and keeps one with commits; a block that
|
|
15
|
+
throws leaves it as it is. `forkSelect` keeps the picked fork and removes the others; `forkRace`
|
|
16
|
+
takes the first fork that passes and cancels the rest.
|
|
17
|
+
- The commit a helper reports is read from the worktree after the agent is done, never from the
|
|
18
|
+
agent's answer, and changes left outside it are a blocking finding: a check must leave the
|
|
19
|
+
worktree as it found it, or have its output ignored by git.
|
|
20
|
+
- `reviewFix`, `tddLoop`, `debugLoop` and `coverageLoop` return their last result and why they
|
|
21
|
+
stopped; a check that does not pass is a blocking finding sent back to the author, and one that
|
|
22
|
+
does not finish (timed out, cancelled) shows neither a failure nor a pass. A coverage measure that
|
|
23
|
+
fails before the first round throws `MeasureFailed`.
|
|
24
|
+
- `reviewPanel` lists the reviewers that could not look in `failures`; with it empty the panel is
|
|
25
|
+
complete.
|
|
26
|
+
- `diagnoseLoop`, `selfTest`, `implement` (`economy`, `race`, `quality`, `safe`)
|
|
27
|
+
and `integrate`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@niwa-labs/flow-code",
|
|
3
|
-
"version": "0.0.0
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"exports": {
|
|
7
|
+
".": "./src/index.ts"
|
|
8
|
+
},
|
|
9
|
+
"dependencies": {
|
|
10
|
+
"@niwa-labs/flow": "0.0.0",
|
|
11
|
+
"@niwa-labs/git": "0.0.0",
|
|
12
|
+
"@niwa-labs/proc": "0.0.0"
|
|
13
|
+
},
|
|
14
|
+
"devDependencies": {
|
|
15
|
+
"@niwa-labs/testing": "0.0.0",
|
|
16
|
+
"zod": "^4.6.7"
|
|
17
|
+
}
|
|
18
|
+
}
|
package/src/build.ts
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whole pieces of work: probe a running system with scenarios, implement a task in one of four
|
|
3
|
+
* modes, and integrate branches.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { askSeat, checkKey, checkKeys, humanGate, map, script, verify } from "@niwa-labs/flow";
|
|
7
|
+
import type { Judged, Seat, Verdict } from "@niwa-labs/flow";
|
|
8
|
+
import { runCheck } from "@niwa-labs/proc/flow";
|
|
9
|
+
import type { Check, CheckResult } from "@niwa-labs/proc/flow";
|
|
10
|
+
|
|
11
|
+
import { changesAfterCheck, checkFailed, commitBy, reviewFix, uncommitted } from "./review.ts";
|
|
12
|
+
import { forkRace, makeWorktree, seatIn, stateOf } from "./worktree.ts";
|
|
13
|
+
import type { Worktree } from "./worktree.ts";
|
|
14
|
+
|
|
15
|
+
/** How a scenario of a self-test went: passed, failed, held behind a gate it cannot pass, or could not be run. */
|
|
16
|
+
export type ScenarioStatus = "pass" | "fail" | "gated" | "error";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The verdict of a self-test: a fail or an error fails it, nothing run (or only gated scenarios) is
|
|
20
|
+
* `INCONCLUSIVE`, passes with gated scenarios are a pass with notes.
|
|
21
|
+
*/
|
|
22
|
+
export function selfTestVerdict(rows: readonly { status: ScenarioStatus }[]): Verdict {
|
|
23
|
+
if (rows.some((row) => row.status === "fail" || row.status === "error")) return "FAIL";
|
|
24
|
+
const passed = rows.filter((row) => row.status === "pass").length;
|
|
25
|
+
if (passed === 0) return "INCONCLUSIVE";
|
|
26
|
+
return passed === rows.length ? "PASS" : "PASS+NOTES";
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Probe scenarios, each in a branch of its own: the probe follows the steps and says whether what it saw is what is expected. */
|
|
30
|
+
export async function selfTest(
|
|
31
|
+
key: string,
|
|
32
|
+
opts: { scenarios: readonly { id: string; steps: string; expect: string }[]; probe: Seat; concurrency?: number },
|
|
33
|
+
): Promise<{ verdict: Verdict; rows: { id: string; status: ScenarioStatus; evidence: string }[] }> {
|
|
34
|
+
checkKey(key, "the key of a self-test");
|
|
35
|
+
checkKeys(
|
|
36
|
+
opts.scenarios.map((one) => one.id),
|
|
37
|
+
"the id of a scenario",
|
|
38
|
+
);
|
|
39
|
+
const { z } = script();
|
|
40
|
+
const answer = z.object({ status: z.enum(["pass", "fail", "gated"]), evidence: z.string() });
|
|
41
|
+
const outcomes = await map(key, opts.scenarios, { key: (one) => one.id, ...(opts.concurrency === undefined ? {} : { concurrency: opts.concurrency }) }, (one) =>
|
|
42
|
+
askSeat(
|
|
43
|
+
opts.probe,
|
|
44
|
+
`Run this scenario and report what you saw.\n\nSteps:\n${one.steps}\n\nExpected:\n${one.expect}\n\n` +
|
|
45
|
+
"Answer pass when what you saw is what is expected, fail when it is not, gated when something you may not pass (a login, an approval) stopped you; give the evidence.",
|
|
46
|
+
answer,
|
|
47
|
+
),
|
|
48
|
+
);
|
|
49
|
+
const rows = outcomes.map((outcome, index) => {
|
|
50
|
+
const id = (opts.scenarios[index] as { id: string }).id;
|
|
51
|
+
return outcome.ok ? { id, status: outcome.value.status as ScenarioStatus, evidence: outcome.value.evidence } : { id, status: "error" as const, evidence: outcome.error.message };
|
|
52
|
+
});
|
|
53
|
+
return { verdict: selfTestVerdict(rows), rows };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** How `implement` works: one author; cheap forks raced on the test; review and fix; or that with the owner's word before and after. */
|
|
57
|
+
export type ImplementMode = "economy" | "race" | "quality" | "safe";
|
|
58
|
+
|
|
59
|
+
/** What the owner did not let through in a `safe` implementation. */
|
|
60
|
+
export class ImplementStopped extends Error {
|
|
61
|
+
override name = "ImplementStopped";
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** The judgement of a check, or none run. */
|
|
65
|
+
async function judgedBy(test: Check | undefined, label: string, wt: Worktree, sha: string): Promise<Judged> {
|
|
66
|
+
if (test === undefined) return { verdict: "INCONCLUSIVE", findings: [], method: "no check was given" };
|
|
67
|
+
const check = await runCheck(label, test);
|
|
68
|
+
if (!check.ok) return checkFailed("the test", check.summary);
|
|
69
|
+
return (await changesAfterCheck(wt, sha)) ?? { verdict: "PASS", findings: [], method: "check (test)" };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** A fork of a race: its commit, test result, and any reason it could not pass. */
|
|
73
|
+
type Candidate = { sha: string; tested?: CheckResult; failure?: Judged };
|
|
74
|
+
|
|
75
|
+
/** The test, run in a worktree. */
|
|
76
|
+
const testIn = (test: Check | undefined, wt: Worktree): Check | undefined => (test === undefined ? undefined : { ...test, cwd: wt.path });
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Implement a task in a new worktree of `repo`; the worktree is kept with the commits.
|
|
80
|
+
*
|
|
81
|
+
* - `economy`: the author works once; the test, if given, judges.
|
|
82
|
+
* - `race`: each cheap seat (the author when none) builds in a fork; the first fork whose test
|
|
83
|
+
* passes wins and the others are cancelled and removed, and the reviewer, if given, judges it.
|
|
84
|
+
* When no fork passes the first seat's is kept and fails with what its test said, whatever the
|
|
85
|
+
* reviewer would say.
|
|
86
|
+
* - `quality`: the author works, then the reviewer (the author when none) reviews and the author fixes.
|
|
87
|
+
* - `safe`: `quality` with the owner's word before the work starts and before it is handed back.
|
|
88
|
+
*/
|
|
89
|
+
export async function implement(
|
|
90
|
+
key: string,
|
|
91
|
+
opts: { task: string; repo: string; mode: ImplementMode; seats: { author: Seat; cheap?: readonly Seat[]; reviewer?: Seat }; test?: Check; base?: string },
|
|
92
|
+
): Promise<{ sha: string; worktree: Worktree; judged: Judged }> {
|
|
93
|
+
checkKey(key, "the key of an implementation");
|
|
94
|
+
const prompt = `Implement this task:\n${opts.task}`;
|
|
95
|
+
return script().scope(key, undefined, async () => {
|
|
96
|
+
if (opts.mode === "safe") {
|
|
97
|
+
const go = await humanGate("start", { summary: `Start this task in ${opts.repo}?\n\n${opts.task}`, options: ["start"] });
|
|
98
|
+
if (go.outcome !== "chosen") throw new ImplementStopped(`the owner did not let the task start (${go.outcome})`);
|
|
99
|
+
}
|
|
100
|
+
if (opts.mode === "race") {
|
|
101
|
+
const raced = await forkRace("forks", {
|
|
102
|
+
seats: opts.seats.cheap !== undefined && opts.seats.cheap.length > 0 ? opts.seats.cheap : [opts.seats.author],
|
|
103
|
+
repo: opts.repo,
|
|
104
|
+
...(opts.base === undefined ? {} : { base: opts.base }),
|
|
105
|
+
build: async (seat, wt): Promise<Candidate> => {
|
|
106
|
+
const built = await commitBy(seat, wt, prompt);
|
|
107
|
+
const test = testIn(opts.test, wt);
|
|
108
|
+
if (built.dirty) return { sha: built.sha, failure: uncommitted() };
|
|
109
|
+
if (test === undefined) return { sha: built.sha };
|
|
110
|
+
const tested = await runCheck("test", test);
|
|
111
|
+
const failure = tested.ok ? await changesAfterCheck(wt, built.sha) : checkFailed("the test", tested.summary);
|
|
112
|
+
return { sha: built.sha, tested, ...(failure === undefined ? {} : { failure }) };
|
|
113
|
+
},
|
|
114
|
+
passes: (one) => one.failure === undefined,
|
|
115
|
+
});
|
|
116
|
+
const { sha, tested, failure } = raced.winner.value;
|
|
117
|
+
const wt = raced.winner.worktree;
|
|
118
|
+
// A winner that did not pass is judged by what its own check said, whoever the reviewer is.
|
|
119
|
+
const judged = failure ?? (opts.seats.reviewer !== undefined
|
|
120
|
+
? await verify("review", { subject: `The work for this task on the branch ${wt.branch} in ${wt.path}, at ${sha}:\n${opts.task}`, by: [seatIn(opts.seats.reviewer, wt.path)] })
|
|
121
|
+
: tested !== undefined
|
|
122
|
+
? { verdict: "PASS" as const, findings: [], method: "check (test)" }
|
|
123
|
+
: await judgedBy(undefined, "judge", wt, sha));
|
|
124
|
+
return { sha, worktree: wt, judged };
|
|
125
|
+
}
|
|
126
|
+
const wt = await makeWorktree({ repo: opts.repo, ...(opts.base === undefined ? {} : { base: opts.base }), description: `implement ${key}` });
|
|
127
|
+
const built = await commitBy(opts.seats.author, wt, prompt);
|
|
128
|
+
const test = testIn(opts.test, wt);
|
|
129
|
+
if (opts.mode === "economy") return { sha: built.sha, worktree: wt, judged: built.dirty ? uncommitted() : await judgedBy(test, "judge", wt, built.sha) };
|
|
130
|
+
const reviewed = await reviewFix("review", { wt, base: wt.baseSha, author: opts.seats.author, reviewers: [opts.seats.reviewer ?? opts.seats.author], ...(test === undefined ? {} : { test }) });
|
|
131
|
+
if (opts.mode === "safe") {
|
|
132
|
+
const done = await humanGate("accept", { summary: `Hand back the work on ${wt.branch} (${reviewed.value.sha}), judged ${reviewed.judged.verdict}?`, options: ["accept"] });
|
|
133
|
+
if (done.outcome !== "chosen") throw new ImplementStopped(`the owner did not accept the work (${done.outcome})`);
|
|
134
|
+
}
|
|
135
|
+
return { sha: reviewed.value.sha, worktree: wt, judged: reviewed.judged };
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Integrate branches into `into`: in a new worktree of it, the integrator merges each branch,
|
|
141
|
+
* resolving conflicts, and commits; the test, if given, judges the result.
|
|
142
|
+
*/
|
|
143
|
+
export async function integrate(
|
|
144
|
+
key: string,
|
|
145
|
+
opts: { repo: string; into: string; branches: readonly string[]; integrator: Seat; test?: Check },
|
|
146
|
+
): Promise<{ sha: string; conflicts: { files: string[]; resolvedBy: Seat }[]; judged: Judged }> {
|
|
147
|
+
checkKey(key, "the key of an integration");
|
|
148
|
+
if (opts.branches.length === 0) throw new TypeError(`integrate "${key}": give at least one branch`);
|
|
149
|
+
const { z } = script();
|
|
150
|
+
return script().scope(key, undefined, async () => {
|
|
151
|
+
const wt = await makeWorktree({ repo: opts.repo, base: opts.into, description: `integrate into ${opts.into}` });
|
|
152
|
+
const merged = await askSeat(
|
|
153
|
+
seatIn(opts.integrator, wt.path),
|
|
154
|
+
`Merge these branches, in this order, into the branch ${wt.branch} checked out in ${wt.path} (made from ${opts.into}): ${opts.branches.join(", ")}.\n` +
|
|
155
|
+
"Resolve every conflict so both sides' intent holds, and commit. Answer with, for each merge that conflicted, the files.",
|
|
156
|
+
z.object({ conflicts: z.array(z.object({ files: z.array(z.string()) })) }),
|
|
157
|
+
);
|
|
158
|
+
const { sha, dirty } = await stateOf(wt);
|
|
159
|
+
const judged = dirty ? uncommitted() : await judgedBy(testIn(opts.test, wt), "test", wt, sha);
|
|
160
|
+
return { sha, conflicts: merged.conflicts.map((one) => ({ files: one.files, resolvedBy: opts.integrator })), judged };
|
|
161
|
+
});
|
|
162
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// Workflow helpers for work in a repository: worktrees, forks, review and fix, test first,
|
|
2
|
+
// diagnosis, debugging, coverage, self-tests, implementation and integration. Each runs inside a
|
|
3
|
+
// script, on its globals, and opens a scope named by the key it is given.
|
|
4
|
+
|
|
5
|
+
export { forkRace, forkSelect, headOf, inWorktree, makeWorktree, removeWorktree, seatIn, stateOf } from "./worktree.ts";
|
|
6
|
+
export type { Worktree } from "./worktree.ts";
|
|
7
|
+
export { checkFailed, commitBy, findingsText, reviewFix, reviewPanel, reviewSubject, uncommitted } from "./review.ts";
|
|
8
|
+
export { MeasureFailed, coverageLoop, coverageOf, debugLoop, diagnoseLoop, tddLoop } from "./loops.ts";
|
|
9
|
+
export type { Hypothesis } from "./loops.ts";
|
|
10
|
+
export { ImplementStopped, implement, integrate, selfTest, selfTestVerdict } from "./build.ts";
|
|
11
|
+
export type { ImplementMode, ScenarioStatus } from "./build.ts";
|
|
12
|
+
export { runCheck } from "@niwa-labs/proc/flow";
|
|
13
|
+
export type { ArtifactRef, Check, CheckResult } from "@niwa-labs/proc/flow";
|
package/src/loops.ts
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Loops of work in a worktree that a command judges: test first, diagnose, debug, raise coverage.
|
|
3
|
+
* Each round's check is a step of the round, and a loop that runs out of rounds gives its last
|
|
4
|
+
* result with the reason it stopped.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { askSeat, checkKey, loop, revise, script } from "@niwa-labs/flow";
|
|
8
|
+
import type { Attempt, Judged, Seat } from "@niwa-labs/flow";
|
|
9
|
+
import { runCheck } from "@niwa-labs/proc/flow";
|
|
10
|
+
import type { ArtifactRef, Check } from "@niwa-labs/proc/flow";
|
|
11
|
+
|
|
12
|
+
import { changesAfterCheck, checkFailed, commitBy, findingsText, uncommitted } from "./review.ts";
|
|
13
|
+
import { seatIn, stateOf } from "./worktree.ts";
|
|
14
|
+
import type { Worktree } from "./worktree.ts";
|
|
15
|
+
|
|
16
|
+
const passed = (what: string): Judged => ({ verdict: "PASS", findings: [], method: `check (${what})` });
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Test first: the author writes a test for the task that fails, then makes it pass. A round whose
|
|
20
|
+
* new test passes before the change is a fail, and so is one that does not finish (it timed out or
|
|
21
|
+
* was cancelled: no failure was seen); the next round writes the test again. At most
|
|
22
|
+
* `max` rounds (three by default).
|
|
23
|
+
*/
|
|
24
|
+
export async function tddLoop(
|
|
25
|
+
key: string,
|
|
26
|
+
opts: { task: string; wt: Worktree; author: Seat; test: Check; max?: number },
|
|
27
|
+
): Promise<Attempt<{ sha: string; redSha: string; check: ArtifactRef }>> {
|
|
28
|
+
checkKey(key, "the key of a test-first loop");
|
|
29
|
+
let redSha = "";
|
|
30
|
+
const outcome = new Map<number, Judged>();
|
|
31
|
+
return revise(key, {
|
|
32
|
+
...(opts.max === undefined ? {} : { max: opts.max }),
|
|
33
|
+
write: async (feedback, round) => {
|
|
34
|
+
const notes = feedback === null ? "" : `\n\nThe last round did not pass:\n${findingsText(feedback.findings)}`;
|
|
35
|
+
if (redSha === "") {
|
|
36
|
+
const red = await commitBy(opts.author, opts.wt, `Write a test for this task that fails now, before any change for it, and nothing else:\n${opts.task}${notes}`);
|
|
37
|
+
const before = await runCheck("red", opts.test);
|
|
38
|
+
if (before.exit === null) {
|
|
39
|
+
// A test that timed out or was cancelled has not failed: nothing was seen.
|
|
40
|
+
outcome.set(round, { verdict: "FAIL", findings: [{ title: "the new test did not finish", evidence: before.summary, severity: "blocks" }], method: "check (red)" });
|
|
41
|
+
return { sha: red.sha, redSha: "", check: before.artifact };
|
|
42
|
+
}
|
|
43
|
+
if (before.ok) {
|
|
44
|
+
outcome.set(round, { verdict: "FAIL", findings: [{ title: "the new test passes before the change", evidence: before.summary, severity: "blocks" }], method: "check (red)" });
|
|
45
|
+
return { sha: red.sha, redSha: "", check: before.artifact };
|
|
46
|
+
}
|
|
47
|
+
const redChange = red.dirty ? uncommitted() : await changesAfterCheck(opts.wt, red.sha);
|
|
48
|
+
if (redChange !== undefined) {
|
|
49
|
+
outcome.set(round, redChange);
|
|
50
|
+
return { sha: red.sha, redSha: "", check: before.artifact };
|
|
51
|
+
}
|
|
52
|
+
redSha = red.sha;
|
|
53
|
+
}
|
|
54
|
+
const green = await commitBy(opts.author, opts.wt, `Make the failing test of the commit ${redSha} pass, for this task:\n${opts.task}${notes}`);
|
|
55
|
+
const after = await runCheck("green", opts.test);
|
|
56
|
+
const greenFailure = green.dirty ? uncommitted() : after.ok ? await changesAfterCheck(opts.wt, green.sha) : checkFailed("the test", after.summary);
|
|
57
|
+
outcome.set(round, greenFailure ?? passed("green"));
|
|
58
|
+
return { sha: green.sha, redSha, check: after.artifact };
|
|
59
|
+
},
|
|
60
|
+
judge: async (_value, round) => outcome.get(round) as Judged,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** A hypothesis about a cause: what it claims, how it stands and what shows it. */
|
|
65
|
+
export type Hypothesis = { claim: string; status: "confirmed" | "refuted" | "open"; evidence: string };
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Find the cause of a symptom: each round runs the repro and the investigator says how the
|
|
69
|
+
* hypotheses stand, adding new ones. It stops when one is confirmed, or after `max` rounds (three by
|
|
70
|
+
* default) with no cause.
|
|
71
|
+
*/
|
|
72
|
+
export async function diagnoseLoop(
|
|
73
|
+
key: string,
|
|
74
|
+
opts: { symptom: string; wt: Worktree; repro: Check; investigator: Seat; max?: number },
|
|
75
|
+
): Promise<{ cause: string | null; hypotheses: Hypothesis[] }> {
|
|
76
|
+
checkKey(key, "the key of a diagnosis");
|
|
77
|
+
const { z } = script();
|
|
78
|
+
const answer = z.object({
|
|
79
|
+
hypotheses: z.array(z.object({ claim: z.string(), status: z.enum(["confirmed", "refuted", "open"]), evidence: z.string() })),
|
|
80
|
+
});
|
|
81
|
+
const found = await loop(key, {
|
|
82
|
+
init: { hypotheses: [] as Hypothesis[], cause: null as string | null },
|
|
83
|
+
max: opts.max ?? 3,
|
|
84
|
+
done: (state) => state.cause !== null,
|
|
85
|
+
step: async (state) => {
|
|
86
|
+
const repro = await runCheck("repro", opts.repro);
|
|
87
|
+
const known = state.hypotheses.map((one) => `- [${one.status}] ${one.claim}: ${one.evidence}`).join("\n") || "(none yet)";
|
|
88
|
+
const told = await askSeat(
|
|
89
|
+
seatIn(opts.investigator, opts.wt.path),
|
|
90
|
+
`Find the cause of this symptom: ${opts.symptom}\n\nThe repro ${repro.ok ? "passed" : "failed"}:\n${repro.summary}\n\nThe hypotheses so far:\n${known}\n\n` +
|
|
91
|
+
"Investigate in this directory without changing it. Answer with every hypothesis, old and new, each confirmed, refuted or still open, with its evidence.",
|
|
92
|
+
answer,
|
|
93
|
+
);
|
|
94
|
+
return { hypotheses: told.hypotheses, cause: told.hypotheses.find((one) => one.status === "confirmed")?.claim ?? null };
|
|
95
|
+
},
|
|
96
|
+
});
|
|
97
|
+
return found.state;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Fix a bug a repro shows: the repro must fail first; then the author fixes it, round after round,
|
|
102
|
+
* until the repro passes or `max` rounds (three by default) are spent. A repro that passes before any
|
|
103
|
+
* change is no bug to fix, and one that does not finish (it timed out or was cancelled) shows none:
|
|
104
|
+
* the answer is `INCONCLUSIVE` with no commit.
|
|
105
|
+
*/
|
|
106
|
+
export async function debugLoop(
|
|
107
|
+
key: string,
|
|
108
|
+
opts: { symptom: string; wt: Worktree; repro: Check; author: Seat; max?: number },
|
|
109
|
+
): Promise<{ cause: string; sha: string; before: ArtifactRef; after: ArtifactRef; judged: Judged }> {
|
|
110
|
+
checkKey(key, "the key of a debug loop");
|
|
111
|
+
const { z } = script();
|
|
112
|
+
return script().scope(key, undefined, async () => {
|
|
113
|
+
const before = await runCheck("before", opts.repro);
|
|
114
|
+
if (before.ok || before.exit === null) {
|
|
115
|
+
// A repro that passes shows no bug; one that did not finish (timed out, cancelled) shows nothing.
|
|
116
|
+
const title = before.ok ? "the repro passes before any change" : "the repro did not finish";
|
|
117
|
+
return {
|
|
118
|
+
cause: "",
|
|
119
|
+
sha: "",
|
|
120
|
+
before: before.artifact,
|
|
121
|
+
after: before.artifact,
|
|
122
|
+
judged: { verdict: "INCONCLUSIVE" as const, findings: [{ title, evidence: before.summary, severity: before.ok ? ("note" as const) : ("blocks" as const) }], method: "check (before)" },
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
const results = new Map<number, { judged: Judged; artifact: ArtifactRef }>();
|
|
126
|
+
const attempt = await revise("fix", {
|
|
127
|
+
...(opts.max === undefined ? {} : { max: opts.max }),
|
|
128
|
+
write: async (feedback, round) => {
|
|
129
|
+
const fixed = await askSeat(
|
|
130
|
+
seatIn(opts.author, opts.wt.path),
|
|
131
|
+
`Fix this bug: ${opts.symptom}\n\nThe repro fails:\n${(feedback === null ? before.summary : findingsText(feedback.findings))}\n\n` +
|
|
132
|
+
`Work in ${opts.wt.path} on the branch ${opts.wt.branch}. Commit the fix; answer with the cause you found.`,
|
|
133
|
+
z.object({ cause: z.string() }),
|
|
134
|
+
);
|
|
135
|
+
const { sha, dirty } = await stateOf(opts.wt);
|
|
136
|
+
const after = await runCheck("after", opts.repro);
|
|
137
|
+
const failure = dirty ? uncommitted() : after.ok ? await changesAfterCheck(opts.wt, sha) : checkFailed("the repro", after.summary);
|
|
138
|
+
results.set(round, { judged: failure ?? passed("repro"), artifact: after.artifact });
|
|
139
|
+
return { sha, cause: fixed.cause };
|
|
140
|
+
},
|
|
141
|
+
judge: async (_value, round) => results.get(round)!.judged,
|
|
142
|
+
});
|
|
143
|
+
return { cause: attempt.value.cause, sha: attempt.value.sha, before: before.artifact, after: (results.get(attempt.rounds) as { artifact: ArtifactRef }).artifact, judged: attempt.judged };
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** The coverage a measure reports: the `coverage` of its report (a JSON report, or lcov totals). */
|
|
148
|
+
export function coverageOf(report: unknown): number | undefined {
|
|
149
|
+
if (typeof report !== "object" || report === null) return undefined;
|
|
150
|
+
const value = (report as { coverage?: unknown }).coverage;
|
|
151
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** A coverage measure that gave no coverage: it failed, or printed no report with one. */
|
|
155
|
+
export class MeasureFailed extends Error {
|
|
156
|
+
override name = "MeasureFailed";
|
|
157
|
+
constructor(
|
|
158
|
+
message: string,
|
|
159
|
+
readonly artifact: ArtifactRef,
|
|
160
|
+
) {
|
|
161
|
+
super(message);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Raise coverage to `threshold`: the measure reports it (`report: "json"` by default, read as its
|
|
167
|
+
* `coverage`), and the author adds tests until it reaches the threshold or `max` rounds (three by
|
|
168
|
+
* default) are spent. A measure that fails gives no coverage, whatever it printed: before the first
|
|
169
|
+
* round that is a `MeasureFailed`; after a round it is sent to the author as the reason the round did
|
|
170
|
+
* not count, and the answer carries it as `failure` when it is the last word.
|
|
171
|
+
*/
|
|
172
|
+
export async function coverageLoop(
|
|
173
|
+
key: string,
|
|
174
|
+
opts: { wt: Worktree; measure: Check; threshold: number; author: Seat; max?: number },
|
|
175
|
+
): Promise<{ before: number; after: number; sha: string; reached: boolean; failure?: { reason: string; artifact: ArtifactRef } }> {
|
|
176
|
+
checkKey(key, "the key of a coverage loop");
|
|
177
|
+
const measure: Check = { ...opts.measure, report: opts.measure.report ?? "json" };
|
|
178
|
+
const read = async (label: string, sha?: string): Promise<{ coverage: number } | { failure: { reason: string; artifact: ArtifactRef } }> => {
|
|
179
|
+
const measured = await runCheck(label, measure);
|
|
180
|
+
if (!measured.ok) return { failure: { reason: `the coverage measure did not pass: ${measured.summary}`, artifact: measured.artifact } };
|
|
181
|
+
const coverage = coverageOf(measured.report);
|
|
182
|
+
if (coverage === undefined) return { failure: { reason: `the coverage measure printed no report with a coverage: ${measured.summary}`, artifact: measured.artifact } };
|
|
183
|
+
if (sha !== undefined) {
|
|
184
|
+
const change = await changesAfterCheck(opts.wt, sha);
|
|
185
|
+
if (change !== undefined) return { failure: { reason: findingsText(change.findings), artifact: measured.artifact } };
|
|
186
|
+
}
|
|
187
|
+
return { coverage };
|
|
188
|
+
};
|
|
189
|
+
type State = { coverage: number; sha: string; clean: boolean; failure?: { reason: string; artifact: ArtifactRef } };
|
|
190
|
+
return script().scope(key, undefined, async () => {
|
|
191
|
+
const first = await read("before");
|
|
192
|
+
if ("failure" in first) throw new MeasureFailed(first.failure.reason, first.failure.artifact);
|
|
193
|
+
const before = first.coverage;
|
|
194
|
+
const raised = await loop<State>("raise", {
|
|
195
|
+
init: { coverage: before, sha: "", clean: true },
|
|
196
|
+
max: opts.max ?? 3,
|
|
197
|
+
done: (state) => state.failure === undefined && state.clean && state.coverage >= opts.threshold,
|
|
198
|
+
step: async (state) => {
|
|
199
|
+
const added = await commitBy(
|
|
200
|
+
opts.author,
|
|
201
|
+
opts.wt,
|
|
202
|
+
`Add tests that raise the coverage from ${String(state.coverage)} to at least ${String(opts.threshold)}. Change no code under test.` +
|
|
203
|
+
(state.clean ? "" : "\n\nThe last round left changes uncommitted: commit them.") +
|
|
204
|
+
(state.failure === undefined ? "" : `\n\nThe last round did not count:\n${state.failure.reason}`),
|
|
205
|
+
);
|
|
206
|
+
const measured = await read("measure", added.dirty ? undefined : added.sha);
|
|
207
|
+
return "failure" in measured
|
|
208
|
+
? { coverage: state.coverage, sha: added.sha, clean: !added.dirty, failure: measured.failure }
|
|
209
|
+
: { coverage: measured.coverage, sha: added.sha, clean: !added.dirty };
|
|
210
|
+
},
|
|
211
|
+
});
|
|
212
|
+
const { coverage, sha, clean, failure } = raised.state;
|
|
213
|
+
return { before, after: coverage, sha, reached: failure === undefined && clean && coverage >= opts.threshold, ...(failure === undefined ? {} : { failure }) };
|
|
214
|
+
});
|
|
215
|
+
}
|
package/src/review.ts
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Review in a worktree: reviewers judge the changes against a base and an author fixes what they
|
|
3
|
+
* found, round after round; or several reviewers each look through lenses and their findings are
|
|
4
|
+
* gathered.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { FindingSchema, askSeat, checkKey, map, revise, script, seatKeys, verify } from "@niwa-labs/flow";
|
|
8
|
+
import type { Attempt, Finding, Judged, Seat } from "@niwa-labs/flow";
|
|
9
|
+
import { runCheck } from "@niwa-labs/proc/flow";
|
|
10
|
+
import type { ArtifactRef, Check } from "@niwa-labs/proc/flow";
|
|
11
|
+
|
|
12
|
+
import { seatIn, stateOf } from "./worktree.ts";
|
|
13
|
+
import type { Worktree } from "./worktree.ts";
|
|
14
|
+
|
|
15
|
+
/** What reviewers are asked to judge: the changes of a worktree's branch against a base, at a commit. */
|
|
16
|
+
export function reviewSubject(wt: Worktree, base: string, sha: string): string {
|
|
17
|
+
return `The changes on the branch ${wt.branch} in ${wt.path}, from ${base} to the commit ${sha}: read them with \`git diff ${base}...${sha}\` there.`;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** The findings of a judgement as a list an author can work through. */
|
|
21
|
+
export function findingsText(findings: readonly Finding[]): string {
|
|
22
|
+
return findings.length === 0 ? "(none listed)" : findings.map((finding) => `- [${finding.severity}] ${finding.title}: ${finding.evidence}`).join("\n");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Ask an author to change the worktree and commit. What comes back is read from the worktree after
|
|
27
|
+
* the author is done, not from the author's word: the commit its branch is at, and whether changes
|
|
28
|
+
* are left out of it (`dirty`). A check of the worktree applies to that commit only when nothing is.
|
|
29
|
+
*/
|
|
30
|
+
export async function commitBy(author: Seat, wt: Worktree, prompt: string): Promise<{ sha: string; dirty: boolean }> {
|
|
31
|
+
const { z } = script();
|
|
32
|
+
await askSeat(seatIn(author, wt.path), `${prompt}\n\nWork in ${wt.path} on the branch ${wt.branch}. Commit what you change, and answer with a short account of what you did.`, z.object({ summary: z.string().optional() }));
|
|
33
|
+
return stateOf(wt);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Changes left out of the commit a check was run for, as a judgement that blocks. */
|
|
37
|
+
export function uncommitted(): Judged {
|
|
38
|
+
return {
|
|
39
|
+
verdict: "FAIL",
|
|
40
|
+
findings: [{ title: "the work was left uncommitted", evidence: "the worktree holds changes (or new files) that are not in the commit it is at: commit them, or remove what is not part of the work", severity: "blocks" }],
|
|
41
|
+
method: "check (worktree)",
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A check can count for a commit only if the worktree is still clean and at that commit. */
|
|
46
|
+
export async function changesAfterCheck(wt: Worktree, sha: string): Promise<Judged | undefined> {
|
|
47
|
+
const after = await stateOf(wt);
|
|
48
|
+
if (after.dirty) return uncommitted();
|
|
49
|
+
if (after.sha !== sha) return checkFailed("the worktree", `the check started at ${sha} but the worktree is now at ${after.sha}`);
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** A check that did not pass, as a judgement that blocks. */
|
|
54
|
+
export function checkFailed(what: string, summary: string): Judged {
|
|
55
|
+
return { verdict: "FAIL", findings: [{ title: `${what} did not pass`, evidence: summary, severity: "blocks" }], method: `check (${what})` };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** What a review and fix is given, apart from its check. */
|
|
59
|
+
type ReviewFixOptions = { wt: Worktree; base: string; author: Seat; reviewers: readonly Seat[]; focus?: readonly string[]; max?: number };
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Review the changes of a worktree against `base` and have `author` fix what blocks, for at most
|
|
63
|
+
* `max` reviews (three by default). The first review is of the worktree as it is. With `test`, the
|
|
64
|
+
* check runs before each review and a failing check is sent back to the author unreviewed. So is a
|
|
65
|
+
* worktree that holds changes outside the commit (the commit is read from the worktree, not taken
|
|
66
|
+
* from the author's word), which includes files a check leaves behind: a check has to leave the
|
|
67
|
+
* worktree as it found it, or have its output ignored by git. The answer carries the artifact of
|
|
68
|
+
* the check when `test` is given, and only the commit when it is not.
|
|
69
|
+
*/
|
|
70
|
+
export function reviewFix(key: string, opts: ReviewFixOptions & { test: Check }): Promise<Attempt<{ sha: string; check: ArtifactRef }>>;
|
|
71
|
+
export function reviewFix(key: string, opts: ReviewFixOptions & { test?: undefined }): Promise<Attempt<{ sha: string }>>;
|
|
72
|
+
export function reviewFix(key: string, opts: ReviewFixOptions & { test?: Check }): Promise<Attempt<{ sha: string; check?: ArtifactRef }>>;
|
|
73
|
+
export async function reviewFix(key: string, opts: ReviewFixOptions & { test?: Check }): Promise<Attempt<{ sha: string; check?: ArtifactRef }>> {
|
|
74
|
+
checkKey(key, "the key of a review");
|
|
75
|
+
if (opts.reviewers.length === 0) throw new TypeError(`reviewFix "${key}": give at least one reviewer`);
|
|
76
|
+
const blocked = new Map<number, Judged>();
|
|
77
|
+
return revise(key, {
|
|
78
|
+
...(opts.max === undefined ? {} : { max: opts.max }),
|
|
79
|
+
write: async (feedback, round) => {
|
|
80
|
+
const state = feedback === null ? await stateOf(opts.wt, "head") : await commitBy(opts.author, opts.wt, `Fix what the review found in the changes against ${opts.base}:\n${findingsText(feedback.findings)}`);
|
|
81
|
+
const check = opts.test === undefined ? undefined : await runCheck("test", opts.test);
|
|
82
|
+
if (state.dirty) blocked.set(round, uncommitted());
|
|
83
|
+
else if (check !== undefined) {
|
|
84
|
+
const failure = check.ok ? await changesAfterCheck(opts.wt, state.sha) : checkFailed("the test", check.summary);
|
|
85
|
+
if (failure !== undefined) blocked.set(round, failure);
|
|
86
|
+
}
|
|
87
|
+
return { sha: state.sha, ...(check === undefined ? {} : { check: check.artifact }) };
|
|
88
|
+
},
|
|
89
|
+
judge: async ({ sha }, round) =>
|
|
90
|
+
blocked.get(round) ??
|
|
91
|
+
verify("review", {
|
|
92
|
+
subject: reviewSubject(opts.wt, opts.base, sha),
|
|
93
|
+
by: opts.reviewers.map((reviewer) => seatIn(reviewer, opts.wt.path)),
|
|
94
|
+
...(opts.focus === undefined ? {} : { lenses: opts.focus }),
|
|
95
|
+
}),
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Reviewers look at the changes of a worktree through each lens and list what they find. The
|
|
101
|
+
* findings are gathered once each (by title and severity), and by lens. A reviewer that could not
|
|
102
|
+
* look through a lens is in `failures` with its error: no findings from it is not a clean review, so
|
|
103
|
+
* a panel is complete only when `failures` is empty.
|
|
104
|
+
*/
|
|
105
|
+
export async function reviewPanel(
|
|
106
|
+
key: string,
|
|
107
|
+
opts: { wt: Worktree; base: string; reviewers: readonly Seat[]; lenses: readonly string[] },
|
|
108
|
+
): Promise<{ findings: Finding[]; byLens: Record<string, Finding[]>; failures: { reviewer: string; lens: string; error: string }[] }> {
|
|
109
|
+
checkKey(key, "the key of a review");
|
|
110
|
+
const keys = seatKeys(opts.reviewers, "the key of a reviewer");
|
|
111
|
+
const { z } = script();
|
|
112
|
+
const legs = opts.reviewers.flatMap((seat, at) => opts.lenses.map((lens, index) => ({ seat, reviewer: keys[at] as string, lens, key: `${keys[at] as string}-l${String(index + 1)}` })));
|
|
113
|
+
const answers = await script().scope(key, undefined, async () => {
|
|
114
|
+
const { sha } = await stateOf(opts.wt, "head");
|
|
115
|
+
return map("legs", legs, { key: (leg) => leg.key }, (leg) =>
|
|
116
|
+
askSeat(
|
|
117
|
+
seatIn(leg.seat, opts.wt.path),
|
|
118
|
+
`Review the work below through this lens only: ${leg.lens}. List what you find, each with its evidence; a finding that must be fixed blocks.\n\n${reviewSubject(opts.wt, opts.base, sha)}`,
|
|
119
|
+
z.object({ findings: z.array(FindingSchema()) }),
|
|
120
|
+
),
|
|
121
|
+
);
|
|
122
|
+
});
|
|
123
|
+
const byLens: Record<string, Finding[]> = Object.fromEntries(opts.lenses.map((lens) => [lens, [] as Finding[]]));
|
|
124
|
+
const findings: Finding[] = [];
|
|
125
|
+
const seen = new Set<string>();
|
|
126
|
+
const failures: { reviewer: string; lens: string; error: string }[] = [];
|
|
127
|
+
answers.forEach((answer, index) => {
|
|
128
|
+
const leg = legs[index] as { reviewer: string; lens: string };
|
|
129
|
+
if (!answer.ok) {
|
|
130
|
+
failures.push({ reviewer: leg.reviewer, lens: leg.lens, error: answer.error.message });
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
for (const finding of answer.value.findings) {
|
|
134
|
+
(byLens[leg.lens] as Finding[]).push(finding);
|
|
135
|
+
const id = `${finding.severity}\0${finding.title}`;
|
|
136
|
+
if (!seen.has(id)) findings.push(finding);
|
|
137
|
+
seen.add(id);
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
return { findings, byLens, failures };
|
|
141
|
+
}
|