@turri-dev/cli 0.1.2 → 0.1.3-next.04c4f71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/chat-skills.js +1 -1
- package/dist/cli-options.js +82 -0
- package/dist/config.js +23 -1
- package/dist/conform-cli.js +13 -5
- package/dist/environment.js +19 -0
- package/dist/extension.js +131 -50
- package/dist/identity.js +1 -1
- package/dist/init-config.js +3 -3
- package/dist/loop-cli.js +156 -28
- package/dist/loop-tools.js +1 -1
- package/dist/pi-launch.js +13 -4
- package/dist/pi.js +14 -2
- package/dist/picture-row.js +21 -0
- package/dist/plain/errors.js +1 -1
- package/dist/plain/files.js +1 -1
- package/dist/plain/liquid.js +2 -2
- package/dist/regenerate/activity.js +19 -0
- package/dist/regenerate/confine.js +1 -0
- package/dist/regenerate/environment-error.js +46 -9
- package/dist/regenerate/loop.js +56 -2
- package/dist/regenerate/panel-dock.js +86 -0
- package/dist/regenerate/prompt.js +1 -1
- package/dist/regenerate/release.js +8 -6
- package/dist/regenerate/side-panel.js +14 -4
- package/dist/regenerate/test-scripts.js +23 -8
- package/dist/regenerate-cli.js +24 -29
- package/dist/run-report.js +289 -0
- package/dist/setup.js +35 -16
- package/dist/spec-diff-cli.js +17 -6
- package/docs/config.md +22 -11
- package/docs/development.md +7 -0
- package/docs/headless.md +170 -16
- package/docs/installation.md +1 -1
- package/docs/parser.md +1 -1
- package/docs/regenerating.md +19 -7
- package/docs/using.md +16 -5
- package/genome/skills/implement-conformance-testing-script/SKILL.md +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { HStack } from "@earendil-works/pi-tui";
|
|
2
|
+
/**
|
|
3
|
+
* The side panel beside the chat rather than over it: the chat drawn on the left in what the panel leaves it, the panel
|
|
4
|
+
* laid over the columns on the right.
|
|
5
|
+
*
|
|
6
|
+
* pi draws its screen one of two ways, and the chat is narrowed in each as it is drawn: full screen (pi's default), a
|
|
7
|
+
* layout - its root put in a row beside a filler as wide as the panel; the regular screen, the lines its render gives -
|
|
8
|
+
* asked for in fewer columns.
|
|
9
|
+
*/
|
|
10
|
+
/** Narrower than this, the chat and the panel have no room side by side: the panel takes the whole screen. */
|
|
11
|
+
const SHARED_FROM = 100;
|
|
12
|
+
/** The columns the panel takes beside the chat: half the screen, the rule between them in the middle - 0 when the terminal is too narrow to share. */
|
|
13
|
+
export function dockedWidth(columns) {
|
|
14
|
+
return columns < SHARED_FROM ? 0 : Math.floor(columns / 2);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The chat narrowed by `panelWidth()` columns, read at each frame - 0 gives it the whole width. Returns what gives it the
|
|
18
|
+
* whole width back for good.
|
|
19
|
+
*/
|
|
20
|
+
export function narrowChat(tui, panelWidth) {
|
|
21
|
+
const screen = tui;
|
|
22
|
+
if (typeof screen.setLayoutRoot === "function" && screen.layoutRoot !== undefined) {
|
|
23
|
+
const root = screen.layoutRoot;
|
|
24
|
+
// A line as wide as the panel: the row gives the filler what it measures, the chat the rest.
|
|
25
|
+
const filler = { render: () => [" ".repeat(panelWidth())], invalidate() { } };
|
|
26
|
+
screen.setLayoutRoot(new HStack([{ component: root, basis: 0, grow: 1 }, { component: filler }]));
|
|
27
|
+
return () => screen.setLayoutRoot(root);
|
|
28
|
+
}
|
|
29
|
+
// Read from the prototype: the screen's own render, not one laid over it.
|
|
30
|
+
const render = Object.getPrototypeOf(tui).render;
|
|
31
|
+
screen.render = function (width) {
|
|
32
|
+
return render.call(this, Math.max(1, width - panelWidth()));
|
|
33
|
+
};
|
|
34
|
+
return () => {
|
|
35
|
+
screen.render = render;
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* A press of a mouse button, as pi's full screen takes it - `side` told whether it was in the chat, left of `chatWidth()`
|
|
40
|
+
* columns, or in the panel - before the press goes on as it would have. Motion and the wheel are not presses. pi's
|
|
41
|
+
* regular screen reads no mouse: nothing is done there. Returns what takes the watch off.
|
|
42
|
+
*/
|
|
43
|
+
export function clickToFocus(tui, chatWidth, side) {
|
|
44
|
+
const screen = tui;
|
|
45
|
+
const handle = screen.handleMouseEvent;
|
|
46
|
+
if (typeof handle !== "function")
|
|
47
|
+
return () => { };
|
|
48
|
+
screen.handleMouseEvent = function (raw) {
|
|
49
|
+
// 32: motion; 64: the wheel.
|
|
50
|
+
if (raw.release !== true && (raw.button & (32 | 64)) === 0)
|
|
51
|
+
side(raw.x < chatWidth() ? "chat" : "panel");
|
|
52
|
+
return handle.call(this, raw);
|
|
53
|
+
};
|
|
54
|
+
return () => {
|
|
55
|
+
screen.handleMouseEvent = handle;
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* A line drawn faint - the side that does not have the keys, in the shade. Faint (SGR 2) holds through a line's colours;
|
|
60
|
+
* only a reset - SGR 0, or 22 - ends it, so it is set again after each one. A line holding an image is left as it is.
|
|
61
|
+
*/
|
|
62
|
+
export function shade(line) {
|
|
63
|
+
// Kitty's image escape, or iTerm2's.
|
|
64
|
+
if (line.includes("\x1b_G") || line.includes("\x1b]1337;File="))
|
|
65
|
+
return line;
|
|
66
|
+
const again = line.replace(/\x1b\[([\d;]*)m/g, (sgr, params) => (params.split(";").some((param) => param === "" || param === "0" || param === "22") ? `${sgr}${FAINT}` : sgr));
|
|
67
|
+
return `${FAINT}${again}${NOT_FAINT}`;
|
|
68
|
+
}
|
|
69
|
+
const FAINT = "\x1b[2m";
|
|
70
|
+
const NOT_FAINT = "\x1b[22m";
|
|
71
|
+
/**
|
|
72
|
+
* The chat in the shade while `shaded()`: each of its rows drawn faint as the screen lays the overlays over it - the
|
|
73
|
+
* docked panel the last of them, which covers its own columns. Returns what takes the shade off.
|
|
74
|
+
*/
|
|
75
|
+
export function shadeChat(tui, shaded) {
|
|
76
|
+
const screen = tui;
|
|
77
|
+
const composite = screen.compositeLineAt;
|
|
78
|
+
if (typeof composite !== "function")
|
|
79
|
+
return () => { };
|
|
80
|
+
screen.compositeLineAt = function (base, overlay, col, width, total) {
|
|
81
|
+
return composite.call(this, shaded() ? shade(base) : base, overlay, col, width, total);
|
|
82
|
+
};
|
|
83
|
+
return () => {
|
|
84
|
+
screen.compositeLineAt = composite;
|
|
85
|
+
};
|
|
86
|
+
}
|
|
@@ -24,7 +24,7 @@ function alreadyImplemented(moduleName, own, required) {
|
|
|
24
24
|
* one another, the current module marked - and each tagged with its module and its id there. A codebase holds the
|
|
25
25
|
* functionalities of every module it is built on, and each module numbers its own from 1, so
|
|
26
26
|
* "functionality 1" alone is ambiguous: the first listed may be base's 1 and the next top's 3.
|
|
27
|
-
* (
|
|
27
|
+
* (the reference implementation's implemented functional requirements prompt section, as tags.)
|
|
28
28
|
*/
|
|
29
29
|
export function functionalitiesByModule(entries, currentModule, status) {
|
|
30
30
|
// In the order given: the bottom of the chain first. A regeneration in a later module's
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { spawn, spawnSync } from "node:child_process";
|
|
2
|
-
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync } from "node:fs";
|
|
2
|
+
import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, rmSync } from "node:fs";
|
|
3
3
|
import { connect } from "node:net";
|
|
4
4
|
import { dirname, join, relative } from "node:path";
|
|
5
5
|
import { HISTORY_DIR } from "./lineage.js";
|
|
@@ -38,15 +38,17 @@ export function deployTarget(plainFile, inModule) {
|
|
|
38
38
|
throw new Error(`Nothing is built yet for ${tip.moduleName}${candidates.length > 1 ? " or the modules it requires" : ""}: there is no code to deploy. Render it first.`);
|
|
39
39
|
}
|
|
40
40
|
/**
|
|
41
|
-
* Export the module's code at its last commit into `dist/<module
|
|
42
|
-
* harness's own files (what the code is built from, the agents' notes) left out.
|
|
41
|
+
* Export the module's code at its last commit into `dist/<module>` - or `into`, `turri regenerate --build-dir` -
|
|
42
|
+
* replacing whatever was there, the harness's own files (what the code is built from, the agents' notes) left out.
|
|
43
43
|
*/
|
|
44
|
-
export function exportModule(plainFile, inModule) {
|
|
44
|
+
export function exportModule(plainFile, inModule, into) {
|
|
45
45
|
const { entry, commit, instead } = deployTarget(plainFile, inModule);
|
|
46
46
|
const moduleDir = dirname(entry.buildDir);
|
|
47
|
-
const dir = distDirFor(projectDirOf(entry.plainFile), entry.moduleName);
|
|
48
|
-
|
|
47
|
+
const dir = into ?? distDirFor(projectDirOf(entry.plainFile), entry.moduleName);
|
|
48
|
+
// Emptied, not removed: the folder may be a mount point - a container's volume - that cannot be removed.
|
|
49
49
|
mkdirSync(dir, { recursive: true });
|
|
50
|
+
for (const entry of readdirSync(dir))
|
|
51
|
+
rmSync(join(dir, entry), { recursive: true, force: true });
|
|
50
52
|
const exported = spawnSync("bash", ["-o", "pipefail", "-c", 'git --git-dir "$1" archive --format=tar "$2" | tar -x -f - -C "$3"', "export", join(moduleDir, HISTORY_DIR), `${commit}:code`, dir], { encoding: "utf8" });
|
|
51
53
|
if (exported.status !== 0)
|
|
52
54
|
throw new Error(`Could not export ${entry.moduleName}'s code at ${commit.slice(0, 8)}: ${(exported.stderr ?? "").trim()}`);
|
|
@@ -12,6 +12,12 @@ export function createSidePanel(options) {
|
|
|
12
12
|
let drawnTree = { rows: [], rowStarts: [], lineCount: 0 };
|
|
13
13
|
let drawnMap = { rows: [], rowStarts: [], lineCount: 0 };
|
|
14
14
|
let pages = [];
|
|
15
|
+
/** The keys back to the chat - and `s`, from over the screen to beside the chat and back. */
|
|
16
|
+
const backKeys = () => {
|
|
17
|
+
if (options.dock === undefined)
|
|
18
|
+
return [`esc or ${options.key} back to the chat`];
|
|
19
|
+
return options.dock.docked() ? ["s full screen", "esc or ⌥⌘← to the chat", `${options.key} close`] : ["s beside the chat", `esc or ${options.key} back to the chat`];
|
|
20
|
+
};
|
|
15
21
|
const waitingNote = () =>
|
|
16
22
|
// A question asked while it showed waits in the chat, unanswered: the person is told it is there.
|
|
17
23
|
options.waiting?.() === true ? ` ${style.warning?.("your question is waiting") ?? style.accent("your question is waiting")} - ${options.key} or esc to answer it, as you left it` : "";
|
|
@@ -59,7 +65,7 @@ export function createSidePanel(options) {
|
|
|
59
65
|
/** The rule under the top. */
|
|
60
66
|
const rule = (width) => style.muted("─".repeat(Math.max(1, width)));
|
|
61
67
|
const renderPulses = (width, rows) => {
|
|
62
|
-
const keys = ["↑↓ move", "enter open/close", "←→", "o/c all", "pgup pgdn scroll", ...(options.map !== undefined ? ["p regen"] : []),
|
|
68
|
+
const keys = ["↑↓ move", "enter open/close", "←→", "o/c all", "pgup pgdn scroll", ...(options.map !== undefined ? ["p regen"] : []), ...backKeys()];
|
|
63
69
|
const top = topBlock([`${style.heading("pulses")}${waitingNote()}`], keys, width);
|
|
64
70
|
// Under the top: the rule, and a line of space - as the map has.
|
|
65
71
|
const bodyHeight = Math.max(3, rows - top.length - 2);
|
|
@@ -77,7 +83,7 @@ export function createSidePanel(options) {
|
|
|
77
83
|
pages = options.map();
|
|
78
84
|
const now = Date.now();
|
|
79
85
|
if (pages.length === 0) {
|
|
80
|
-
const top = topBlock([`${style.heading("regen")}${waitingNote()}`], ["p pulses",
|
|
86
|
+
const top = topBlock([`${style.heading("regen")}${waitingNote()}`], ["p pulses", ...backKeys()], width);
|
|
81
87
|
lastBodyHeight = Math.max(3, rows - top.length - 1);
|
|
82
88
|
drawnMap = { rows: [], rowStarts: [], lineCount: 0 };
|
|
83
89
|
const body = [style.muted(" No .plain file to map: name one to regenerate, or set module: in config.turri.yaml."), ...new Array(Math.max(0, lastBodyHeight - 1)).fill("")];
|
|
@@ -87,7 +93,7 @@ export function createSidePanel(options) {
|
|
|
87
93
|
const tabbed = pages.length > 1;
|
|
88
94
|
const page = shownPage(pages, map);
|
|
89
95
|
const discuss = page.stuck && options.discuss !== undefined ? [`${DISCUSS_KEY} talk it through in the chat`] : [];
|
|
90
|
-
const keys = [...discuss, "↑↓ move", "enter open/close", ...(tabbed ? ["←→ module"] : []), "o/c all", "pgup pgdn", "p pulses",
|
|
96
|
+
const keys = [...discuss, "↑↓ move", "enter open/close", ...(tabbed ? ["←→ module"] : []), "o/c all", "pgup pgdn", "p pulses", ...backKeys()];
|
|
91
97
|
// The legend's box in the upper right corner, over what is under it; too narrow a panel has none.
|
|
92
98
|
const box = width >= 60 ? legendBox(legendLines(page, style)) : [];
|
|
93
99
|
const room = box.length > 0 ? width - visibleWidth(box[0]) - 1 : width;
|
|
@@ -210,7 +216,7 @@ export function createSidePanel(options) {
|
|
|
210
216
|
}
|
|
211
217
|
catch (error) {
|
|
212
218
|
// A frame that throws is never drawn - nor is anything after it: the screen would freeze. What failed is shown instead.
|
|
213
|
-
const top = topBlock([`${style.heading(map.view === "map" ? "regen" : "pulses")}${waitingNote()}`], [...(options.map !== undefined ? ["p the other view"] : []),
|
|
219
|
+
const top = topBlock([`${style.heading(map.view === "map" ? "regen" : "pulses")}${waitingNote()}`], [...(options.map !== undefined ? ["p the other view"] : []), ...backKeys()], width);
|
|
214
220
|
const lines = (error instanceof Error ? (error.stack ?? error.message) : String(error)).split("\n").slice(0, Math.max(1, rows - top.length - 1));
|
|
215
221
|
return [
|
|
216
222
|
...top,
|
|
@@ -225,6 +231,10 @@ export function createSidePanel(options) {
|
|
|
225
231
|
options.back();
|
|
226
232
|
return;
|
|
227
233
|
}
|
|
234
|
+
if (options.dock !== undefined && data === "s") {
|
|
235
|
+
options.dock.toggle();
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
228
238
|
if (options.map !== undefined && data === "p") {
|
|
229
239
|
if (map.view === "map")
|
|
230
240
|
toPulses();
|
|
@@ -609,19 +609,34 @@ function timedOutFailure(run, what) {
|
|
|
609
609
|
`the same ${limit} seconds, and a skipped or deleted test does not count as a fix.`,
|
|
610
610
|
].join("\n");
|
|
611
611
|
}
|
|
612
|
+
/** A command `timeout` could not start, or a shell could not find - on a line of its own. */
|
|
613
|
+
const COULD_NOT_RUN = /^(?:[^:]*: )?failed to run command [‘'`"]?([\w.+-]+)[’'`"]?: No such file or directory$/;
|
|
614
|
+
const NOT_FOUND = /^(?:[^:]*: )?(?:line \d+: )?([\w.+-]+): command not found$/;
|
|
612
615
|
/**
|
|
613
616
|
* What a failed run says is not installed where it ran - the command, when its output names it - or undefined when
|
|
614
|
-
* the run failed for another reason.
|
|
615
|
-
* or a shell or `timeout`
|
|
617
|
+
* the run failed for another reason. A missing command is the cause only on evidence that it ended the run: exit 69,
|
|
618
|
+
* the testing scripts' own way to say a toolchain is missing, or 127, what a shell or `timeout` gives for a command it
|
|
619
|
+
* could not find - else a command not found on the last line the run printed, nothing having run after it. Earlier in
|
|
620
|
+
* the output it is a warning, not the failure: a shell profile's, an optional tool probed for, a fallback taken - and a
|
|
621
|
+
* run that failed for its own reasons is the agent's to fix, not a stop for the person.
|
|
616
622
|
*/
|
|
617
623
|
export function missingToolchain(run) {
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
if (run.exitCode === 69 || run.exitCode === 127)
|
|
624
|
+
if (run.exitCode === 0)
|
|
625
|
+
return undefined;
|
|
626
|
+
const lines = run.output.split("\n").map((line) => line.trim()).filter((line) => line !== "");
|
|
627
|
+
const named = (line) => COULD_NOT_RUN.exec(line)?.[1] ?? NOT_FOUND.exec(line)?.[1];
|
|
628
|
+
if (run.exitCode === 69 || run.exitCode === 127) {
|
|
629
|
+
// The command is the last one named: the one the run stopped on.
|
|
630
|
+
for (let at = lines.length - 1; at >= 0; at -= 1) {
|
|
631
|
+
const command = named(lines[at]);
|
|
632
|
+
if (command !== undefined)
|
|
633
|
+
return { command };
|
|
634
|
+
}
|
|
623
635
|
return {};
|
|
624
|
-
|
|
636
|
+
}
|
|
637
|
+
const last = lines.at(-1);
|
|
638
|
+
const command = last !== undefined ? named(last) : undefined;
|
|
639
|
+
return command !== undefined ? { command } : undefined;
|
|
625
640
|
}
|
|
626
641
|
/** Where `command` is on this machine's PATH, or undefined when it is not there. */
|
|
627
642
|
export function onThisMachine(command) {
|
package/dist/regenerate-cli.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CONFIG_FILE, readProjectConfig } from "./config.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { parseArgs } from "./cli-options.js";
|
|
3
|
+
import { headlessOptions, runLoop } from "./loop-cli.js";
|
|
4
4
|
const USAGE = `usage: turri regenerate [file.plain] [frid ...] [options]
|
|
5
5
|
|
|
6
6
|
Regenerates the module's code from its spec as it is now, and everything it requires, bottom first, as
|
|
@@ -12,33 +12,27 @@ does in the chat with functionalities named: their code rewritten or started ove
|
|
|
12
12
|
|
|
13
13
|
The project's config.turri.yaml, in the current directory, gives the defaults: module (the file to
|
|
14
14
|
build when none is given), coding-model (the agents that build), harness-model (the planner), and
|
|
15
|
-
conformance-tests.
|
|
15
|
+
conformance-tests. It exits 0 only when every module is complete: regenerated from the spec as it is,
|
|
16
|
+
its conformance tests too when they are on.
|
|
16
17
|
|
|
17
18
|
--dry-run plan the work and print the prompt of the first job, and run nothing (turri glimpse)
|
|
18
|
-
--start-over start over, without asking, a codebase regenerated without a module it now requires
|
|
19
|
+
--start-over start over, without asking, a codebase regenerated without a module it now requires -
|
|
20
|
+
with no terminal to ask and without it, the run stops there
|
|
19
21
|
--from-scratch delete the module's code and conformance tests (its lineage keeps them) and regenerate all of it
|
|
20
|
-
--
|
|
21
|
-
--model <
|
|
22
|
+
--harness-model <p/id> config.turri.yaml's harness-model for this run: the planner, the conformance chooser, a diagnosis
|
|
23
|
+
--coding-model <p/id> its coding-model, --model too: the agents that build; p/id:level for how hard a model thinks
|
|
24
|
+
--judge-model <p/id> its judge-model: the judge and the test review
|
|
25
|
+
--review <true|false> its review: whether the code and its tests are reviewed once they pass
|
|
26
|
+
--conformance-tests <all|chosen|module|false>
|
|
27
|
+
its conformance-tests: which conformance tests are written and run
|
|
28
|
+
--build-dir <dir> as the run ends, however it ends, export the code there: the furthest-built codebase's
|
|
29
|
+
last commit, which holds every module it requires - the folder's contents are replaced
|
|
30
|
+
--report <file.json> write a report of the run there - outcome, usage, pulses, functional specs completed -
|
|
31
|
+
as it starts, as each pulse ends and as it exits (docs/headless.md)
|
|
22
32
|
|
|
23
33
|
PLAIN_TEMPLATE_DIRS colon-separated template directories`;
|
|
24
|
-
|
|
25
|
-
const
|
|
26
|
-
force: "to regenerate a functionality again, name it: turri regenerate <file.plain> <frid>",
|
|
27
|
-
only: "every module the spec requires is built, bottom first",
|
|
28
|
-
to: "the whole spec is built: stop the jobs to stop partway",
|
|
29
|
-
from: "every functionality not built is queued; built ones are skipped",
|
|
30
|
-
"conformance-root": "conformance tests are in plain_modules/<module>/tests",
|
|
31
|
-
"max-regression-rounds": "each conformance job runs the whole suite",
|
|
32
|
-
"try-it": "run turri release to try what is regenerated",
|
|
33
|
-
conformance: `set conformance-tests: in ${CONFIG_FILE}, or turri conform`,
|
|
34
|
-
"no-conformance": `set conformance-tests: false in ${CONFIG_FILE}`,
|
|
35
|
-
"template-dir": "set PLAIN_TEMPLATE_DIRS",
|
|
36
|
-
};
|
|
37
|
-
const { positional, options } = parseArgs(process.argv.slice(2), ["--only", "--force", "--conformance", "--no-conformance", "--try-it", "--dry-run", "--start-over", "--from-scratch"], USAGE);
|
|
38
|
-
for (const [name, instead] of Object.entries(GONE)) {
|
|
39
|
-
if (options[name] !== undefined)
|
|
40
|
-
console.log(c.dim(`--${name} is no longer an option: ${instead}.`));
|
|
41
|
-
}
|
|
34
|
+
const { positional, options } = parseArgs(process.argv.slice(2), ["--dry-run", "--start-over", "--from-scratch"], USAGE, ["--build-dir", "--report", "--conformance-tests", "--harness-model", "--judge-model", "--review"]);
|
|
35
|
+
const { report, buildDir } = headlessOptions(options, true);
|
|
42
36
|
const named = positional[0] ?? readProjectConfig(process.cwd()).module;
|
|
43
37
|
// Functional specs named after the file: regenerated again though their spec did not change.
|
|
44
38
|
const again = positional.slice(1);
|
|
@@ -53,15 +47,16 @@ if (options["from-scratch"] !== undefined && options["dry-run"] !== undefined) {
|
|
|
53
47
|
}
|
|
54
48
|
runLoop({
|
|
55
49
|
plainFile: named,
|
|
56
|
-
...(
|
|
57
|
-
...(
|
|
50
|
+
...(buildDir !== undefined ? { buildDir } : {}),
|
|
51
|
+
...(report !== undefined ? { report } : {}),
|
|
52
|
+
command: ["regenerate", ...process.argv.slice(2)],
|
|
58
53
|
dryRun: options["dry-run"] !== undefined,
|
|
59
54
|
startOver: options["start-over"] !== undefined,
|
|
60
55
|
ask: (loop) => again.length > 0
|
|
61
|
-
? loop.regenerateAgain(named, again
|
|
56
|
+
? loop.regenerateAgain(named, again)
|
|
62
57
|
: options["from-scratch"] !== undefined
|
|
63
|
-
? loop.fromScratch(named
|
|
64
|
-
: loop.bringUp(named
|
|
58
|
+
? loop.fromScratch(named)
|
|
59
|
+
: loop.bringUp(named),
|
|
65
60
|
}).catch((error) => {
|
|
66
61
|
console.error(error instanceof Error ? error.message : error);
|
|
67
62
|
process.exit(1);
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
import { mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, join, relative, resolve } from "node:path";
|
|
3
|
+
import { turriIdentity } from "./environment.js";
|
|
4
|
+
import { onModelEvent } from "./regenerate/activity.js";
|
|
5
|
+
/**
|
|
6
|
+
* A headless run's report: one JSON file, `turri regenerate --report <file>`, saying what the run did - how it ended,
|
|
7
|
+
* what it spent, which pulses landed, how many functional specs are completed - for a benchmark or CI to read rather
|
|
8
|
+
* than the log. See docs/headless.md for every field.
|
|
9
|
+
*
|
|
10
|
+
* Written when the run starts, again as each pulse ends, and last as the process exits - however it exits: done, a
|
|
11
|
+
* pulse failed, a crash, a signal. A process killed outright (SIGKILL) leaves the one written as the last pulse ended.
|
|
12
|
+
* Each write is atomic: the file is written beside it and renamed over it, so a reader never sees half of one.
|
|
13
|
+
*/
|
|
14
|
+
export const REPORT_SCHEMA_VERSION = 1;
|
|
15
|
+
const SIGNALS = { 129: "SIGHUP", 130: "SIGINT", 143: "SIGTERM" };
|
|
16
|
+
const emptyTotals = () => ({
|
|
17
|
+
input_tokens: 0,
|
|
18
|
+
output_tokens: 0,
|
|
19
|
+
cache_read_tokens: 0,
|
|
20
|
+
cache_write_tokens: 0,
|
|
21
|
+
cost_usd: 0,
|
|
22
|
+
llm_calls: 0,
|
|
23
|
+
agent_turns: 0,
|
|
24
|
+
compactions: 0,
|
|
25
|
+
model_seconds: 0,
|
|
26
|
+
});
|
|
27
|
+
function addUsage(totals, usage) {
|
|
28
|
+
if (usage === undefined)
|
|
29
|
+
return;
|
|
30
|
+
totals.input_tokens += usage.input;
|
|
31
|
+
totals.output_tokens += usage.output;
|
|
32
|
+
totals.cache_read_tokens += usage.cacheRead;
|
|
33
|
+
totals.cache_write_tokens += usage.cacheWrite;
|
|
34
|
+
totals.cost_usd += usage.cost;
|
|
35
|
+
}
|
|
36
|
+
/** An event's counts - a call, a turn, a compaction, the model's time - and, with `tokens`, its usage too. */
|
|
37
|
+
function addEvent(totals, event, tokens) {
|
|
38
|
+
if (event.kind === "call") {
|
|
39
|
+
totals.llm_calls += 1;
|
|
40
|
+
totals.model_seconds += (event.ms ?? 0) / 1000;
|
|
41
|
+
if (tokens)
|
|
42
|
+
addUsage(totals, event.usage);
|
|
43
|
+
}
|
|
44
|
+
else if (event.kind === "turn")
|
|
45
|
+
totals.agent_turns += 1;
|
|
46
|
+
else
|
|
47
|
+
totals.compactions += 1;
|
|
48
|
+
}
|
|
49
|
+
function sumTotals(a, b) {
|
|
50
|
+
const sum = emptyTotals();
|
|
51
|
+
for (const key of Object.keys(sum))
|
|
52
|
+
sum[key] = a[key] + b[key];
|
|
53
|
+
return sum;
|
|
54
|
+
}
|
|
55
|
+
/** Tidied for reading: the cost to a millionth of a dollar, the time to a tenth of a second - float sums otherwise show their noise. */
|
|
56
|
+
function rounded(totals) {
|
|
57
|
+
return { ...totals, cost_usd: Math.round(totals.cost_usd * 1e6) / 1e6, model_seconds: Math.round(totals.model_seconds * 10) / 10 };
|
|
58
|
+
}
|
|
59
|
+
const pulseKind = (job) => (job.kind === "regenerate" ? "code" : job.kind);
|
|
60
|
+
const firstLines = (text) => {
|
|
61
|
+
const trimmed = text?.trim();
|
|
62
|
+
return trimmed === undefined || trimmed === "" ? null : trimmed.length > 1000 ? `${trimmed.slice(0, 1000)}…` : trimmed;
|
|
63
|
+
};
|
|
64
|
+
/** Write a file whole or not at all: beside it first, then renamed over it. */
|
|
65
|
+
export function writeAtomically(file, text) {
|
|
66
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
67
|
+
const temporary = join(dirname(file), `.${basename(file)}.${process.pid}.tmp`);
|
|
68
|
+
try {
|
|
69
|
+
writeFileSync(temporary, text);
|
|
70
|
+
renameSync(temporary, file);
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
rmSync(temporary, { force: true });
|
|
74
|
+
throw error;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
export function startRunReport(options) {
|
|
78
|
+
const file = resolve(options.file);
|
|
79
|
+
const startedAt = Date.now();
|
|
80
|
+
const identity = turriIdentity();
|
|
81
|
+
const events = [];
|
|
82
|
+
const stopListening = onModelEvent((event) => events.push(event));
|
|
83
|
+
let sources = { jobs: () => [] };
|
|
84
|
+
let ended;
|
|
85
|
+
let crash;
|
|
86
|
+
let exitParts;
|
|
87
|
+
let finishedAt;
|
|
88
|
+
const fromProject = (path) => relative(options.projectDir, path) || ".";
|
|
89
|
+
const attempt = (read) => {
|
|
90
|
+
if (read === undefined)
|
|
91
|
+
return undefined;
|
|
92
|
+
try {
|
|
93
|
+
return read();
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
const usageOf = (jobs) => {
|
|
100
|
+
const ids = new Set(jobs.map((job) => job.id));
|
|
101
|
+
const kindOf = new Map(jobs.map((job) => [job.id, pulseKind(job)]));
|
|
102
|
+
// In pulses: their tokens as each job adds them up - what its pulse line prints - and their calls from the events.
|
|
103
|
+
const inPulses = emptyTotals();
|
|
104
|
+
for (const job of jobs)
|
|
105
|
+
addUsage(inPulses, job.usage);
|
|
106
|
+
const outside = emptyTotals();
|
|
107
|
+
const byModel = {};
|
|
108
|
+
const byRole = {};
|
|
109
|
+
const byKind = {};
|
|
110
|
+
for (const job of jobs)
|
|
111
|
+
addUsage((byKind[pulseKind(job)] ??= emptyTotals()), job.usage);
|
|
112
|
+
for (const event of events) {
|
|
113
|
+
const inPulse = event.job !== undefined && ids.has(event.job);
|
|
114
|
+
addEvent(inPulse ? inPulses : outside, event, !inPulse);
|
|
115
|
+
addEvent((byModel[event.model ?? "unknown"] ??= emptyTotals()), event, true);
|
|
116
|
+
addEvent((byRole[event.role ?? "unknown"] ??= emptyTotals()), event, true);
|
|
117
|
+
if (inPulse)
|
|
118
|
+
addEvent((byKind[kindOf.get(event.job)] ??= emptyTotals()), event, false);
|
|
119
|
+
}
|
|
120
|
+
const each = (table) => Object.fromEntries(Object.entries(table).map(([key, totals]) => [key, rounded(totals)]));
|
|
121
|
+
return {
|
|
122
|
+
total: rounded(sumTotals(inPulses, outside)),
|
|
123
|
+
in_pulses: rounded(inPulses),
|
|
124
|
+
outside_pulses: rounded(outside),
|
|
125
|
+
by_model: each(byModel),
|
|
126
|
+
by_role: each(byRole),
|
|
127
|
+
by_pulse_kind: each(byKind),
|
|
128
|
+
};
|
|
129
|
+
};
|
|
130
|
+
const pulsesOf = (jobs, now) => {
|
|
131
|
+
const callsOf = new Map();
|
|
132
|
+
for (const event of events)
|
|
133
|
+
if (event.job !== undefined)
|
|
134
|
+
addEvent(callsOf.get(event.job) ?? callsOf.set(event.job, emptyTotals()).get(event.job), event, false);
|
|
135
|
+
const list = jobs.map((job) => {
|
|
136
|
+
const landed = job.status === "succeeded" && job.details?.landed === true;
|
|
137
|
+
const ran = job.status !== "queued";
|
|
138
|
+
const usage = sumTotals(callsOf.get(job.id) ?? emptyTotals(), (() => {
|
|
139
|
+
const tokens = emptyTotals();
|
|
140
|
+
addUsage(tokens, job.usage);
|
|
141
|
+
return tokens;
|
|
142
|
+
})());
|
|
143
|
+
return {
|
|
144
|
+
id: job.id,
|
|
145
|
+
module: job.module ?? null,
|
|
146
|
+
number: job.pulse ?? null,
|
|
147
|
+
kind: pulseKind(job),
|
|
148
|
+
title: job.title ?? job.summary,
|
|
149
|
+
status: job.status,
|
|
150
|
+
landed,
|
|
151
|
+
failed_for_good: job.failedForGood === true,
|
|
152
|
+
started_at: ran ? new Date(job.startedAt).toISOString() : null,
|
|
153
|
+
duration_seconds: ran ? Math.round(((job.finishedAt ?? now) - job.startedAt) / 100) / 10 : null,
|
|
154
|
+
usage: rounded(usage),
|
|
155
|
+
agents: job.agents.map((agent) => ({
|
|
156
|
+
label: agent.label,
|
|
157
|
+
model: agent.model ?? null,
|
|
158
|
+
turns: agent.turns,
|
|
159
|
+
compactions: agent.compactions,
|
|
160
|
+
input_tokens: agent.usage.input,
|
|
161
|
+
output_tokens: agent.usage.output,
|
|
162
|
+
cache_read_tokens: agent.usage.cacheRead,
|
|
163
|
+
cache_write_tokens: agent.usage.cacheWrite,
|
|
164
|
+
cost_usd: Math.round(agent.usage.cost * 1e6) / 1e6,
|
|
165
|
+
})),
|
|
166
|
+
log_file: job.logFile !== undefined ? fromProject(job.logFile) : null,
|
|
167
|
+
error: job.status === "failed" || job.status === "stopped" ? firstLines(job.error) : null,
|
|
168
|
+
};
|
|
169
|
+
});
|
|
170
|
+
const ran = list.filter((pulse) => pulse.status !== "queued");
|
|
171
|
+
return {
|
|
172
|
+
counts: {
|
|
173
|
+
total: ran.length,
|
|
174
|
+
landed: ran.filter((pulse) => pulse.landed).length,
|
|
175
|
+
failed: ran.filter((pulse) => pulse.status === "failed").length,
|
|
176
|
+
stopped: ran.filter((pulse) => pulse.status === "stopped").length,
|
|
177
|
+
rolled_back: ran.filter((pulse) => !pulse.landed && pulse.status !== "running").length,
|
|
178
|
+
running: ran.filter((pulse) => pulse.status === "running").length,
|
|
179
|
+
not_started: list.length - ran.length,
|
|
180
|
+
},
|
|
181
|
+
list,
|
|
182
|
+
};
|
|
183
|
+
};
|
|
184
|
+
const functionalitiesOf = (counts) => {
|
|
185
|
+
if (counts === undefined)
|
|
186
|
+
return null;
|
|
187
|
+
const modules = counts.flatMap((chain) => chain.modules.map((module) => ({
|
|
188
|
+
module: module.module,
|
|
189
|
+
specified: module.specified,
|
|
190
|
+
completed: module.completed,
|
|
191
|
+
conformance_tested: module.conformanceTested,
|
|
192
|
+
incomplete_frids: module.incomplete,
|
|
193
|
+
})));
|
|
194
|
+
return {
|
|
195
|
+
total: {
|
|
196
|
+
specified: modules.reduce((sum, module) => sum + module.specified, 0),
|
|
197
|
+
completed: modules.reduce((sum, module) => sum + module.completed, 0),
|
|
198
|
+
conformance_tested: modules.reduce((sum, module) => sum + module.conformance_tested, 0),
|
|
199
|
+
},
|
|
200
|
+
modules,
|
|
201
|
+
codebases: counts.map((chain) => fromProject(chain.codebase)),
|
|
202
|
+
};
|
|
203
|
+
};
|
|
204
|
+
const statusAt = (exitCode) => {
|
|
205
|
+
const signal = exitCode !== undefined ? (SIGNALS[exitCode] ?? null) : null;
|
|
206
|
+
if (crash !== undefined)
|
|
207
|
+
return { status: "crashed", signal: null, error: crash };
|
|
208
|
+
if (ended !== undefined)
|
|
209
|
+
return { status: ended.status, signal: null, error: ended.error ?? null };
|
|
210
|
+
if (exitCode === undefined)
|
|
211
|
+
return { status: "running", signal: null, error: null };
|
|
212
|
+
if (signal !== null)
|
|
213
|
+
return { status: "interrupted", signal, error: null };
|
|
214
|
+
return { status: exitCode === 0 ? "completed" : "crashed", signal: null, error: exitCode === 0 ? null : `exited with ${exitCode} before the run ended` };
|
|
215
|
+
};
|
|
216
|
+
const build = (exitCode) => {
|
|
217
|
+
const now = finishedAt ?? Date.now();
|
|
218
|
+
// In the order they started; one queued again as another never ran, and is not a pulse of its own.
|
|
219
|
+
const jobs = (attempt(sources.jobs) ?? []).filter((job) => job.replacedBy === undefined).sort((a, b) => a.sequence - b.sequence);
|
|
220
|
+
const outcome = statusAt(exitCode);
|
|
221
|
+
const projectRoot = options.projectDir;
|
|
222
|
+
return {
|
|
223
|
+
schema_version: REPORT_SCHEMA_VERSION,
|
|
224
|
+
turri: identity,
|
|
225
|
+
command: options.command,
|
|
226
|
+
spec_file: resolve(options.specFile),
|
|
227
|
+
project_dir: projectRoot,
|
|
228
|
+
build_dir: options.buildDir !== undefined ? resolve(options.buildDir) : null,
|
|
229
|
+
conformance_tests: options.conformanceTests,
|
|
230
|
+
review: options.review ?? null,
|
|
231
|
+
models: {
|
|
232
|
+
...Object.fromEntries(Object.entries(options.models).map(([role, choice]) => [role, { model: choice.model ?? null, thinking: choice.thinking ?? null }])),
|
|
233
|
+
observed: Object.fromEntries([...new Set(events.map((event) => event.role ?? "unknown"))].sort().map((role) => [role, [...new Set(events.filter((event) => (event.role ?? "unknown") === role).map((event) => event.model ?? "unknown"))].sort()])),
|
|
234
|
+
},
|
|
235
|
+
started_at: new Date(startedAt).toISOString(),
|
|
236
|
+
finished_at: finishedAt !== undefined ? new Date(finishedAt).toISOString() : null,
|
|
237
|
+
duration_seconds: Math.round((now - startedAt) / 100) / 10,
|
|
238
|
+
outcome: {
|
|
239
|
+
status: outcome.status,
|
|
240
|
+
exit_code: exitCode ?? null,
|
|
241
|
+
signal: outcome.signal,
|
|
242
|
+
error: outcome.error,
|
|
243
|
+
modules: attempt(sources.modules) ?? null,
|
|
244
|
+
},
|
|
245
|
+
usage: usageOf(jobs),
|
|
246
|
+
pulses: pulsesOf(jobs, now),
|
|
247
|
+
functionalities: functionalitiesOf(attempt(sources.completion)),
|
|
248
|
+
...(exitParts ?? {}),
|
|
249
|
+
logs: {
|
|
250
|
+
jobs_dir: fromProject(join(projectRoot, ".turri", "jobs")),
|
|
251
|
+
debug_dir: fromProject(join(projectRoot, ".turri", "debug")),
|
|
252
|
+
sessions_dir: fromProject(join(projectRoot, ".turri", "sessions")),
|
|
253
|
+
},
|
|
254
|
+
};
|
|
255
|
+
};
|
|
256
|
+
const write = (exitCode) => {
|
|
257
|
+
try {
|
|
258
|
+
writeAtomically(file, `${JSON.stringify(build(exitCode), null, 2)}\n`);
|
|
259
|
+
}
|
|
260
|
+
catch (error) {
|
|
261
|
+
console.error(`Could not write the run report ${file}: ${error instanceof Error ? error.message : String(error)}`);
|
|
262
|
+
}
|
|
263
|
+
};
|
|
264
|
+
if (options.onExit !== false) {
|
|
265
|
+
process.on("uncaughtExceptionMonitor", (error) => {
|
|
266
|
+
crash ??= error instanceof Error ? (error.stack ?? error.message) : String(error);
|
|
267
|
+
});
|
|
268
|
+
process.once("exit", (code) => {
|
|
269
|
+
finishedAt = Date.now();
|
|
270
|
+
exitParts = attempt(sources.atExit);
|
|
271
|
+
stopListening();
|
|
272
|
+
write(code);
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
write();
|
|
276
|
+
return {
|
|
277
|
+
attach(given) {
|
|
278
|
+
sources = given;
|
|
279
|
+
},
|
|
280
|
+
end(status, error) {
|
|
281
|
+
ended = { status, ...(error !== undefined ? { error } : {}) };
|
|
282
|
+
},
|
|
283
|
+
crashed(error) {
|
|
284
|
+
crash ??= error instanceof Error ? error.message : String(error);
|
|
285
|
+
},
|
|
286
|
+
write: () => write(),
|
|
287
|
+
build,
|
|
288
|
+
};
|
|
289
|
+
}
|