@turri-dev/cli 0.1.2 → 0.1.3-next.26f08c5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat-skills.js +1 -1
- package/dist/cli-options.js +82 -0
- package/dist/config.js +23 -1
- package/dist/conform-cli.js +13 -5
- package/dist/environment.js +19 -0
- package/dist/extension.js +7 -11
- package/dist/init-config.js +3 -3
- package/dist/loop-cli.js +156 -28
- package/dist/pi-launch.js +13 -4
- package/dist/pi.js +14 -2
- package/dist/plain/errors.js +1 -1
- package/dist/plain/files.js +1 -1
- package/dist/plain/liquid.js +2 -2
- package/dist/regenerate/activity.js +19 -0
- package/dist/regenerate/loop.js +56 -2
- package/dist/regenerate/prompt.js +1 -1
- package/dist/regenerate/release.js +8 -6
- package/dist/regenerate-cli.js +24 -29
- package/dist/run-report.js +289 -0
- package/dist/setup.js +35 -16
- package/dist/spec-diff-cli.js +17 -6
- package/docs/config.md +22 -11
- package/docs/development.md +6 -0
- package/docs/headless.md +170 -16
- package/docs/installation.md +1 -1
- package/docs/parser.md +1 -1
- package/docs/regenerating.md +2 -2
- package/docs/using.md +4 -2
- package/genome/skills/implement-conformance-testing-script/SKILL.md +1 -1
- package/package.json +1 -1
package/dist/spec-diff-cli.js
CHANGED
|
@@ -2,16 +2,25 @@ import { readFileSync } from "node:fs";
|
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { checkPlainFile } from "./plain/check.js";
|
|
4
4
|
import { applySpecDiff, revertSpecDiff } from "./regenerate/regenerate.js";
|
|
5
|
-
import { parseArgs
|
|
5
|
+
import { parseArgs } from "./cli-options.js";
|
|
6
|
+
import { headlessOptions, runLoop } from "./loop-cli.js";
|
|
6
7
|
const USAGE = `usage: turri spec-diff <file.plain> <diff-file> [options]
|
|
7
8
|
|
|
8
9
|
Applies a change to the specs, given as a git diff of the .plain files, and builds it: the headless
|
|
9
10
|
counterpart of regenerate, given a diff. The diff is applied first, unless it is already in; then, as
|
|
10
11
|
turri regenerate does, what is left to build is planned into jobs and built.
|
|
11
12
|
|
|
12
|
-
--model <
|
|
13
|
-
--
|
|
14
|
-
|
|
13
|
+
--harness-model <p/id> config.turri.yaml's harness-model for this run: the planner, the conformance chooser, a diagnosis
|
|
14
|
+
--coding-model <p/id> its coding-model, --model too: the agents that build
|
|
15
|
+
--start-over start over, without asking, a codebase regenerated without a module it now requires
|
|
16
|
+
--judge-model <p/id> its judge-model: the judge and the test review
|
|
17
|
+
--review <true|false> its review: whether the code and its tests are reviewed once they pass
|
|
18
|
+
--conformance-tests <all|chosen|module|false>
|
|
19
|
+
its conformance-tests: which conformance tests are written and run
|
|
20
|
+
--build-dir <dir> as the run ends, export the code there (see turri regenerate --help)
|
|
21
|
+
--report <file.json> write a report of the run there (see turri regenerate --help)`;
|
|
22
|
+
const { positional, options } = parseArgs(process.argv.slice(2), ["--start-over"], USAGE, ["--build-dir", "--report", "--conformance-tests", "--harness-model", "--judge-model", "--review"]);
|
|
23
|
+
const { report, buildDir } = headlessOptions(options, true);
|
|
15
24
|
const [plainFile, diffFile] = positional;
|
|
16
25
|
if (plainFile === undefined || diffFile === undefined) {
|
|
17
26
|
console.error(USAGE);
|
|
@@ -28,9 +37,11 @@ if (!check.ok) {
|
|
|
28
37
|
}
|
|
29
38
|
runLoop({
|
|
30
39
|
plainFile,
|
|
31
|
-
...(options.model !== undefined ? { model: options.model } : {}),
|
|
32
40
|
startOver: options["start-over"] !== undefined,
|
|
33
|
-
|
|
41
|
+
...(buildDir !== undefined ? { buildDir } : {}),
|
|
42
|
+
...(report !== undefined ? { report } : {}),
|
|
43
|
+
command: ["spec-diff", ...process.argv.slice(2)],
|
|
44
|
+
ask: (loop) => loop.bringUp(plainFile),
|
|
34
45
|
}).catch((error) => {
|
|
35
46
|
console.error(error instanceof Error ? error.message : error);
|
|
36
47
|
process.exit(1);
|
package/docs/config.md
CHANGED
|
@@ -5,7 +5,7 @@ is optional: with no file at all, turri finds the testing scripts on its own, tu
|
|
|
5
5
|
tests on wherever a module has test reqs, and runs on pi's configured model. A project whose
|
|
6
6
|
parts are on different stacks - a backend and a frontend - adds a file per stack,
|
|
7
7
|
`config.turri.<stack>.yaml`, beside it, and chooses the stack to build with (see
|
|
8
|
-
[stacks](#stacks)). turri never writes a `config.yaml`: it may be another tool's
|
|
8
|
+
[stacks](#stacks)). turri never writes a `config.yaml`: it may be another tool's, and
|
|
9
9
|
is read only by a project that has no file of turri's yet (see [the old config.yaml](#the-old-configyaml)).
|
|
10
10
|
|
|
11
11
|
`turri init-config` starts one: the two models, a strong one to talk to and a smaller one to write the code. It
|
|
@@ -42,7 +42,7 @@ change one by changing it. The optional ones, which have no value of their own -
|
|
|
42
42
|
skill that writes them), `module`, `conformance-budget`, `judge-model`, `models` - are there commented out, each with an
|
|
43
43
|
example: uncomment one to use it. A key taken out of the file is still read as the table below says.
|
|
44
44
|
|
|
45
|
-
Beside a `config.yaml` -
|
|
45
|
+
Beside a `config.yaml` - another tool's, say - the new file takes from it the keys turri reads too, each in its place, its
|
|
46
46
|
testing scripts above all, and says which it left (see [the old config.yaml](#the-old-configyaml)); `config.yaml`
|
|
47
47
|
is left as it is. A `config.turri.yaml` already there keeps everything but its two models, which it sets - `--force` writes the whole
|
|
48
48
|
file again - and a `config.yaml` turri read until then is copied to it first. It takes a folder other than the current
|
|
@@ -133,7 +133,7 @@ models: # each agent role's model, thinking and tur
|
|
|
133
133
|
| `release-after-pulse` | `true` or `false`: whether every pulse that lands code releases it, in the chat, as its last stage - before the module's next pulse starts, whatever the chat is doing: an agent on the coding model looks for the project's release guidance as the chat does (what was passed to `release` this conversation, AGENTS.md, the project's skills) and, when it says to release after a build, does what it says - see [deploy](deploy.md#after-every-pulse). One release a pulse that lands; a release that fails is said in the pulse's report, and what it landed stays | `false` |
|
|
134
134
|
| `unittests-script` | the unit testing script, by path from the project root - it runs the code's static checks too, exiting 80 when only they fail (see [the static checks](regenerating.md#the-static-checks)) | looked for - see below |
|
|
135
135
|
| `conformance-tests-script` | the conformance testing script, by path from the project root | looked for - see below |
|
|
136
|
-
| `prepare-environment-script` | optional
|
|
136
|
+
| `prepare-environment-script` | optional: run as `<script> <build>` before every run of the conformance testing script, which runs only once it passes - see [the testing scripts](regenerating.md#the-testing-scripts). turri never writes one | the one `prepare_environment*.sh` at the root or in `test_scripts/`, else none |
|
|
137
137
|
| `compact-at` | how many tokens of work - past its instructions - an agent kept through a job holds before the harness compacts it between turns; at least 1000; the project's alone | 100,000 - see [agent sessions](regenerating.md#agent-sessions) |
|
|
138
138
|
| `max-specs-per-pulse` | the most functional specs the planner groups into one pulse - what builds one feature goes together, up to this many; a stack's file wins | 3 |
|
|
139
139
|
| `max-lines-per-pulse` | the most lines of spec in one pulse - its functional specs with the definitions they bring; a group past it is split again, a functional spec a pulse; a stack's file wins | 120 |
|
|
@@ -145,7 +145,7 @@ models: # each agent role's model, thinking and tur
|
|
|
145
145
|
| `testing-scripts-run-on` | what the testing scripts run on - always on this machine, in your environment. `in-place`: the real build and tests folders, the script at its own path, from the project - so a script that finds a host codebase from its own path finds it. `copy`: a fresh copy of the code, staged in `plain_modules/.turri/scripts/`, with the script copied beside it; it gives the coding agent `can_it_fail` (see [the testing scripts](regenerating.md#the-testing-scripts)) | `in-place` |
|
|
146
146
|
|
|
147
147
|
Keys are written with dashes; underscores are read the same (`conformance_tests`). A key turri
|
|
148
|
-
does not know is ignored -
|
|
148
|
+
does not know is ignored - another tool's `config.yaml`, read while it is the old file, may carry several - and so is a value of the
|
|
149
149
|
wrong type, as if it were not there. `model:`, from before there were separate models, is read as
|
|
150
150
|
`coding-model`.
|
|
151
151
|
|
|
@@ -218,8 +218,13 @@ They are defaults. What is asked for wins over them:
|
|
|
218
218
|
| model | wins over the config |
|
|
219
219
|
| --- | --- |
|
|
220
220
|
| harness | `turri --harness-model <m>` (or pi's own `--model`), and pi's `/model` in the conversation |
|
|
221
|
-
| coding | `turri --coding-model <m>`, `engine` in the conversation
|
|
222
|
-
| judge |
|
|
221
|
+
| coding | `turri --coding-model <m>`, `engine` in the conversation |
|
|
222
|
+
| judge | - |
|
|
223
|
+
|
|
224
|
+
`turri regenerate`, `conform` and `spec-diff` read `--harness-model`, `--coding-model` (or `--model`),
|
|
225
|
+
`--judge-model`, `--review` and `--conformance-tests` differently: as the keys themselves, as if the file had
|
|
226
|
+
them. Each takes its key's value and its place in the order above - a `models:` entry still wins over it - so a
|
|
227
|
+
headless run is the same as a chat with those keys in the file.
|
|
223
228
|
|
|
224
229
|
A job keeps the model it was queued with: choosing another holds for what is queued after.
|
|
225
230
|
|
|
@@ -231,8 +236,8 @@ codebase are not run, and no agent is shown the conformance testing script. No k
|
|
|
231
236
|
`false`: conformance tests are off by default. `all` (or `true`) means on wherever a module has
|
|
232
237
|
`***test reqs***` - a module without them has nothing to write conformance tests from, so it writes
|
|
233
238
|
none, but still keeps the codebase's existing ones passing. Asked in the
|
|
234
|
-
conversation, or with `--conformance
|
|
235
|
-
otherwise.
|
|
239
|
+
conversation (`conform`), or with `--conformance-tests all|chosen|module|false` on `turri regenerate` and
|
|
240
|
+
`spec-diff` - for that run, the file left as it is - a render does otherwise.
|
|
236
241
|
|
|
237
242
|
`spawn` writes `conformance-tests: false` into a new project unless you ask for tests;
|
|
238
243
|
`brood` asks, whatever the key says, offering what it says first.
|
|
@@ -346,8 +351,14 @@ The first time turri itself writes a setting there with `/turri-config`, it copi
|
|
|
346
351
|
`config.turri.yaml` and each `config.<name>.yaml` to `config.turri.<name>.yaml` first, leaves the old
|
|
347
352
|
files as they are, and from then on reads only the new ones. `turri init-config` copies the stacks'
|
|
348
353
|
files the same way, but writes `config.turri.yaml` as its quick start, taking from `config.yaml` only
|
|
349
|
-
the keys turri reads too -
|
|
354
|
+
the keys turri reads too - `unittests-script`, `conformance-tests-script` and
|
|
350
355
|
`prepare-environment-script`, say - each in its place in the file, and saying which keys it left:
|
|
351
|
-
`template_dir`, `verbose`, `api`,
|
|
356
|
+
`template_dir`, `verbose`, `api`, the other tool's alone. The two models are the vendor's, never taken. To change a
|
|
352
357
|
setting by hand in such a project, do the same: copy the files to their new names, and edit the
|
|
353
|
-
copies - never `config.yaml`.
|
|
358
|
+
copies - never `config.yaml`. Writing a `config.turri.yaml` of your own beside it is the same move: from then on
|
|
359
|
+
`config.yaml` is not read, its testing scripts with it. For a headless run that only needs conformance tests or
|
|
360
|
+
another judge, or review, those options leave the files alone: `--conformance-tests`, `--judge-model`, `--review true|false`, and the models' too.
|
|
361
|
+
|
|
362
|
+
Keys turri does not read - another tool's `verbose`, `api-key`, `build-folder`, ... - are ignored without a word, and so
|
|
363
|
+
is a value of the wrong type. A file that does not parse as YAML is read as empty, so its testing scripts are then
|
|
364
|
+
looked for by name.
|
package/docs/development.md
CHANGED
|
@@ -18,6 +18,12 @@ which tests reach which mutant (`vitest.stryker.config.ts`, the suite without th
|
|
|
18
18
|
mutant against those tests. `scripts/mutation.mjs` sets two things in Stryker's vitest runner the suite needs - forked
|
|
19
19
|
workers, and a test-name filter that matches vitest 5's names - again after every `npm install`.
|
|
20
20
|
|
|
21
|
+
The `mutation` workflow (`.github/workflows/mutation.yml`) runs it on GitHub, started by hand: Actions, mutation, Run
|
|
22
|
+
workflow. It takes the parts to run, space-separated (empty for every part), each a job of its own side by side, on
|
|
23
|
+
four cores (`TURRI_MUTATION_CONCURRENCY`; six here). Each part's `mutation.html` and `mutation.json` are uploaded as an
|
|
24
|
+
artifact, `mutation-<part>`, and summed up on the run's page. Stryker's incremental file is cached from run to run, so
|
|
25
|
+
a run tests again only the mutants whose code or tests changed; tick `fresh` to test every one again.
|
|
26
|
+
|
|
21
27
|
Working on turri itself, `npm link` points the global `turri` at this clone, so edits
|
|
22
28
|
take effect immediately. `npm run chat` starts pi with the extension loaded from `src/`.
|
|
23
29
|
|
package/docs/headless.md
CHANGED
|
@@ -14,7 +14,8 @@ Everything marked `✗` stops the run, so fix those first. Two more things matte
|
|
|
14
14
|
a module's implementation reqs name that this machine lacks.
|
|
15
15
|
- **The models.** In a project, `turri doctor` sends each model `config.turri.yaml` names one short request, as turri's
|
|
16
16
|
agents will - so a key the provider refuses, a model pi does not know or an account without access to it shows now,
|
|
17
|
-
not an hour into the run.
|
|
17
|
+
not an hour into the run. Models the run is given on the command line are given to doctor the same way -
|
|
18
|
+
`--harness-model`, `--coding-model`, `--judge-model`, read as the config's keys - and are checked with the config's.
|
|
18
19
|
- **Credentials that do not expire mid-run.** An API key in the environment
|
|
19
20
|
(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, ...) or pi's saved login - whatever
|
|
20
21
|
`doctor` shows as the model provider.
|
|
@@ -62,9 +63,14 @@ conformance-tests: true # write and run conformance tests where there
|
|
|
62
63
|
```
|
|
63
64
|
|
|
64
65
|
The same file names the [testing scripts](regenerating.md#the-testing-scripts); every key is in
|
|
65
|
-
[the project config](config.md).
|
|
66
|
-
|
|
67
|
-
`
|
|
66
|
+
[the project config](config.md). `turri regenerate`, `spec-diff` and `conform` take five of its keys on
|
|
67
|
+
the command line too - `--harness-model`, `--coding-model` (`--model` is the same), `--judge-model`,
|
|
68
|
+
`--review` and, but for `conform`, `--conformance-tests`. Each is read as if the file had it, in its place:
|
|
69
|
+
it takes that key's value, and every key keeps its place in the order the config resolves them in - a
|
|
70
|
+
`models:` entry still wins over `harness-model`, `coding-model` or `judge-model`, the planner and the
|
|
71
|
+
conformance chooser run on the harness model, the judge falls back on the coding model. So a run with
|
|
72
|
+
the options is the same as one, or a chat, with those keys in `config.turri.yaml` - and a `config.yaml`
|
|
73
|
+
the project has is left as it is.
|
|
68
74
|
A project whose parts are on different stacks keeps each stack's settings in
|
|
69
75
|
`config.turri.<stack>.yaml` beside it ([stacks](config.md#stacks)). Choose the one to build with
|
|
70
76
|
`--stack <name>` - on any command: `regenerate`, `conform`, `spec-diff`, `glimpse`, `release`, `cultivate` -
|
|
@@ -75,29 +81,39 @@ Useful options - `turri regenerate --help` lists all of them:
|
|
|
75
81
|
|
|
76
82
|
| | |
|
|
77
83
|
| --- | --- |
|
|
78
|
-
| `--model <provider/id>` |
|
|
79
|
-
| `--
|
|
84
|
+
| `--harness-model <provider/id>` | `harness-model`: the planner, the conformance chooser, the diagnosis of a failed pulse |
|
|
85
|
+
| `--coding-model <provider/id>` | `coding-model` (`--model` too): the agents that build - with `:level` for how hard it thinks (`provider/id:high`); not given, the harness model's |
|
|
86
|
+
| `--judge-model <provider/id>` | `judge-model`: the judge and the test review; not given, the coding model |
|
|
87
|
+
| `--conformance-tests <all\|chosen\|module\|false>` | `conformance-tests`, with its values (`true` is `all`) |
|
|
88
|
+
| `--review <true\|false>` | `review`: whether the code and its tests are reviewed once they pass |
|
|
80
89
|
| `--stack <name>` | build with the stack's `config.turri.<name>.yaml` over `config.turri.yaml`; default, `TURRI_STACK`, else none |
|
|
81
|
-
| `--
|
|
82
|
-
| `--
|
|
83
|
-
| `--
|
|
90
|
+
| `--start-over` | start over, unasked, a codebase built without a module it now requires - without a terminal, a run that needs it and lacks it stops |
|
|
91
|
+
| `--build-dir <dir>` | as the run ends, export the code there - see below |
|
|
92
|
+
| `--report <file.json>` | write [the run report](#the-run-report) there |
|
|
93
|
+
|
|
94
|
+
An option a command does not take is refused, not ignored.
|
|
84
95
|
|
|
85
96
|
`turri regenerate` queues the jobs the chat's `regenerate` does - a spec diff that brings what
|
|
86
97
|
is built in line, when it no longer matches the spec, then one render functionality job per
|
|
87
98
|
functional spec not built, each with its conformance job - and runs them all to the end, the queue
|
|
88
99
|
master reworking them as in the chat (see [a whole module](regenerating.md#a-whole-module)).
|
|
89
|
-
Conformance tests follow `conformance-tests:` in `config.turri.yaml`.
|
|
100
|
+
Conformance tests follow `conformance-tests:` in `config.turri.yaml`, or `--conformance-tests`.
|
|
90
101
|
|
|
91
102
|
## In the morning
|
|
92
103
|
|
|
93
|
-
The command exits `0` only when every module is complete
|
|
94
|
-
|
|
104
|
+
The command exits `0` only when every module is complete: its code regenerated from the spec as it is
|
|
105
|
+
now, and its conformance tests written for what is built when they are on - no pulse failed, nothing
|
|
106
|
+
halted, nothing waiting for a start over. Anything else exits `1`, and the last lines say which
|
|
107
|
+
module is not complete, and why. The log says, per functionality, what was built and whether its
|
|
108
|
+
tests passed.
|
|
95
109
|
|
|
96
110
|
| | |
|
|
97
111
|
| --- | --- |
|
|
98
112
|
| `plain_modules/<module>/code/` | the code and unit tests |
|
|
99
113
|
| `plain_modules/<module>/tests/` | the conformance tests |
|
|
100
114
|
| `plain_modules/<module>/code/.turri.json` | the marshalled plain tree the code is built from |
|
|
115
|
+
| `.turri/jobs/` | each pulse's log: its agents, its test runs, how it ended |
|
|
116
|
+
| `.turri/debug/` | each agent's whole conversation |
|
|
101
117
|
|
|
102
118
|
And what happened, step by step, in the module's history:
|
|
103
119
|
|
|
@@ -105,13 +121,112 @@ And what happened, step by step, in the module's history:
|
|
|
105
121
|
git --git-dir=plain_modules/<module>/.plaingit --work-tree=plain_modules/<module> log --stat
|
|
106
122
|
```
|
|
107
123
|
|
|
124
|
+
### Where the code is
|
|
125
|
+
|
|
126
|
+
A module that `requires` others is built in its own codebase on top of theirs: the code of
|
|
127
|
+
`plain_modules/<module>/code` holds every module below it. `--build-dir <dir>` exports it as the run
|
|
128
|
+
ends, however it ends - done, failed, stopped by a signal: the last commit of the furthest-built
|
|
129
|
+
codebase of the chain, with the harness's own files (`.turri.json`, the notes) left out, so what is in
|
|
130
|
+
it is only what landed, its unit tests passing. The folder's contents are replaced each time; a
|
|
131
|
+
folder that holds the project, or one inside `plain_modules`, is refused. Point an external test
|
|
132
|
+
suite at it.
|
|
133
|
+
|
|
134
|
+
### The run report
|
|
135
|
+
|
|
136
|
+
`--report <file.json>` writes one JSON summary of the run, for a benchmark or CI to read rather than
|
|
137
|
+
the log. It is written as the run starts, again as each pulse ends, and last as the process exits -
|
|
138
|
+
whether the run is done, a pulse failed, the provider failed, it crashed, or `SIGTERM` / `SIGINT` /
|
|
139
|
+
`SIGHUP` stopped it. Only a process killed outright (`SIGKILL`) keeps the one written as its last pulse
|
|
140
|
+
ended. Each write replaces the file whole (written beside it, then renamed), so a reader never sees
|
|
141
|
+
half of one. Keys are snake_case, numbers are numbers, and `schema_version` changes only when a field
|
|
142
|
+
changes meaning or goes; new fields may appear in the same version. Paths in `pulses`,
|
|
143
|
+
`functionalities` and `logs` are relative to `project_dir`.
|
|
144
|
+
|
|
145
|
+
```jsonc
|
|
146
|
+
{
|
|
147
|
+
"schema_version": 1,
|
|
148
|
+
"turri": { "version": "0.1.1", "git_commit": "48dacfc…", "git_dirty": false }, // commit: null outside a clone
|
|
149
|
+
"command": ["regenerate", "app.plain", "--model", "…", "--report", "report.json"],
|
|
150
|
+
"spec_file": "/work/app.plain", "project_dir": "/work", "build_dir": "/work/out", // build_dir: null without one
|
|
151
|
+
"conformance_tests": "all", // all | chosen | module | false
|
|
152
|
+
"review": false,
|
|
153
|
+
"models": {
|
|
154
|
+
"harness": { "model": "…", "thinking": "medium" }, // the planner, the chooser, the diagnosis
|
|
155
|
+
"coding": { "model": "…", "thinking": "low" },
|
|
156
|
+
"judge": { "model": "…", "thinking": "medium" },
|
|
157
|
+
"observed": { "coding": ["provider/id:low"], "planner": ["provider/id:medium"] } // what each role really ran on
|
|
158
|
+
},
|
|
159
|
+
"started_at": "…", "finished_at": "…", "duration_seconds": 4351.4,
|
|
160
|
+
"outcome": {
|
|
161
|
+
"status": "completed", // completed | incomplete | failed | interrupted | crashed | running (written mid-run)
|
|
162
|
+
"exit_code": 0, "signal": null, "error": null,
|
|
163
|
+
"modules": [{ "module": "app", "state": "complete" }] // complete | incomplete | halted | start_over_needed | running, with "why"
|
|
164
|
+
},
|
|
165
|
+
"usage": {
|
|
166
|
+
"total": {
|
|
167
|
+
"input_tokens": 0, "output_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cost_usd": 0,
|
|
168
|
+
"llm_calls": 0, "agent_turns": 0, "compactions": 0, "model_seconds": 0
|
|
169
|
+
},
|
|
170
|
+
"in_pulses": { }, // the pulses' usage added up - the same fields
|
|
171
|
+
"outside_pulses": { }, // the planner, the conformance chooser, a failed pulse's diagnosis
|
|
172
|
+
"by_model": { "provider/id:low": { } },
|
|
173
|
+
"by_role": { "coding": { }, "reviewer": { }, "planner": { } },
|
|
174
|
+
"by_pulse_kind": { "code": { }, "conformance": { } }
|
|
175
|
+
},
|
|
176
|
+
"pulses": {
|
|
177
|
+
"counts": { "total": 7, "landed": 5, "failed": 1, "stopped": 0, "rolled_back": 2, "running": 0, "not_started": 0 },
|
|
178
|
+
"list": [{
|
|
179
|
+
"id": "2a058729", "module": "app", "number": 1, "kind": "code", "title": "…",
|
|
180
|
+
"status": "succeeded", "landed": true, "failed_for_good": false,
|
|
181
|
+
"started_at": "…", "duration_seconds": 512.3, "usage": { },
|
|
182
|
+
"agents": [{ "label": "app 1 coding agent", "model": "…", "turns": 14, "compactions": 0, "input_tokens": 0, "…": 0 }],
|
|
183
|
+
"log_file": ".turri/jobs/….log", "error": null
|
|
184
|
+
}]
|
|
185
|
+
},
|
|
186
|
+
"functionalities": {
|
|
187
|
+
"total": { "specified": 12, "completed": 10, "conformance_tested": 8 },
|
|
188
|
+
"modules": [
|
|
189
|
+
{ "module": "base", "specified": 4, "completed": 4, "conformance_tested": 4, "incomplete_frids": [] },
|
|
190
|
+
{ "module": "app", "specified": 8, "completed": 6, "conformance_tested": 4, "incomplete_frids": ["7", "8"] }
|
|
191
|
+
],
|
|
192
|
+
"codebases": ["plain_modules/app/code"]
|
|
193
|
+
},
|
|
194
|
+
"build_dir_export": { "exported": true, "dir": "/work/out", "module": "app", "commit": "8036b3b", "files": 23 },
|
|
195
|
+
"logs": { "jobs_dir": ".turri/jobs", "debug_dir": ".turri/debug", "sessions_dir": ".turri/sessions" }
|
|
196
|
+
}
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
**Usage.** Every model response an agent gets counts, in a pulse or not. `total` is `in_pulses` plus
|
|
200
|
+
`outside_pulses`, and `in_pulses` is the pulses' own usage added up - the figures their lines in the
|
|
201
|
+
notes and the pulse logs print - so the totals can be checked by hand against them. `llm_calls` counts
|
|
202
|
+
model responses, a compaction's included; `agent_turns` the harness's requests to an agent, each as
|
|
203
|
+
many responses as it takes; `model_seconds` the time from each request to its response's end, added
|
|
204
|
+
up - several agents working at once each count theirs, so it can exceed the run's duration. Tokens and
|
|
205
|
+
cost are as the provider reports them to pi. `by_model`, `by_role` and `by_pulse_kind` split the same
|
|
206
|
+
calls; a role or model turri could not tell is `unknown`.
|
|
207
|
+
|
|
208
|
+
**Pulses.** `landed`: its work is committed. `rolled_back`: it ended without landing - failed,
|
|
209
|
+
stopped, or did not pass - and everything it wrote was undone. A pulse that failed is tried once more
|
|
210
|
+
before it counts as failed; both tries are the one pulse.
|
|
211
|
+
|
|
212
|
+
**Completed functional specs** - the pass rate, the figure to compare runs by.
|
|
213
|
+
A functional spec is completed when the code record of the codebase the run delivers - the
|
|
214
|
+
furthest-built of the chain, the one `--build-dir` exports - as last committed, holds it as the spec has
|
|
215
|
+
it now: the same text, every file it links holding the same, no definition it uses changed, and not
|
|
216
|
+
marked to regenerate again. A record moves only when a pulse lands, and a pulse lands only once the
|
|
217
|
+
code's unit tests pass (its conformance tests too, when they are on), so completed means landed and
|
|
218
|
+
passing. Each module counts its own functional specs: a module below the tip counts from what the
|
|
219
|
+
tip's code was built on, and a module above the delivered codebase has none completed.
|
|
220
|
+
`conformance_tested` is those of the completed ones whose conformance tests are recorded as written
|
|
221
|
+
for them as they are now (`plain_modules/<module>/tests/.turri.json`).
|
|
222
|
+
|
|
108
223
|
## If it stopped
|
|
109
224
|
|
|
110
225
|
Run the same command again. Functionalities already built from the current spec are left alone,
|
|
111
226
|
and modules that are complete are skipped, so it picks up where it stopped - after a crash, a
|
|
112
227
|
provider outage, or a machine that went to sleep anyway. Whatever the stopped run left
|
|
113
228
|
half-written is undone first: each step starts from the module's last commit. A functionality whose spec has changed
|
|
114
|
-
since counts as not built, so editing the spec and re-running rebuilds what changed. `--
|
|
229
|
+
since counts as not built, so editing the spec and re-running rebuilds what changed. `--from-scratch`
|
|
115
230
|
rebuilds everything.
|
|
116
231
|
|
|
117
232
|
## Rebuilding one functionality
|
|
@@ -125,14 +240,53 @@ it is - without rebuilding the ones before it. To change what it does, change it
|
|
|
125
240
|
spec is what the code is built from, and a change made there lasts. Then it updates the
|
|
126
241
|
functionality's conformance tests and runs every conformance test in the codebase, since a change can break something built
|
|
127
242
|
later as easily as something built earlier. It works in the codebase the functionality now lives
|
|
128
|
-
in: when later modules are built on top of this one, the furthest-built of them
|
|
129
|
-
|
|
130
|
-
no code yet - build it with `module` first.
|
|
243
|
+
in: when later modules are built on top of this one, the furthest-built of them. It exits `0` when
|
|
244
|
+
everything passes, and refuses when there is no code yet - regenerate the module first.
|
|
131
245
|
|
|
132
246
|
Several changes do not need one command each: after editing the spec, re-running `turri regenerate`
|
|
133
247
|
brings the code up to it - one spec diff for what changed in what is built, then the functional
|
|
134
248
|
specs not built yet.
|
|
135
249
|
|
|
250
|
+
## In a container
|
|
251
|
+
|
|
252
|
+
A benchmark or CI task runs it with no terminal and nothing to ask, so give it everything up front:
|
|
253
|
+
|
|
254
|
+
```bash
|
|
255
|
+
turri regenerate app.plain --harness-model amazon-bedrock/eu.anthropic.claude-opus-5-5 \
|
|
256
|
+
--coding-model amazon-bedrock/eu.anthropic.claude-sonnet-5-5 \
|
|
257
|
+
--conformance-tests all --review false --judge-model amazon-bedrock/eu.anthropic.claude-opus-5-5 \
|
|
258
|
+
--start-over --build-dir /out/build --report /out/report.json < /dev/null
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
- **What it needs:** Node 22.19 or newer, `bash`, `git` (each module's history, and the spec diffs
|
|
262
|
+
the work is planned from), `/bin/sh` with `kill` and `sleep`, a writable `HOME` (pi keeps
|
|
263
|
+
`~/.pi/agent/auth.json` there, even with no login - or set `PI_CODING_AGENT_DIR`) and `TMPDIR`, and
|
|
264
|
+
the toolchains the project's testing scripts and implementation reqs name. `turri doctor` in the project, given
|
|
265
|
+
the run's models, checks them all and that each model answers - at the start of the task's log, not when a pulse
|
|
266
|
+
fails:
|
|
267
|
+
|
|
268
|
+
```bash
|
|
269
|
+
turri doctor --harness-model amazon-bedrock/eu.anthropic.claude-opus-5-5 \
|
|
270
|
+
--coding-model amazon-bedrock/eu.anthropic.claude-sonnet-5-5 --judge-model amazon-bedrock/eu.anthropic.claude-opus-5-5
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
It exits 1 on anything marked `✗`.
|
|
274
|
+
- **Without bubblewrap:** a container that cannot make user namespaces - no
|
|
275
|
+
`--security-opt seccomp=unconfined` - has no OS sandbox. turri runs anyway, the command check and the
|
|
276
|
+
audit holding alone under the agents' commands; `turri doctor` shows it as `!`, not `✗`. Installing
|
|
277
|
+
the `bubblewrap` package is worth it where it can start.
|
|
278
|
+
- **As another user:** run it as the user who will own `plain_modules/`. git refuses a repository
|
|
279
|
+
another user made (`dubious ownership`), so do not let root create it first.
|
|
280
|
+
- **Amazon Bedrock:** `AWS_BEARER_TOKEN_BEDROCK` (or the usual AWS credentials) and `AWS_REGION` are
|
|
281
|
+
enough - no pi login. An `eu.` inference profile needs an EU region. A model id pi does not know
|
|
282
|
+
stops the first pulse; `turri --list-models bedrock` lists the ones it does.
|
|
283
|
+
- **A project with only a `config.yaml`:** it is read while there is no `config.turri*.yaml` -
|
|
284
|
+
`unittests-script`, `conformance-tests-script` and `prepare-environment-script` included, relative to
|
|
285
|
+
the project; the keys turri does not know are ignored. Do not write a `config.turri.yaml` beside it to
|
|
286
|
+
turn on conformance tests or review: from then on `config.yaml` is not read. The options above do it instead.
|
|
287
|
+
- **Long test suites:** a testing script run is stopped after 120 seconds; set `TURRI_COMMAND_TIMEOUT`
|
|
288
|
+
(seconds) for more.
|
|
289
|
+
|
|
136
290
|
## Asking in words instead
|
|
137
291
|
|
|
138
292
|
pi's print mode runs one request and exits, with the render tools waiting for their work rather
|
package/docs/installation.md
CHANGED
|
@@ -68,7 +68,7 @@ In a project - a folder with `config.turri.yaml` (or the old `config.yaml`) or `
|
|
|
68
68
|
| --- | --- |
|
|
69
69
|
| each module's toolchains | what its implementation reqs name - Python, Node, Java, Go, Rust, C/C++, Zig, ... - against this machine's PATH: the agents use this machine's toolchains and install none, so a missing one is yours to install |
|
|
70
70
|
| the testing scripts | what in them this machine does not have: a program named by its whole path that is not here (a macOS-only `/usr/libexec/java_home` on Linux), a toolchain they run that is not on the PATH |
|
|
71
|
-
| the models | each model `config.turri.yaml` names - `harness-model`, `coding-model`, `judge-model`, every role under `models:` - sent one short request through pi, with the credentials turri's agents will use, for any provider pi supports. One that answers is ready; for one that does not, it says what the provider said and what to do: the credentials pi reads for that provider (`AWS_PROFILE` and `AWS_REGION` for Amazon Bedrock, `OPENAI_API_KEY`, `/login`, ...), model access in the Bedrock console, the closest model ids pi knows for one it does not. `--no-models` skips it |
|
|
71
|
+
| the models | each model `config.turri.yaml` names - `harness-model`, `coding-model`, `judge-model`, every role under `models:` - sent one short request through pi, with the credentials turri's agents will use, for any provider pi supports. One that answers is ready; for one that does not, it says what the provider said and what to do: the credentials pi reads for that provider (`AWS_PROFILE` and `AWS_REGION` for Amazon Bedrock, `OPENAI_API_KEY`, `/login`, ...), model access in the Bedrock console, the closest model ids pi knows for one it does not. `--harness-model`, `--coding-model` (`--model`) and `--judge-model` name models as the config's keys, as `turri regenerate` reads them - checked in a project or not. `--no-models` skips it |
|
|
72
72
|
|
|
73
73
|
Removing that file is the one thing `doctor` changes, and it asks first; `turri doctor --yes` says yes. Run without a
|
|
74
74
|
terminal - piped, or in CI - it asks nothing and changes nothing. Run it again whenever something breaks;
|
package/docs/parser.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
| `src/plain/spec.ts` | `plain_spec.py` - functionality ids, the per-functionality context, linked resources, ranges |
|
|
10
10
|
| `src/plain/concepts.ts` | `concept_utils.py` - concept extraction, validation, topological ordering |
|
|
11
11
|
| `src/plain/markdown.ts` | mistletoe's AST + `PlainRenderer` (mdast + a matching serializer) |
|
|
12
|
-
| `src/plain/liquid.ts` |
|
|
12
|
+
| `src/plain/liquid.ts` | the reference implementation's Liquid nodes + the two filters in `plain_spec.py` (liquidjs) |
|
|
13
13
|
| `src/plain/files.ts` | `file_utils.open_from` / `get_loaded_templates`, base64 blob detection |
|
|
14
14
|
| `src/plain/stdlib.ts` | nothing - turri's own: `stdlib/` is a reserved import prefix, read from turri's standard library only, and a project module at such a path is an error (see [the standard library](authoring-plain.md#the-standard-library)) |
|
|
15
15
|
|
package/docs/regenerating.md
CHANGED
|
@@ -495,7 +495,7 @@ scripts are not linked from the specs; a link to one is ignored.
|
|
|
495
495
|
They are required: a build with no unit testing script, or with conformance on and no conformance
|
|
496
496
|
testing script, stops before it starts and says which skill adds it.
|
|
497
497
|
|
|
498
|
-
**The prepare script** is
|
|
498
|
+
**The prepare script** is optional: turri never writes one, and runs one the project has -
|
|
499
499
|
`prepare-environment-script:` in the config, else the one `prepare_environment*.sh` at the root, else in
|
|
500
500
|
`test_scripts/` - on the build, before every run of the conformance testing script: the harness's own, and the agents'
|
|
501
501
|
`run_conformance_tests`. Both get the same build folder - the code in place, or the same fresh copy - so what it sets up
|
|
@@ -685,7 +685,7 @@ change to the code brings that module's code in, so the codebase stops: it has t
|
|
|
685
685
|
and conformance tests taken out, committed so its history keeps them, and the chain built again from the
|
|
686
686
|
bottom. That is never done unasked. In the chat the model asks the person, saying why, and calls
|
|
687
687
|
`regenerate` with `from_scratch` once they agree; `turri regenerate` and `spec-diff` ask at the terminal, and
|
|
688
|
-
`--start-over` agrees in advance - with no terminal and no flag,
|
|
688
|
+
`--start-over` agrees in advance - with no terminal and no flag, the run stops there, exit 1, saying so.
|
|
689
689
|
|
|
690
690
|
## Conformance tests
|
|
691
691
|
|
package/docs/using.md
CHANGED
|
@@ -13,8 +13,10 @@ banner - the jellyfish it is named after, swimming until the conversation fills
|
|
|
13
13
|
terminal that shows images (Kitty, Ghostty, WezTerm, Warp, iTerm2), and drawn in text in any other, tmux
|
|
14
14
|
included - swimming a stroke a loop, its tentacles following its bell, frames `scripts/jellyfish-frames.mjs`
|
|
15
15
|
writes -
|
|
16
|
-
|
|
17
|
-
and resource list; `turri --verbose` shows
|
|
16
|
+
turri's version (and, run from a clone, the commit it is on, and whether the clone has changes), where commands
|
|
17
|
+
will run, the rules and skills loaded - rather than pi's startup header and resource list; `turri --verbose` shows
|
|
18
|
+
those too. pi's "Update Available" notice is left out - turri's pi is a pinned dependency `pi update` never
|
|
19
|
+
touches - and the terminal's tab reads `🪼 turri - <folder>`, not pi's `π`. Talk to it: ask it to start a
|
|
18
20
|
project, add a feature, render a module, fix a spec. It decides which skill or tool applies - but
|
|
19
21
|
not *how* a skill runs: `cultivate` hands the work to the gated runner, which owns the steps, the
|
|
20
22
|
gates and the judges (see [skills and gates](skills-and-gates.md)).
|
|
@@ -36,7 +36,7 @@ bash run_conformance_tests_<lang>.sh <build_folder> <conformance_tests_folder>
|
|
|
36
36
|
|
|
37
37
|
Both arguments are absolute paths: `<build_folder>` is the module's generated code (`plain_modules/<module>/code`), and `<conformance_tests_folder>` is the folder holding the whole suite. The harness runs the script on the person's machine - macOS, Linux or WSL, with its toolchains, in the person's own environment - and its exit code alone decides whether the conformance tests pass.
|
|
38
38
|
|
|
39
|
-
Every conformance script is self-contained: toolchain check, argument validation, stage the build into `<temp>/<lang>_<basename of $1>`, install dependencies there in isolation, check the tests in `$2` statically, run the suite in `$2` against it, clean up on exit. Nothing is prepared ahead of time by another script - unless the project already has a `prepare_environment*.sh` (
|
|
39
|
+
Every conformance script is self-contained: toolchain check, argument validation, stage the build into `<temp>/<lang>_<basename of $1>`, install dependencies there in isolation, check the tests in `$2` statically, run the suite in `$2` against it, clean up on exit. Nothing is prepared ahead of time by another script - unless the project already has a `prepare_environment*.sh` (a prepare script the project brought, or one `prepare-environment-script:` names): turri runs it on the same build before every run of the conformance script, so keep it, and let the conformance script rely on what it sets up rather than doing it again. Never write a prepare script yourself.
|
|
40
40
|
|
|
41
41
|
## How conformance scripts differ from unit-test scripts
|
|
42
42
|
|