@tduyng/pi-harness-loop 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -335
- package/package.json +2 -2
- package/src/check.ts +57 -7
- package/src/diversity.ts +1 -1
- package/src/install.ts +14 -3
- package/src/store.ts +10 -4
package/README.md
CHANGED
|
@@ -1,255 +1,138 @@
|
|
|
1
1
|
# @tduyng/pi-harness-loop
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
-
|
|
27
|
-
-
|
|
28
|
-
|
|
29
|
-
-
|
|
30
|
-
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
so a skill
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
in does not force a rescan.
|
|
67
|
-
|
|
68
|
-
The cache holds 400 sessions, so the oldest leave only when that ceiling is reached, not when
|
|
69
|
-
the newest few exceed a byte budget.
|
|
70
|
-
|
|
71
|
-
A failure that shows up in two sessions or more is a property of the harness, and the loop acts
|
|
72
|
-
on it. A command or a file that shows up in every session is a habit, and a habit is a
|
|
73
|
-
judgement call the operator makes, so the loop leaves those to `/harness-loop propose`.
|
|
74
|
-
|
|
75
|
-
A new error string that is a smaller variant of an edit matching failure the loop already tried
|
|
76
|
-
is not a new finding. The edit tool has one matching mechanic, so the 2-session "oldText must be
|
|
77
|
-
unique" is the 9-session "oldText must match" with a different error string, and a second
|
|
78
|
-
proposal for it is the same rule said twice.
|
|
79
|
-
|
|
80
|
-
The failure's own command is replayed before a proposal. When it passes, the failure no longer
|
|
81
|
-
reproduces, and the loop writes nothing. The same command runs again after an install, and its
|
|
82
|
-
result stays with the install.
|
|
83
|
-
|
|
84
|
-
A failure whose message the tree no longer carries is not a property of the harness any more.
|
|
85
|
-
Before it proposes, the loop cuts the ids, the paths and the numbers out of the message, and
|
|
86
|
-
searches the tree for what is left, then the dependencies. When neither has it and git shows a
|
|
87
|
-
commit that touched that text after the last session that hit it, the producer was removed and
|
|
88
|
-
the finding is skipped. The verdict is cached in `.pi/harness/producers.json`, keyed by the
|
|
89
|
-
commit and the last session, because it is a search of the tree.
|
|
90
|
-
|
|
91
|
-
The loop stands out of the way when the session has something else driving it. A goal that is
|
|
92
|
-
active and the day's budget of five proposals each stop it. The writer is one completion, not a
|
|
93
|
-
turn: it carries the brief, it has no tools, and it returns the files of the part as text. The
|
|
94
|
-
files are parsed, checked and installed by the loop, so nothing the writer says is executed. The
|
|
95
|
-
session is notified with what landed, and it keeps its own turn.
|
|
96
|
-
|
|
97
|
-
The lesson tool arms itself for the run whose request names a trigger of a lesson that is still
|
|
98
|
-
in force, and it is off otherwise, so the request with nothing to do with a lesson pays
|
|
99
|
-
nothing. The cost on the runs that do arm it is about 259 tokens: the schema, the snippet and
|
|
100
|
-
the guidelines.
|
|
3
|
+
The harness fixes itself. The extension reads what your sessions actually repeated, writes the skill or rule that would have prevented it, installs it, and takes it back out if the failure rate does not drop.
|
|
4
|
+
|
|
5
|
+
## What it costs
|
|
6
|
+
|
|
7
|
+
Nothing until it acts. The report is a slash command rather than a tool, because a tool would put its schema in the system prompt of every session, which is the tax this package exists to reduce.
|
|
8
|
+
|
|
9
|
+
The `lesson` tool is the one tool, at 252 tokens on the runs that arm it. It arms only for a run whose request names a trigger of a lesson still in force, so a request with nothing to recall pays nothing.
|
|
10
|
+
|
|
11
|
+
A proposal costs one completion, and a part adds to the front of every later request. Both are counted: `maxStandingTokens` (1,500 by default) is the ceiling on what active installs may add together, and three installs a day is the ceiling on the loop.
|
|
12
|
+
|
|
13
|
+
## The loop
|
|
14
|
+
|
|
15
|
+
The session goes idle. The loop reads the sessions of this project, and if a failure recurred across them it writes a brief with the evidence. One model call turns that brief into the part, and the part is installed. Nothing has to be typed, and the loop never takes the session over.
|
|
16
|
+
|
|
17
|
+
A part that does not lower the rate it was written for is taken back out at the next idle, so the always-on context does not keep a rule the sessions did not need.
|
|
18
|
+
|
|
19
|
+
`PI_HARNESS_LOOP=off` stops all of it.
|
|
20
|
+
|
|
21
|
+
## Commands
|
|
22
|
+
|
|
23
|
+
| Command | Effect |
|
|
24
|
+
| --- | --- |
|
|
25
|
+
| `/harness-loop` | Report the setup, the findings, the cost, and the lessons |
|
|
26
|
+
| `/harness-loop propose [id]` | Run a proposal now |
|
|
27
|
+
| `/harness-loop lint` | Report only the problems |
|
|
28
|
+
| `/harness-loop revert <install>` | Put the files back by hand |
|
|
29
|
+
| `/harness-loop install <id>` | Install a part already on disk |
|
|
30
|
+
| `/harness-loop clear` | Take the report off the screen |
|
|
31
|
+
|
|
32
|
+
A report is cleared by the next prompt as well, so one you have read does not sit above the editor for the rest of the session.
|
|
33
|
+
|
|
34
|
+
## What the report shows
|
|
35
|
+
|
|
36
|
+
The harness tree: the skills and agent profiles pi loads, the problems in them, and what the skill block costs on every request. The rules come from pi's own loader, so they do not drift from it.
|
|
37
|
+
|
|
38
|
+
- A skill with no `description` is dropped at load, so it never reaches the model.
|
|
39
|
+
- A skill whose frontmatter name differs from its directory is advertised under one name and looked for under another.
|
|
40
|
+
- An agent profile with no description is listed as "no description", so the model cannot choose it.
|
|
41
|
+
- A skill that tells the agent to run a script it does not ship sends it after a file that is not there.
|
|
42
|
+
|
|
43
|
+
The findings: what recurred in this session. The same failure three times, the same command three times, the same file read three times, a run that hit a failure and changed nothing, a tool call with no result.
|
|
44
|
+
|
|
45
|
+
The recurrences: what recurred across the sessions of this project. The same failure in two sessions, the same command in three, the same file read in five. A defect in one run is noise, and a defect in every run is a property of the harness. A signature that reduces to `<path>` or a bare `Script failed` is not a finding, because it names nothing a part can change.
|
|
46
|
+
|
|
47
|
+
A failure also needs independent evidence. When every session that hit it edited a file and each pair shares one, they are one piece of work, and a rule written from it would encode that task rather than the harness. The recurrence is listed and held back. A session that edited nothing met the failure outside the edit loop, so it counts as independent.
|
|
48
|
+
|
|
49
|
+
The skills nobody opened: pi advertises every skill by name and description in the system prompt, so a skill nobody opens is a tax on every request of every session. The report lists the advertised ones with the characters they cost, and says how many are not advertised at all, since a skill the model cannot open cannot be opened by accident.
|
|
50
|
+
|
|
51
|
+
The lessons: the ones whose cue matches the request in the editor, read from `.pi/harness/lessons.jsonl`.
|
|
52
|
+
|
|
53
|
+
## What it costs to scan
|
|
54
|
+
|
|
55
|
+
The scan is cached in `.pi/harness/scan.json` with a fingerprint of the session files, so a session that changed nothing does not pay for it again. Each parsed session is cached beside it in `.pi/harness/corpus-cache.json`, so a scan reads the files that changed and takes the rest from cache. At most 8 MB of new session files are parsed per idle, and the cache holds 400 sessions, so the oldest leave only at that ceiling rather than when the newest few exceed a byte budget.
|
|
56
|
+
|
|
57
|
+
The session being written is left out, so the file the loop is in does not force a rescan.
|
|
58
|
+
|
|
59
|
+
A failure's own command is replayed before a proposal. When it passes, the failure no longer reproduces and the loop writes nothing. The same command runs again after an install, and its result stays with the install.
|
|
60
|
+
|
|
61
|
+
A failure whose message the tree no longer carries is not a property of the harness any more. Before proposing, the loop cuts the ids, paths, and numbers out of the message and searches the tree, then the dependencies. When neither has it and git shows a commit that touched that text after the last session that hit it, the producer was removed and the finding is skipped. That verdict is cached in `.pi/harness/producers.json`.
|
|
62
|
+
|
|
63
|
+
A new error string that is a smaller variant of an edit failure the loop already tried is not a new finding. The edit tool has one matching mechanic, so the 2-session "oldText must be unique" is the 9-session "oldText must match" with a different error string.
|
|
64
|
+
|
|
65
|
+
The loop stands out of the way when something else drives the session: an active goal, or the day's budget of five proposals, stops it. The writer is one completion rather than a turn. It carries the brief, has no tools, and returns the files as text, which the loop parses, checks, and installs, so nothing the writer says is executed.
|
|
101
66
|
|
|
102
67
|
## The proposal
|
|
103
68
|
|
|
104
|
-
`/harness-loop propose [id]` runs the same path for one finding, one recurrence or one skill:
|
|
105
|
-
brief, part, install. The brief asks for a skill, an agent profile or a rule, and the writer
|
|
106
|
-
puts the files in `.pi/harness/<id>/`. The part is read back, checked against the same rules the
|
|
107
|
-
tree is checked against, and installed. A part that breaks a rule is not installed, and the rule
|
|
108
|
-
it broke is recorded with it. A reply with no file block leaves `REPLY.txt` beside the brief,
|
|
109
|
-
which is what the writer said instead.
|
|
110
|
-
|
|
111
|
-
A part under `global/` holds in every project. A skill under `global/` goes to the skills every
|
|
112
|
-
project loads, and a global rule goes to the context file every project reads. It is not
|
|
113
|
-
installed unless `global` is set in `.pi/harness.json`, because one project's failure does not
|
|
114
|
-
get to become every project's rule. Without it the part lands in the project: `.agents/skills/<name>/`,
|
|
115
|
-
`.pi/agents/<name>.md`, and the project's context file for a rule.
|
|
116
|
-
|
|
117
|
-
The prompt says what each form costs. A skill is paid for by its name and description, about 300
|
|
118
|
-
characters on every request, and its body costs nothing until the model opens it. A rule is paid
|
|
119
|
-
for by its length and always applies. The rule wins when the behaviour has to hold every time,
|
|
120
|
-
and the skill wins when the situation is specific enough to route on.
|
|
121
|
-
|
|
122
|
-
The part may also carry `CONFIRM.md`, one shell command that exits 0 only when the finding can
|
|
123
|
-
no longer happen. It must not write files or use the network. The loop runs it after the project
|
|
124
|
-
check, and a confirmation that fails puts the part back. The brief asks for the command, and a
|
|
125
|
-
part without one installs on the project check alone, recorded as unconfirmed.
|
|
126
|
-
|
|
127
|
-
When the part passes, the session records the lesson that transfers: the cue (the shape of
|
|
128
|
-
situation), the lesson (the operation on the harness), the way to confirm it worked, the
|
|
129
|
-
evidence it rests on, and a confidence. The model may decline with `NO_ENTRY`, and then nothing
|
|
130
|
-
is written. A block with no confirm line is dropped, because a rule nobody can check is a rule
|
|
131
|
-
of thumb. The lesson is appended to `.pi/harness/lessons.jsonl` and matched against later
|
|
132
|
-
requests by its trigger words. A lesson about a situation an earlier lesson already covers
|
|
133
|
-
replaces it: the old one keeps its record, stops matching, and the report names the install
|
|
134
|
-
that still holds the old rule.
|
|
135
|
-
|
|
136
|
-
The trigger words are the ones the prompts of this project actually used, because a word no
|
|
137
|
-
request ever carried cannot route to anything. A trigger seen once has to be at least eight
|
|
138
|
-
characters and pass a consonant check. A typo is the rarest token in the corpus, and it routes
|
|
139
|
-
nowhere.
|
|
140
|
-
|
|
141
|
-
A request matches when it carries two triggers, or one trigger of six characters or more that
|
|
142
|
-
the project rarely uses: one short word is a coincidence, and `key` is in half the requests of a
|
|
143
|
-
project that does not talk about credentials. Two processes that record the same cue leave two
|
|
144
|
-
lines in the file, and the read keeps the newest active one, so the same situation does not arm
|
|
145
|
-
twice. A match records a use, so a lesson nobody ever recalls is visible in the record.
|
|
69
|
+
`/harness-loop propose [id]` runs the same path for one finding, one recurrence, or one skill: brief, part, install. The brief asks for a skill, an agent profile, or a rule, and the writer puts the files in `.pi/harness/<id>/`. The part is read back, checked against the same rules the tree is checked against, and installed. A part that breaks a rule is not installed and the rule it broke is recorded with it. A reply with no file block leaves `REPLY.txt` beside the brief, which is what the writer said instead.
|
|
146
70
|
|
|
147
|
-
|
|
71
|
+
A part under `global/` holds in every project, and it is not installed unless `global` is set in `.pi/harness.json`, because one project's failure does not get to become every project's rule. Without it the part lands in the project: `.agents/skills/<name>/`, `.pi/agents/<name>.md`, or the project's context file for a rule.
|
|
72
|
+
|
|
73
|
+
The prompt says what each form costs. A skill is paid for by its name and description, about 300 characters on every request, and its body costs nothing until opened. A rule is paid for by its length and applies always. The rule wins when the behaviour has to hold every time, and the skill wins when the situation is specific enough to route on.
|
|
74
|
+
|
|
75
|
+
A part may carry `CONFIRM.md`: one shell command that exits 0 only when the finding can no longer happen. It must not write files or use the network. It runs after the project check, and a failing confirmation puts the part back. A part without one installs on the project check alone, recorded as unconfirmed.
|
|
76
|
+
|
|
77
|
+
A rule part installs without a project check, because no test reads a markdown file. A skill or profile waits for the check, because it changes what the loader sees.
|
|
78
|
+
|
|
79
|
+
## The lesson
|
|
148
80
|
|
|
149
|
-
|
|
150
|
-
against the rules pi's own loader applies, and the extension layer on top.
|
|
81
|
+
When a part passes, the session records the lesson that transfers: the cue (the shape of situation), the lesson (the operation on the harness), how to confirm it worked, the evidence, and a confidence. The model may decline with `NO_ENTRY`, and a block with no confirm line is dropped, because a rule nobody can check is a rule of thumb.
|
|
151
82
|
|
|
152
|
-
The
|
|
153
|
-
load and never reaches the model. One whose frontmatter name does not match its directory is
|
|
154
|
-
advertised under one name and looked for under another. Another that calls a helper script it
|
|
155
|
-
does not ship points at a file that is not there. Two skills with the same name means the second
|
|
156
|
-
one is dropped. An agent profile with no `description` is listed as "no description".
|
|
83
|
+
The lesson goes to `.pi/harness/lessons.jsonl` and matches later requests by its trigger words, which are the words this project's prompts actually used. A trigger seen once has to be at least eight characters and pass a consonant check, because a typo is the rarest token in the corpus and routes nowhere.
|
|
157
84
|
|
|
158
|
-
|
|
159
|
-
`extensions/` that declares an entry the root manifest never loads is a package nobody gets.
|
|
160
|
-
An entry file that is not on disk is a manifest that points at nothing.
|
|
85
|
+
A request matches when it carries two triggers, or one trigger of six characters or more that the project rarely uses. One short word is a coincidence, and `key` is in half the requests of a project that never talks about credentials. A match records a use, so a lesson nobody recalls is visible in the record.
|
|
161
86
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
87
|
+
A lesson about a situation an earlier lesson already covers replaces it: the old one keeps its record, stops matching, and the report names the install that still holds the old rule.
|
|
88
|
+
|
|
89
|
+
## The lint
|
|
90
|
+
|
|
91
|
+
`/harness-loop lint` is the harness check on its own.
|
|
92
|
+
|
|
93
|
+
The skill rules are the ones that bite silently, listed above. The extension rules are the ones the repo's own scripts do not cover: a package under `extensions/` that declares an entry the root manifest never loads is a package nobody gets, and an entry file that is not on disk is a manifest pointing at nothing.
|
|
94
|
+
|
|
95
|
+
The rule rules matter because a project's check cannot fail for a rule. A rule is markdown in a file no test reads, so the lint is the only gate it passes. A rule without a heading cannot be replaced later, a rule longer than thirty lines pays for itself on every request, and a rule carrying a credential lands in a file every session reads.
|
|
166
96
|
|
|
167
97
|
## The check decides
|
|
168
98
|
|
|
169
|
-
A part that passes the rules can still be wrong, and pi has no grader to say so. So the part is
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
- the `test` recipe in `justfile`
|
|
177
|
-
|
|
178
|
-
The check runs before the change and after it. The part is installed first, so the common case
|
|
179
|
-
costs one run of the suite: when the check passes, the part stays. When it fails, the part goes
|
|
180
|
-
back the way it was and the check runs once more on the tree without it, so the verdict is about
|
|
181
|
-
the difference. A suite that was already failing the same way is not read as a verdict against
|
|
182
|
-
the change. A part that adds a failure is put back, and no lesson is recorded, because a part the check turned
|
|
183
|
-
down taught nothing.
|
|
184
|
-
|
|
185
|
-
When the part carries a `CONFIRM.md` command, that command runs after the project check. A
|
|
186
|
-
confirmation that fails puts the part back, because the part did not prove the finding gone. The
|
|
187
|
-
command and its output stay with the install.
|
|
188
|
-
|
|
189
|
-
When the finding carried a command, it is replayed after the install. Its result is recorded but
|
|
190
|
-
does not decide the install, because the project check and the confirm command are the gates.
|
|
191
|
-
Three installs a day is the ceiling for the loop, and the operator's own hand is not counted
|
|
192
|
-
against it. The count is in `.pi/harness/installs.jsonl`. The ceiling is a budget that frees
|
|
193
|
-
after a day, not a verdict: a proposal that lost to the budget stays open and the loop tries it
|
|
194
|
-
again, and the spend is counted once.
|
|
195
|
-
|
|
196
|
-
A second ceiling is the prompt itself. `maxStandingTokens`, 1,500 by default, is how many tokens
|
|
197
|
-
the active installs may add together to the front of every request. An install that would cross
|
|
198
|
-
it stays on disk with its brief, and the message names the current total, so the way through is
|
|
199
|
-
to revert a part that did not carry rather than to raise the cap. A rule is paid in full on every
|
|
200
|
-
request, which is why this ceiling counts the rule text and only the name and description of a
|
|
201
|
-
skill. The report prints the standing total in tokens. `0` turns the ceiling off.
|
|
202
|
-
|
|
203
|
-
A rule is identified by its heading, so a rule that is revised replaces the rule it revises
|
|
204
|
-
rather than saying it twice.
|
|
99
|
+
A part that passes the rules can still be wrong, and pi has no grader to say so. So the part is installed and the project's own check runs against it. A rule instals without that check, and the rate verdict below is what holds it instead.
|
|
100
|
+
|
|
101
|
+
The check is the first of: `check` in `.pi/harness.json`, the `test` script in `package.json`, or the `test` recipe in `justfile`.
|
|
102
|
+
|
|
103
|
+
It runs before and after. The part is installed first, so the common case costs one run: when the check passes, the part stays. When it fails, the part is put back and the check runs once more without it, so the verdict is about the difference rather than about a suite that was already red. A part that adds a failure is put back and no lesson is recorded, because a part the check turned down taught nothing.
|
|
104
|
+
|
|
105
|
+
`maxStandingTokens` is the second ceiling, at 1,500 tokens by default. An install that would cross it stays on disk with its brief and the message names the current total, so the way through is to revert a part that did not carry rather than to raise the cap. A rule is paid in full on every request, which is why this ceiling counts a rule's text and only the name and description of a skill. The report prints the standing total. `0` turns the ceiling off.
|
|
205
106
|
|
|
206
107
|
## The sessions decide whether it held
|
|
207
108
|
|
|
208
|
-
The check says the part did not break the project. It cannot say the failure stopped, and for a
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
the brief says the earlier form did not carry and shows the two rates and the form itself, and
|
|
236
|
-
the lesson that came with it stops matching, because a rule nobody can act on is worse than no
|
|
237
|
-
rule. A finding whose part was all markdown and still did not carry is marked `needs code`. The
|
|
238
|
-
loop stops proposing prose for it, because the failure lives in tool behavior and a rule cannot
|
|
239
|
-
reach it. The report shows the verdicts, and a `held` finding is the one thing the loop can call
|
|
240
|
-
done.
|
|
241
|
-
|
|
242
|
-
A finding comes back as soon as the failure grows past the count it had at the last proposal,
|
|
243
|
-
and a finding stops after three proposals, so one failure cannot spend the week's budget
|
|
244
|
-
rephrasing the same rule.
|
|
245
|
-
|
|
246
|
-
`/harness-loop revert <install>` puts the files back by hand, and the newest install goes first
|
|
247
|
-
when two of them touched the same file. A file whose content changed after the install is left
|
|
248
|
-
alone and named, because the operator's edit is not the loop's to overwrite. `/harness-loop
|
|
249
|
-
install <id>` runs the same path over a part that is already on disk. One install runs at a time,
|
|
250
|
-
through a lock beside the record.
|
|
251
|
-
|
|
252
|
-
`.pi/harness.json` turns the loop off or changes the ceiling:
|
|
109
|
+
The check says the part did not break the project. It cannot say the failure stopped, and for a rule or a skill it cannot see the part at all. So the install is read against the sessions that come after it.
|
|
110
|
+
|
|
111
|
+
The signature of the finding is counted per session, before and after, and every session counts whether it hit the failure or not, because a session that did not fail is the evidence.
|
|
112
|
+
|
|
113
|
+
| Verdict | Meaning |
|
|
114
|
+
| --- | --- |
|
|
115
|
+
| `held` | The rate fell to three quarters of what it was, and sessions still call tools at more than half their old rate |
|
|
116
|
+
| `flat` | The rate stayed within a quarter either way |
|
|
117
|
+
| `worse` | The rate rose by a quarter or more |
|
|
118
|
+
| `avoided` | The rate fell but the work did too, so the failure stopped because the agent did less |
|
|
119
|
+
| `confounded` | A commit touched the failure in the window, so the rate is not the part's |
|
|
120
|
+
| `early` | Fewer than three sessions have run since the install |
|
|
121
|
+
| `out-of-window` | The corpus holds no session from before the install |
|
|
122
|
+
|
|
123
|
+
The work rate is tool calls per session from the same parse. It is counted across tools rather than per tool, because a session that switched from `edit` to `write` did the work and fixed the failure.
|
|
124
|
+
|
|
125
|
+
The verdict is computed on idle as well as in the report, so a lesson whose part did not carry stops matching without anyone typing a command.
|
|
126
|
+
|
|
127
|
+
A `flat`, `worse`, or `avoided` install is not the end of the finding. The loop tries again, the brief says the earlier form did not carry and shows both rates, and the lesson that came with it stops matching, because a rule nobody can act on is worse than no rule. A finding whose part was all markdown and still did not carry is marked `needs code`, and the loop stops proposing prose for it, because the failure lives in tool behaviour and a rule cannot reach it.
|
|
128
|
+
|
|
129
|
+
A finding comes back when the failure grows past the count it had at the last proposal, and it stops after three proposals, so one failure cannot spend the week's budget rephrasing the same rule.
|
|
130
|
+
|
|
131
|
+
`/harness-loop revert <install>` puts the files back by hand, and the newest install goes first when two touched the same file. A file whose content changed after the install is left alone and named, because your edit is not the loop's to overwrite. One install runs at a time, through a lock beside the record.
|
|
132
|
+
|
|
133
|
+
## Configure
|
|
134
|
+
|
|
135
|
+
`.pi/harness.json`:
|
|
253
136
|
|
|
254
137
|
```json
|
|
255
138
|
{
|
|
@@ -263,119 +146,42 @@ through a lock beside the record.
|
|
|
263
146
|
}
|
|
264
147
|
```
|
|
265
148
|
|
|
266
|
-
`writerModel` is the model that writes the part. The loop reads three answers in
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
and nothing else, so another model loses no context by writing it.
|
|
272
|
-
|
|
273
|
-
`auto` is read when the session goes idle, and it stops the loop from writing a part on its
|
|
274
|
-
own. The manual command still works. `autoInstall` is read when the part is ready, so a project
|
|
275
|
-
that wants every change by hand sets it to `false` and the part stays on disk.
|
|
276
|
-
`PI_HARNESS_LOOP=off` in the environment turns the automatic run off for every project without
|
|
277
|
-
changing a file, and the command still works.
|
|
278
|
-
|
|
279
|
-
A rule part installs without a project check, because no test reads a markdown file. A skill or
|
|
280
|
-
an agent profile needs the check, because it changes what the loader sees. What a rule is held
|
|
281
|
-
to instead is the rate of the failure it was written for: flat, worse or avoided after three
|
|
282
|
-
sessions and the loop reverts it, records `auto-reverted`, and stops proposing prose for that
|
|
283
|
-
finding.
|
|
284
|
-
|
|
285
|
-
## Taking it back out
|
|
286
|
-
|
|
287
|
-
The loop reverts its own part when the sessions after it show the same rate or a worse one. The
|
|
288
|
-
files go back byte for byte, the install is recorded as reverted with the two rates, and the
|
|
289
|
-
finding is marked needs code, so the next attempt has to live in the tool rather than in a rule.
|
|
290
|
-
A file the operator changed after the install is left alone and named, because that edit is not
|
|
291
|
-
the loop's to overwrite.
|
|
292
|
-
|
|
293
|
-
## What the loop spends
|
|
294
|
-
|
|
295
|
-
The loop is not free. A proposal costs one completion, and the completion carries the brief
|
|
296
|
-
with the evidence, which is the whole input. `writerModel` names the model that runs it, and
|
|
297
|
-
the output is the part. The tokens the provider reported are written to
|
|
298
|
-
`.pi/harness/spend.jsonl` with the outcome, so a budget of five proposals a day is five
|
|
299
|
-
completions and not five sessions. The report sums them.
|
|
300
|
-
|
|
301
|
-
A part also costs the prompt after it is installed, and that cost is not a completion. Every
|
|
302
|
-
install records the characters it adds to the front of the prompt, which is the rule as it is
|
|
303
|
-
appended or the name and description a skill is routed on, and the report sums the active ones.
|
|
304
|
-
The brief carries that sum, so the writer knows what this project already pays on every request
|
|
305
|
-
before it chooses a rule over a skill.
|
|
149
|
+
`writerModel` is the model that writes the part. The loop reads three answers in order: `writerModel` in the project file, `PI_HARNESS_WRITER_MODEL` in the environment, then the `model` in `.pi/subagent.json` or `~/.pi/agent/subagent.json`, which is the model pi already runs chores on. With none of them it uses the session's own model, and a session on an expensive model pays that model for a short chore. The writer sees the brief and nothing else, so another model loses no context by writing it.
|
|
150
|
+
|
|
151
|
+
`auto` is read when the session goes idle and stops the loop writing a part on its own, while the manual command still works. `autoInstall` is read when the part is ready, so a project that wants every change by hand sets it to `false` and the part stays on disk.
|
|
152
|
+
|
|
153
|
+
The tokens the provider reported are written to `.pi/harness/spend.jsonl` with the outcome, so a budget of five proposals a day is five completions rather than five sessions. Every install records the characters it adds to the front of the prompt, and the report sums the active ones, so a new proposal knows what the project already pays before choosing a rule over a skill.
|
|
306
154
|
|
|
307
155
|
## What it works with
|
|
308
156
|
|
|
309
157
|
The other extensions drive the same session, so the loop reads their state instead of guessing at it.
|
|
310
158
|
|
|
311
|
-
`pi-goal` owns the turns of a session with a goal
|
|
312
|
-
active, and it waits to install while one is, because both of them would be driving the same
|
|
313
|
-
session at the same time.
|
|
314
|
-
|
|
315
|
-
`pi-goal` owns the check runner in `src/check.ts`. The loop uses the same runner, so
|
|
316
|
-
the two do not disagree about what a check said.
|
|
159
|
+
`pi-goal` owns the turns of a session with a goal, so the loop does not propose while one is active and waits to install while one is. The check runner in `pi-goal` is the loop's runner too, so the two do not disagree about what a check said.
|
|
317
160
|
|
|
318
|
-
`pi-context` prunes
|
|
319
|
-
whole branch. The recurrences read the session files, which no compaction touches.
|
|
161
|
+
`pi-context` prunes what the model sees, not the session, so the findings still see the whole branch.
|
|
320
162
|
|
|
321
|
-
`pi-guard` blocks the shell from reading credentials
|
|
322
|
-
context, and it says what to do instead of what is blocked.
|
|
163
|
+
`pi-guard` blocks the shell from reading credentials, and the rule it needs is in the global context, saying what to do instead of what is blocked.
|
|
323
164
|
|
|
324
165
|
## What it found here
|
|
325
166
|
|
|
326
|
-
The measurements come from the sessions on this machine, so they
|
|
327
|
-
about pi.
|
|
328
|
-
|
|
329
|
-
- 12 of the 19 skills are not advertised at all, because their frontmatter says
|
|
330
|
-
`disable-model-invocation`, so they cost nothing. The other 7 are advertised, and the block
|
|
331
|
-
they make is 2,652 characters, about 663 tokens on every request of every project.
|
|
332
|
-
|
|
333
|
-
- 5 of those 7 were opened in none of the 26 sessions scanned last. They cost 1,447 of the 2,652
|
|
334
|
-
characters, about 362 tokens per request. `tdd` was the sixth, and the loop's finding for it
|
|
335
|
-
is the one change that came from a proposal and nothing else: 39 sessions, no open, and the
|
|
336
|
-
skill stopped being advertised on 2026-10-26.
|
|
337
|
-
|
|
338
|
-
An open is a read of a `SKILL.md` or a `/skill:<name>` command in a prompt, so a skill reached
|
|
339
|
-
by hand is not called dead. The report names the rest and the number, and the operator
|
|
340
|
-
decides, because a skill that was never needed is not the same as a skill that cannot be
|
|
341
|
-
found.
|
|
342
|
-
|
|
343
|
-
- The shell blocked 14 reads of credential files across 2 sessions, and every retry with another
|
|
344
|
-
shell reached the same block.
|
|
167
|
+
The measurements come from the sessions on this machine, so they describe this setup.
|
|
345
168
|
|
|
346
|
-
|
|
347
|
-
|
|
169
|
+
- 12 of 19 skills are not advertised at all, because their frontmatter says `disable-model-invocation`, so they cost nothing. The other 7 make a block of 2,652 characters, about 663 tokens on every request of every project.
|
|
170
|
+
- 5 of those 7 were opened in none of 26 sessions scanned. They cost 1,447 of the 2,652 characters, about 362 tokens per request. `tdd` was the sixth: 39 sessions with no open, and the skill stopped being advertised on 2026-10-26, from a proposal and nothing else. An open is a read of a `SKILL.md` or a `/skill:<name>` command, so a skill reached by hand is not called dead.
|
|
171
|
+
- The shell blocked 14 reads of credential files across 2 sessions, and every retry with another shell reached the same block. The rule that came out of it says what to do instead.
|
|
172
|
+
- The edit tool rejected 9 old texts across 4 sessions, and each rejection reported a closest match the model had paraphrased. The rule that came out of it says to copy old text out of the read that preceded the edit, and to read the file again when an edit is rejected.
|
|
173
|
+
- 4 reads were refused for being over 350 lines across 3 sessions, and each was a file a grep would have narrowed. The rule that came out of it says to grep first.
|
|
348
174
|
|
|
349
|
-
|
|
350
|
-
match the model had paraphrased.
|
|
351
|
-
|
|
352
|
-
The rule that came out of it says to copy the old text out of the output of the read that
|
|
353
|
-
preceded the edit, and to read the file again when an edit is rejected.
|
|
354
|
-
|
|
355
|
-
- 4 reads were refused for being over 350 lines across 3 sessions, and each one was a file a
|
|
356
|
-
grep would have narrowed.
|
|
357
|
-
|
|
358
|
-
The rule that came out of it says to grep first.
|
|
359
|
-
|
|
360
|
-
## Why the report is a command and not a tool
|
|
175
|
+
## What it is not
|
|
361
176
|
|
|
362
|
-
|
|
363
|
-
exists to reduce. The command costs nothing until it is typed.
|
|
177
|
+
It does not score the harness. It measures the rate of the failure a part was written for, which is not the same as the work being better: a session ends, and nothing says whether the work was good. The findings are the part that can be read without a grader, and the rate is the part that says whether the part held.
|
|
364
178
|
|
|
365
|
-
The
|
|
366
|
-
of a lesson in force, so a session can record a lesson the moment it hits the failure that
|
|
367
|
-
taught it. It is 259 tokens on the runs that arm it and nothing on the others. The words that
|
|
368
|
-
count are the ones that say the situation: a word in more than a fiftieth of the prompts is
|
|
369
|
-
noise, and a word no prompt ever used cannot route. The tool is fixed for the run, so a failure
|
|
370
|
-
that surfaces in a later turn of the same run still finds it.
|
|
179
|
+
It does not compare two runs of the same task. The comparison it makes is across the sessions of this project, which are different tasks. The same task run twice is what a benchmark does, and what a working session does not.
|
|
371
180
|
|
|
372
|
-
##
|
|
181
|
+
## Install
|
|
373
182
|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
part that says whether the part held.
|
|
183
|
+
```bash
|
|
184
|
+
pi install npm:@tduyng/pi-harness-loop
|
|
185
|
+
```
|
|
378
186
|
|
|
379
|
-
|
|
380
|
-
of this project, which are different tasks. The same task run twice is what a benchmark does,
|
|
381
|
-
and what a working session does not.
|
|
187
|
+
Local checkout: `pi install /path/to/dotfiles/pi/agent/extensions/pi-harness-loop`
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tduyng/pi-harness-loop",
|
|
3
|
-
"version": "0.0.
|
|
4
|
-
"description": "Reads what the
|
|
3
|
+
"version": "0.0.3",
|
|
4
|
+
"description": "Reads what the sessions repeated, writes the skill or rule that would have prevented it, and takes it back out when the rate does not drop.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi",
|
|
7
7
|
"pi-package",
|
package/src/check.ts
CHANGED
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from 'node:fs'
|
|
2
2
|
import { join } from 'node:path'
|
|
3
3
|
|
|
4
|
-
import { createLocalBashOperations } from '@earendil-works/pi-coding-agent'
|
|
4
|
+
import { createLocalBashOperations, truncateTail } from '@earendil-works/pi-coding-agent'
|
|
5
5
|
import { Validator } from 'ata-validator'
|
|
6
6
|
import { t } from 'ata-validator/t'
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
import { runCheck as runSharedCheck, type ExecFn } from '../../pi-goal/src/check.js'
|
|
10
|
-
|
|
11
8
|
export interface CheckResult {
|
|
12
9
|
command: string
|
|
13
10
|
ok: boolean
|
|
@@ -22,6 +19,13 @@ const PackageJson = t.object({
|
|
|
22
19
|
})
|
|
23
20
|
const isPackageJson = new Validator(PackageJson).isValidObject
|
|
24
21
|
|
|
22
|
+
const TEST_BY_LOCKFILE = [
|
|
23
|
+
['bun.lock', 'bun run test'],
|
|
24
|
+
['bun.lockb', 'bun run test'],
|
|
25
|
+
['pnpm-lock.yaml', 'pnpm test'],
|
|
26
|
+
['yarn.lock', 'yarn test']
|
|
27
|
+
] as const satisfies readonly (readonly [string, string])[]
|
|
28
|
+
|
|
25
29
|
function packageTest(cwd: string): string {
|
|
26
30
|
const path = join(cwd, 'package.json')
|
|
27
31
|
if (!existsSync(path)) return ''
|
|
@@ -33,7 +37,8 @@ function packageTest(cwd: string): string {
|
|
|
33
37
|
}
|
|
34
38
|
if (!isPackageJson(parsed)) return ''
|
|
35
39
|
if (parsed.scripts?.['test'] === undefined) return ''
|
|
36
|
-
|
|
40
|
+
const lockfile = TEST_BY_LOCKFILE.find(([file]) => existsSync(join(cwd, file)))
|
|
41
|
+
return lockfile?.[1] ?? 'npm test'
|
|
37
42
|
}
|
|
38
43
|
|
|
39
44
|
function justTest(cwd: string): string {
|
|
@@ -54,14 +59,59 @@ export function failedCount(output: string): number | null {
|
|
|
54
59
|
return generic?.[1] === undefined ? null : Number(generic[1])
|
|
55
60
|
}
|
|
56
61
|
|
|
57
|
-
|
|
62
|
+
export function failureNames(output: string): string[] | null {
|
|
63
|
+
const names = [...output.matchAll(/^\s*(?:\(fail\)|✕|×|✗)\s+(.+)$/gm)]
|
|
64
|
+
.map((match) => (match[1] ?? '').trim())
|
|
65
|
+
.filter((name) => name !== '')
|
|
66
|
+
if (names.length > 0) return names.toSorted()
|
|
67
|
+
return failedCount(output) === 0 ? [] : null
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const MAX_CHECK_OUTPUT_BYTES = 1500
|
|
71
|
+
const DEFAULT_CHECK_TIMEOUT_MS = 300_000
|
|
72
|
+
|
|
73
|
+
async function runSharedCheck(
|
|
74
|
+
cwd: string,
|
|
75
|
+
command: string,
|
|
76
|
+
options: { timeoutMs?: number } = {}
|
|
77
|
+
): Promise<{ ok: boolean; exitCode: number | null; durationMs: number; output: string }> {
|
|
78
|
+
const timeoutMs = options.timeoutMs && options.timeoutMs > 0 ? options.timeoutMs : DEFAULT_CHECK_TIMEOUT_MS
|
|
79
|
+
let captured = Buffer.alloc(0)
|
|
80
|
+
const started = Date.now()
|
|
81
|
+
const finish = (exitCode: number | null, error?: string) => {
|
|
82
|
+
const durationMs = Date.now() - started
|
|
83
|
+
const output = captured.toString('utf8')
|
|
84
|
+
const body = error ? `${output}${output ? '\n' : ''}${error}` : output
|
|
85
|
+
const tail = truncateTail(body, { maxBytes: MAX_CHECK_OUTPUT_BYTES })
|
|
86
|
+
return {
|
|
87
|
+
ok: exitCode === 0,
|
|
88
|
+
exitCode,
|
|
89
|
+
durationMs,
|
|
90
|
+
output: tail.content.trim()
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
let exitCode: number | null
|
|
94
|
+
try {
|
|
95
|
+
;({ exitCode } = await createLocalBashOperations().exec(command, cwd, {
|
|
96
|
+
onData: (data) => {
|
|
97
|
+
const combined = Buffer.concat([captured, data])
|
|
98
|
+
captured = combined.subarray(Math.max(0, combined.length - MAX_CHECK_OUTPUT_BYTES))
|
|
99
|
+
},
|
|
100
|
+
timeout: timeoutMs / 1000
|
|
101
|
+
}))
|
|
102
|
+
} catch (error) {
|
|
103
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
104
|
+
return finish(null, message)
|
|
105
|
+
}
|
|
106
|
+
return finish(exitCode)
|
|
107
|
+
}
|
|
58
108
|
|
|
59
109
|
export async function runCheck(
|
|
60
110
|
cwd: string,
|
|
61
111
|
command: string,
|
|
62
112
|
options: { timeoutMs?: number } = {}
|
|
63
113
|
): Promise<CheckResult> {
|
|
64
|
-
const result = await runSharedCheck(
|
|
114
|
+
const result = await runSharedCheck(cwd, command, options)
|
|
65
115
|
return {
|
|
66
116
|
command,
|
|
67
117
|
ok: result.ok,
|
package/src/diversity.ts
CHANGED
|
@@ -9,7 +9,7 @@ export interface ProposalText {
|
|
|
9
9
|
export function proposalTexts(harnessDir: string): ProposalText[] {
|
|
10
10
|
if (!existsSync(harnessDir)) return []
|
|
11
11
|
const out: ProposalText[] = []
|
|
12
|
-
for (const name of readdirSync(harnessDir)) {
|
|
12
|
+
for (const name of readdirSync(harnessDir).toSorted()) {
|
|
13
13
|
const dir = join(harnessDir, name)
|
|
14
14
|
let entries: string[]
|
|
15
15
|
try {
|
package/src/install.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { dirname, join } from 'node:path'
|
|
|
4
4
|
|
|
5
5
|
import { applyProposal, readProposal, revertApplied } from './apply.js'
|
|
6
6
|
import { readScan } from './auto.js'
|
|
7
|
-
import { discoverCheck, runCheck } from './check.js'
|
|
7
|
+
import { discoverCheck, failureNames, runCheck } from './check.js'
|
|
8
8
|
import { runReplay } from './replay.js'
|
|
9
9
|
import { acquireLock, appendInstall, readInstalls, readSettings } from './store.js'
|
|
10
10
|
import { loadHarness } from './tree.js'
|
|
@@ -270,15 +270,26 @@ async function installLocked(
|
|
|
270
270
|
key: options.key,
|
|
271
271
|
cost: advertisedChars(applied)
|
|
272
272
|
}
|
|
273
|
-
if (check === '')
|
|
273
|
+
if (check === '')
|
|
274
|
+
return finalize(cwd, dir, install, context, `${refusal}, installed without a check`, options.command)
|
|
274
275
|
|
|
275
276
|
const result = await runCheck(cwd, check)
|
|
276
277
|
if (result.ok) return finalize(cwd, dir, install, context, refusal, options.command)
|
|
277
278
|
|
|
278
279
|
revertApplied(applied, installRoots(context), true)
|
|
279
280
|
const baseline = await runCheck(cwd, check)
|
|
281
|
+
const resultNames = failureNames(result.output)
|
|
282
|
+
const baselineNames = failureNames(baseline.output)
|
|
283
|
+
const sameFailure =
|
|
284
|
+
result.output === baseline.output ||
|
|
285
|
+
(resultNames !== null && baselineNames !== null && resultNames.every((name) => baselineNames.includes(name)))
|
|
280
286
|
const innocent =
|
|
281
|
-
!baseline.ok &&
|
|
287
|
+
!baseline.ok &&
|
|
288
|
+
result.failures !== null &&
|
|
289
|
+
result.failures > 0 &&
|
|
290
|
+
baseline.failures !== null &&
|
|
291
|
+
result.failures <= baseline.failures &&
|
|
292
|
+
sameFailure
|
|
282
293
|
if (innocent) {
|
|
283
294
|
for (const file of applied) {
|
|
284
295
|
mkdirSync(dirname(file.to), { recursive: true })
|
package/src/store.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
2
|
+
import { randomUUID } from 'node:crypto'
|
|
2
3
|
import { dirname, join } from 'node:path'
|
|
3
4
|
|
|
4
5
|
import { Validator } from 'ata-validator'
|
|
5
6
|
import { t } from 'ata-validator/t'
|
|
6
7
|
|
|
7
|
-
|
|
8
8
|
import { writeFileAtomic } from './atomic.js'
|
|
9
9
|
import type { HarnessSettings, Install, Lesson } from './types.js'
|
|
10
10
|
|
|
@@ -243,9 +243,10 @@ const LOCK_STALE_MS = 5 * 60_000
|
|
|
243
243
|
export function acquireLock(cwd: string): (() => void) | null {
|
|
244
244
|
const path = join(harnessDir(cwd), 'lock')
|
|
245
245
|
mkdirSync(dirname(path), { recursive: true })
|
|
246
|
+
const token = `${process.pid} ${Date.now()} ${randomUUID()}\n`
|
|
246
247
|
const take = (): boolean => {
|
|
247
248
|
try {
|
|
248
|
-
writeFileSync(path,
|
|
249
|
+
writeFileSync(path, token, { encoding: 'utf8', flag: 'wx' })
|
|
249
250
|
return true
|
|
250
251
|
} catch {
|
|
251
252
|
return false
|
|
@@ -253,12 +254,17 @@ export function acquireLock(cwd: string): (() => void) | null {
|
|
|
253
254
|
}
|
|
254
255
|
if (!take()) {
|
|
255
256
|
try {
|
|
256
|
-
const
|
|
257
|
+
const existing = readFileSync(path, 'utf8')
|
|
258
|
+
const at = Number(existing.split(' ')[1])
|
|
257
259
|
if (Number.isFinite(at) && Date.now() - at > LOCK_STALE_MS) rmSync(path, { force: true })
|
|
258
260
|
} catch {}
|
|
259
261
|
if (!take()) return null
|
|
260
262
|
}
|
|
261
|
-
return () =>
|
|
263
|
+
return () => {
|
|
264
|
+
try {
|
|
265
|
+
if (readFileSync(path, 'utf8') === token) rmSync(path, { force: true })
|
|
266
|
+
} catch {}
|
|
267
|
+
}
|
|
262
268
|
}
|
|
263
269
|
|
|
264
270
|
export function recordLessonUse(cwd: string, id: string): void {
|