@mjasnikovs/pi-task 0.38.22 → 0.38.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/reasoning-args.d.ts +12 -1
- package/dist/config/reasoning-args.js +5 -2
- package/dist/config/reasoning.d.ts +60 -2
- package/dist/config/reasoning.js +344 -37
- package/dist/config/register.d.ts +76 -32
- package/dist/config/register.js +124 -82
- package/dist/shared/reasoning-capability.d.ts +2 -5
- package/dist/shared/reasoning-capability.js +31 -4
- package/dist/task/auto-orchestrator.d.ts +2 -0
- package/dist/task/auto-orchestrator.js +28 -41
- package/dist/task/child-runner.d.ts +89 -24
- package/dist/task/child-runner.js +67 -46
- package/dist/task/orchestrator.d.ts +14 -20
- package/dist/task/orchestrator.js +12 -9
- package/dist/task/phases.d.ts +16 -23
- package/dist/task/phases.js +47 -452
- package/dist/task/question-dialog.d.ts +56 -0
- package/dist/task/question-dialog.js +53 -0
- package/dist/task/research-worker.d.ts +180 -0
- package/dist/task/research-worker.js +432 -0
- package/dist/workers/brave-warning.js +4 -30
- package/dist/workers/docs-core.d.ts +8 -4
- package/dist/workers/docs-core.js +30 -21
- package/dist/workers/docs-lookup.d.ts +72 -0
- package/dist/workers/docs-lookup.js +53 -0
- package/dist/workers/docs-project.d.ts +9 -0
- package/dist/workers/docs-project.js +15 -0
- package/dist/workers/pi-worker-core.d.ts +87 -1
- package/dist/workers/pi-worker-core.js +3 -7
- package/dist/workers/pi-worker-docs.js +27 -31
- package/dist/workers/reasoning-warning.d.ts +10 -16
- package/dist/workers/reasoning-warning.js +25 -57
- package/dist/workers/session-hint.d.ts +37 -0
- package/dist/workers/session-hint.js +82 -0
- package/dist/workers/worker-failure.d.ts +34 -0
- package/dist/workers/worker-failure.js +27 -16
- package/dist/workers/worker-kill.d.ts +84 -0
- package/dist/workers/worker-kill.js +124 -0
- package/package.json +1 -1
- package/dist/task/reasoning-groups.d.ts +0 -36
- package/dist/task/reasoning-groups.js +0 -36
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
[](https://www.npmjs.com/package/@mjasnikovs/pi-task)
|
|
10
10
|
[](./LICENSE)
|
|
11
11
|
[](https://www.npmjs.com/package/@earendil-works/pi-coding-agent)
|
|
12
|
-
[](#development)
|
|
13
13
|
[](./tsconfig.json)
|
|
14
14
|
|
|
15
15
|
</div>
|
|
@@ -243,7 +243,7 @@ them checked in.
|
|
|
243
243
|
|
|
244
244
|
```sh
|
|
245
245
|
bun install
|
|
246
|
-
bun run test #
|
|
246
|
+
bun run test # 4321 tests across 242 files
|
|
247
247
|
bun run lint # prettier + eslint + tsc --noEmit
|
|
248
248
|
bun run build # tsc → dist/
|
|
249
249
|
```
|
|
@@ -1,3 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The live-config bridge for reasoning profiles: group in, argv fragment out.
|
|
3
|
+
*
|
|
4
|
+
* Separate from reasoning.ts because that module must stay import-free — see its
|
|
5
|
+
* header. This file is the one hop that reads `getConfig()`, so it imports both
|
|
6
|
+
* and neither imports it back: a tree, not a cycle.
|
|
7
|
+
*
|
|
8
|
+
* Read PER CALL, never cached at module scope, so a /task-config change lands on
|
|
9
|
+
* the next child without a restart — the same contract childBaseArgs states.
|
|
10
|
+
*/
|
|
11
|
+
import { type PiTaskConfig } from './config.js';
|
|
1
12
|
import { type ReasoningGroup } from './reasoning.js';
|
|
2
13
|
/**
|
|
3
14
|
* The `['--thinking', level]` fragment for a group, or `[]` when the group is
|
|
@@ -7,4 +18,4 @@ import { type ReasoningGroup } from './reasoning.js';
|
|
|
7
18
|
* themselves — an argv builder that resolves its own policy is one that cannot
|
|
8
19
|
* be told to do something else, which is how childBaseArgs became universal.
|
|
9
20
|
*/
|
|
10
|
-
export declare function groupThinkingArgs(group: ReasoningGroup): string[];
|
|
21
|
+
export declare function groupThinkingArgs(group: ReasoningGroup, cfg?: PiTaskConfig): string[];
|
|
@@ -18,6 +18,9 @@ import { resolveReasoning, thinkingArgs } from './reasoning.js';
|
|
|
18
18
|
* themselves — an argv builder that resolves its own policy is one that cannot
|
|
19
19
|
* be told to do something else, which is how childBaseArgs became universal.
|
|
20
20
|
*/
|
|
21
|
-
export function groupThinkingArgs(group) {
|
|
22
|
-
|
|
21
|
+
export function groupThinkingArgs(group, cfg) {
|
|
22
|
+
// The default is EVALUATED HERE, per call, which is the contract this module
|
|
23
|
+
// exists to keep. Hoisting the read to module scope would leave every test
|
|
24
|
+
// green, so the parameter is what makes the contract assertable at all.
|
|
25
|
+
return thinkingArgs(resolveReasoning(group, cfg ?? getConfig()));
|
|
23
26
|
}
|
|
@@ -31,7 +31,16 @@ import type { PiTaskConfig } from './config.js';
|
|
|
31
31
|
* thinking, while two that reach the model by different code paths (`refine` via
|
|
32
32
|
* runPhaseChild, `compress-label` via the same) want the same.
|
|
33
33
|
*
|
|
34
|
-
* - `research` the
|
|
34
|
+
* - `research` the ad-hoc `pi-worker` subagent tool, and the FALLBACK the four
|
|
35
|
+
* research workers use when their own cell is unset
|
|
36
|
+
* - `research:files` / `research:apis` / `research:context` / `research:tooling`
|
|
37
|
+
* one cell per research worker. They are FOUR CELLS, not one,
|
|
38
|
+
* because the four wander differently: over mx5-n 2026-08-27
|
|
39
|
+
* every one of the 40.7 wasted research minutes was a restart
|
|
40
|
+
* in `tooling` or `context`, and `files` and `apis` never
|
|
41
|
+
* restarted once. A single `research` cell cannot be set to
|
|
42
|
+
* pay for thinking where it is needed without also paying for
|
|
43
|
+
* it where it measurably is not.
|
|
35
44
|
* - `phase` refine, verify-tooling, grill, compose, critique, compress-label
|
|
36
45
|
* - `planning` /task-auto's clarify / decompose / extract children
|
|
37
46
|
* - `plan` /task-plan's question and answer children
|
|
@@ -39,7 +48,7 @@ import type { PiTaskConfig } from './config.js';
|
|
|
39
48
|
* - `extraction` the --no-tools focused docs/fetch extractors
|
|
40
49
|
* - `implementation` the host-session turn that writes the code (not a child)
|
|
41
50
|
*/
|
|
42
|
-
export type ReasoningGroup = 'research' | 'phase' | 'planning' | 'plan' | 'gate' | 'extraction' | 'implementation';
|
|
51
|
+
export type ReasoningGroup = 'research' | 'research:files' | 'research:apis' | 'research:context' | 'research:tooling' | 'phase' | 'planning' | 'plan' | 'gate' | 'extraction' | 'implementation';
|
|
43
52
|
export declare const REASONING_GROUPS: readonly ReasoningGroup[];
|
|
44
53
|
/**
|
|
45
54
|
* The four profiles offered by /task-config.
|
|
@@ -132,6 +141,42 @@ export declare const DEFAULT_REASONING_TABLE: Readonly<Record<ReasoningGroup, Gr
|
|
|
132
141
|
* would see "custom" in the file and the default table in behaviour.
|
|
133
142
|
*/
|
|
134
143
|
export declare function sanitizeReasoningMode(value: unknown): ReasoningMode;
|
|
144
|
+
/**
|
|
145
|
+
* Child NAME → reasoning group, for every child that goes through
|
|
146
|
+
* `runPhaseChild` / `runPlanningChild`.
|
|
147
|
+
*
|
|
148
|
+
* WHY KEYED ON THE NAME
|
|
149
|
+
* ---------------------
|
|
150
|
+
* The name is the only identifier in scope at all three spawn paths (phases,
|
|
151
|
+
* /task-auto planning, /task-plan), it is what the loader and the debug trail
|
|
152
|
+
* already print, and it is the one thing a reader can check against the phase
|
|
153
|
+
* list without following the call graph. Threading a group parameter through
|
|
154
|
+
* `PhaseDeps` / `AutoDeps` instead would touch both orchestrators' dep bags to
|
|
155
|
+
* express something the call site already says out loud.
|
|
156
|
+
*
|
|
157
|
+
* AN UNMAPPED NAME IS A BUILD FAILURE, not a silent `inherit`.
|
|
158
|
+
* `reasoning-groups.test.ts` scans every literal child name in src/ and fails if
|
|
159
|
+
* it is missing here. A defaulting lookup would let a phase added next year opt
|
|
160
|
+
* itself out of a measured setting without anyone deciding to — which is exactly
|
|
161
|
+
* how `/no_think` ended up applied to eight prompts and read by none of them.
|
|
162
|
+
*
|
|
163
|
+
* The gate and extraction groups are NOT here: those children reach the model
|
|
164
|
+
* through `runWorker` / `focusedChildArgs` at a site with no name in scope, so
|
|
165
|
+
* the group is passed directly. The four RESEARCH workers do have a name — their
|
|
166
|
+
* `spec.label` — and so they are here.
|
|
167
|
+
*/
|
|
168
|
+
export declare const REASONING_GROUP_BY_CHILD: Readonly<Record<string, ReasoningGroup>>;
|
|
169
|
+
/**
|
|
170
|
+
* The group a named child belongs to.
|
|
171
|
+
*
|
|
172
|
+
* Returns `undefined` for a name the table does not know, and the CALLER decides
|
|
173
|
+
* what that means. `runPhaseChild` treats it as `inherit` — a child that reaches
|
|
174
|
+
* the model with today's argv is always safe — while the test treats it as a
|
|
175
|
+
* failure. That split is deliberate: the guard belongs at build time, where
|
|
176
|
+
* someone can fix it, not at run time, where it would abort a user's task over a
|
|
177
|
+
* missing table row.
|
|
178
|
+
*/
|
|
179
|
+
export declare function reasoningGroupForChild(name: string): ReasoningGroup | undefined;
|
|
135
180
|
/**
|
|
136
181
|
* Always returns a COMPLETE record, never a partial one.
|
|
137
182
|
*
|
|
@@ -150,6 +195,19 @@ export declare function sanitizeReasoningLevels(value: unknown): Record<Reasonin
|
|
|
150
195
|
* `groupThinkingArgs` from reasoning-args.ts.
|
|
151
196
|
*/
|
|
152
197
|
export declare function resolveReasoning(group: ReasoningGroup, cfg: PiTaskConfig): GroupSetting;
|
|
198
|
+
/**
|
|
199
|
+
* The WHOLE table, as this config will actually run it.
|
|
200
|
+
*
|
|
201
|
+
* This is the question every caller has — the settings menu when it repaints,
|
|
202
|
+
* the mismatch warning when it scans, the custom-mode seeder when it freezes the
|
|
203
|
+
* table — and each of them used to write the same loop over `resolveReasoning`.
|
|
204
|
+
* A per-key accessor with no whole-table companion is also how a THIRD shape got
|
|
205
|
+
* invented (`Array<{group, setting}>`) and leaked into `reasoningMismatches`.
|
|
206
|
+
*
|
|
207
|
+
* `resolveReasoning` stays: one group is still a fair question, and it is the
|
|
208
|
+
* only place the four modes are interpreted.
|
|
209
|
+
*/
|
|
210
|
+
export declare function effectiveReasoning(cfg: PiTaskConfig): Record<ReasoningGroup, GroupSetting>;
|
|
153
211
|
/**
|
|
154
212
|
* The argv fragment for a setting. `inherit` is the empty fragment — no flag at
|
|
155
213
|
* all — which is what makes an all-`inherit` config byte-identical to the
|
package/dist/config/reasoning.js
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
export const REASONING_GROUPS = [
|
|
2
2
|
'research',
|
|
3
|
+
'research:files',
|
|
4
|
+
'research:apis',
|
|
5
|
+
'research:context',
|
|
6
|
+
'research:tooling',
|
|
3
7
|
'phase',
|
|
4
8
|
'planning',
|
|
5
9
|
'plan',
|
|
@@ -182,6 +186,180 @@ export const DEFAULT_REASONING_TABLE = {
|
|
|
182
186
|
// children only), so a rotation is killed ~8 calls into its second lap
|
|
183
187
|
// rather than at the 20-minute ceiling.
|
|
184
188
|
research: 'medium',
|
|
189
|
+
// ── THE FOUR RESEARCH WORKERS, SPLIT OUT OF `research` 2026-08-28 ──
|
|
190
|
+
//
|
|
191
|
+
// WHY THE SPLIT EXISTS, from `research`'s own evidence. The override to
|
|
192
|
+
// `medium` above was aimed at ONE pathology — the restart tail — and that
|
|
193
|
+
// tail is not spread evenly. mx5-n 2026-08-27: 40.7 of the research phase's
|
|
194
|
+
// 81.4 wall-clock minutes were lost to restarts, ALL of them in
|
|
195
|
+
// `worker:tooling` and `worker:context`; `worker:files` and `worker:apis`
|
|
196
|
+
// never restarted once. With one cell, paying for thinking in the two that
|
|
197
|
+
// wander means paying for it in the two that do not.
|
|
198
|
+
//
|
|
199
|
+
// ── research:files — MEASURED, `off`. WRITTEN 2026-08-28 ──
|
|
200
|
+
//
|
|
201
|
+
// A/B 2026-08-26, Qwen3.8-27B-NVFP4-MTP-VERY-HIGH.gguf (llama.cpp
|
|
202
|
+
// b10620-0f3b51e03), scripts/live-reasoning-group-ab.ts, n=12/arm over 12
|
|
203
|
+
// distinct mx5 tasks. off 10/12 vs medium 11/12 (p=1.0000); clock PAIRED
|
|
204
|
+
// over the 10 usable pairs p=0.7090, mean off 79.4s vs medium 59.3s.
|
|
205
|
+
// off 95% CI [0.55, 0.95]. RUNG 3 — a DECISION, not a finding.
|
|
206
|
+
//
|
|
207
|
+
// THIS IS NOT A NEW RUN. It is ledger-research.jsonl, rescored under the
|
|
208
|
+
// group's new name:
|
|
209
|
+
//
|
|
210
|
+
// AB_CORPUS=/home/edgars/hub/ab-grouplab/mx5-copy bun run \
|
|
211
|
+
// scripts/rescore-reasoning-ledger.ts \
|
|
212
|
+
// /home/edgars/hub/ab-grouplab/ledger-research.jsonl research:files --from-text
|
|
213
|
+
//
|
|
214
|
+
// That ledger was ALWAYS a FILES worker — `child: 'worker:files'`, prompt
|
|
215
|
+
// `RESEARCH_FILES_PROMPT`, run in each task's own before-tree — so the split
|
|
216
|
+
// did not strand it, it named it. The axis is the CONJUNCTION: every path
|
|
217
|
+
// named is real AND every pre-existing file the task edited is named. The
|
|
218
|
+
// full derivation, the stimulus screen, the parser fix and the confound sit
|
|
219
|
+
// on the `research` cell above; they are that ledger's, and this is that
|
|
220
|
+
// ledger.
|
|
221
|
+
//
|
|
222
|
+
// WHY THIS CELL DOES NOT FOLLOW `research`'s OVERRIDE. That override was
|
|
223
|
+
// aimed at the restart tail, and the tail is not here: `worker:files` never
|
|
224
|
+
// restarted once. The reason for paying does not apply to this worker, and
|
|
225
|
+
// its own ledger says the arms tied.
|
|
226
|
+
'research:files': 'off',
|
|
227
|
+
// NOT MEASURED. DECIDED BY PRIOR: a worker with no trial of its own keeps
|
|
228
|
+
// whatever `research` was decided to run at. No trial in this repo has ever
|
|
229
|
+
// run apis, context or tooling as its own arm, so a cell here that differed
|
|
230
|
+
// from `research` would be intuition wearing the authority of a default.
|
|
231
|
+
//
|
|
232
|
+
// AND AN AXIS IS THE HARD PART, not the GPU. Two candidates were screened
|
|
233
|
+
// offline and DIED there — scripts/research-worker-axes-step0.ts, no model,
|
|
234
|
+
// no GPU. For APIS: "every dotted symbol named is present in the tree"
|
|
235
|
+
// scores the recorded answers 28/28 tasks and 81/81 items. SATURATED, and
|
|
236
|
+
// loose with it — the check greps the symbol's LAST SEGMENT, so
|
|
237
|
+
// `Hono.c.json` passes on the word `json`. Tightening it has nothing to
|
|
238
|
+
// bite on, because APIS names are model-composed pseudo-symbols
|
|
239
|
+
// (`Hono.c.var`, `UUID regex`); that is the same wall
|
|
240
|
+
// [[apis-contract-stage3-refuted]] hit from the other side.
|
|
241
|
+
// TO REPLACE THIS WITH A MEASUREMENT, the axis must be an EXECUTION or a
|
|
242
|
+
// production adjudication, the way gate's and planning's are — not a
|
|
243
|
+
// property of the text.
|
|
244
|
+
//
|
|
245
|
+
// A COST RUN WAS DONE ANYWAY, 2026-08-28, and it found nothing to act on.
|
|
246
|
+
// n=20/arm on the same corpus and trees, production's tools, extensions,
|
|
247
|
+
// search hint and the task's own recorded FILES map: off 19/20 answered vs
|
|
248
|
+
// medium 20/20, and the PAIRED clock over 19 stimuli is off 127.2s vs
|
|
249
|
+
// medium 144.4s, p=0.5262. So thinking is neither better nor cheaper here
|
|
250
|
+
// by anything this run can see, and there is no cost argument for moving
|
|
251
|
+
// the cell either way. Ledger: ledger-research-apis.jsonl.
|
|
252
|
+
// THAT RUN CANNOT WRITE THIS CELL. Its axis is TERMINATION — "did it answer
|
|
253
|
+
// at all" — so its `off [rung 3]` line is the standing prior speaking with
|
|
254
|
+
// no quality reading behind it. See README-research-cost-runs.txt.
|
|
255
|
+
// ITS BUILD IS b10665-ca3d5a3e1, not the b10620-0f3b51e03 every other cell
|
|
256
|
+
// was measured on: llama.cpp was rebuilt mid-session. Internally paired, so
|
|
257
|
+
// off-vs-medium is fair within it; its SECONDS are not comparable to the
|
|
258
|
+
// files or tooling ledgers.
|
|
259
|
+
'research:apis': 'medium',
|
|
260
|
+
// NOT MEASURED. DECIDED BY PRIOR — see the block above `research:apis`.
|
|
261
|
+
// Its candidate axis died offline too: "every backticked project path in a
|
|
262
|
+
// CONTEXT bullet exists in the tree" scores the recorded answers 6/53 tasks
|
|
263
|
+
// and 213/391 items. THE CHECK LOSES, on exactly the residue the phase
|
|
264
|
+
// path-axis audit catalogued — npm specifiers (`@hono/zod-validator`),
|
|
265
|
+
// import paths (`hono/cookie`, `../server/auth`), bare filenames
|
|
266
|
+
// (`schema.ts`) and `src/` prefix elision.
|
|
267
|
+
// TO REPLACE THIS WITH A MEASUREMENT, note that this worker is one of the
|
|
268
|
+
// two the restart tail lives in, so the honest axis is the wander itself —
|
|
269
|
+
// and no Trial field records a tool-call count today.
|
|
270
|
+
//
|
|
271
|
+
// A COST RUN WAS DONE ANYWAY, 2026-08-28, n=20/arm, production's `read,grep`
|
|
272
|
+
// and no extensions. It found no clock difference — medians 81.2s vs 80.8s,
|
|
273
|
+
// paired p=0.8594 over 16 stimuli — but it did see the wander directly:
|
|
274
|
+
//
|
|
275
|
+
// died in a loop off 3/20 vs medium 0/20 p=0.2308
|
|
276
|
+
// emitted bullets off 17/20 vs medium 20/20 p=0.2308
|
|
277
|
+
//
|
|
278
|
+
// Neither reaches significance at n=20, and the second is TERMINATION, not
|
|
279
|
+
// quality. But this is the first time the restart tail has appeared INSIDE
|
|
280
|
+
// a measured run rather than in a production log, and it appeared only in
|
|
281
|
+
// the `off` arm. off's higher MEAN (97.8s vs 69.3s) is those three deaths,
|
|
282
|
+
// not slower work. THIS RUN CANNOT WRITE THIS CELL either: an underpowered
|
|
283
|
+
// termination signal is not a measurement, and its `off [rung 3]` verdict
|
|
284
|
+
// line is the prior, not a reading. THE CELL IS UNCHANGED. Ledger: ledger-research-context.jsonl,
|
|
285
|
+
// README-research-cost-runs.txt, build b10665-ca3d5a3e1.
|
|
286
|
+
//
|
|
287
|
+
// INSTRUMENT NOTE, because that run ABSTAINED first with 40 good answers
|
|
288
|
+
// already stored. It scored CONTEXT with `hasAnswerContent` — the FILES
|
|
289
|
+
// `name<gap>description` shape — and this worker emits a BULLET list of
|
|
290
|
+
// prose sentences, which `isEntryLine` rejects for ending in a full stop.
|
|
291
|
+
// Both arms read 0/20. Fixed by `contextEmittedBullets` and the stored
|
|
292
|
+
// trials rescored with no GPU. Fourth instance of
|
|
293
|
+
// [[ab-scorer-must-match-the-real-prompt]] in one harness.
|
|
294
|
+
// AND THE REPLACEMENT WAS WRONG BY ONE, found in review the same day: it
|
|
295
|
+
// asked for >=2 bullets where production's `classifyContextSilence` calls
|
|
296
|
+
// >=1 PRODUCTIVE, so a one-bullet answer production accepts read UNUSABLE
|
|
297
|
+
// in both arms. `countBullets` — production's own function, called not
|
|
298
|
+
// restated — moved 37/40 stored trials and the emitted-bullets line above
|
|
299
|
+
// from off 16/20 to off 17/20. The verdict, the cell and the clock are
|
|
300
|
+
// unchanged: still an underpowered termination signal, still a prior.
|
|
301
|
+
'research:context': 'medium',
|
|
302
|
+
// ── research:tooling — MEASURED, `medium`. RUNG 1. WRITTEN 2026-08-28 ──
|
|
303
|
+
//
|
|
304
|
+
// A/B 2026-08-28, Qwen3.8-27B-NVFP4-MTP-VERY-HIGH.gguf (llama.cpp
|
|
305
|
+
// b10620-0f3b51e03), scripts/live-reasoning-group-ab.ts, n=20/arm over 20
|
|
306
|
+
// distinct mx5 tasks, one rep each so the clock can pair.
|
|
307
|
+
//
|
|
308
|
+
// every command runs off 13/20 vs medium 20/20 p=0.0083
|
|
309
|
+
// wall clock (paired) off 49.5s vs medium 15.9s p=0.0400
|
|
310
|
+
// off 95% CI [0.43, 0.82]. Ledger: ledger-research-tooling.jsonl.
|
|
311
|
+
//
|
|
312
|
+
// RUNG 1 — quality decided it, and this is only the SECOND cell in the table
|
|
313
|
+
// a quality axis has ever decided. THE FIRST WAS extraction, which went the
|
|
314
|
+
// other way. Note there is no trade here: medium is both better AND 3.1x
|
|
315
|
+
// faster, so the sampler caveat below cannot be what produced it.
|
|
316
|
+
//
|
|
317
|
+
// THE AXIS: every command the checker can adjudicate must RESOLVE in the
|
|
318
|
+
// tree the worker inspected (scripts/reasoning-ab-tooling-truth.ts). Not a
|
|
319
|
+
// property of the text — the same kind of truth as gate's executed VERIFY,
|
|
320
|
+
// which is why it lives where four `phase` text axes died. `unknown` is a
|
|
321
|
+
// first-class answer and is scored neither way: three of the four command
|
|
322
|
+
// shapes can only return `real` or `unknown`, because an undeclared binary
|
|
323
|
+
// may still be on PATH and `bun test` resolves through the runtime.
|
|
324
|
+
//
|
|
325
|
+
// OFF'S FAILURE IS ONE CLASS, AUDITED ROW BY ROW. All seven name a
|
|
326
|
+
// dev-server command against `src/server/index.ts` — a file that exists in
|
|
327
|
+
// NEITHER the before-tree nor the after-tree of any of those seven tasks;
|
|
328
|
+
// it first appears around TASK_0057. The worker's own prompt says "If a
|
|
329
|
+
// tool isn't present in the repo, omit it — don't invent." Medium omits it.
|
|
330
|
+
//
|
|
331
|
+
// THE CHECKER WAS WRONG THREE TIMES FIRST, and all three are pinned in
|
|
332
|
+
// reasoning-ab-tooling-truth.test.ts:
|
|
333
|
+
// 1. `bun run <file.ts>` is a file invocation, not a script lookup.
|
|
334
|
+
// Caught offline; four of twelve reported failures were this bug.
|
|
335
|
+
// 2. `bun run <installed-bin>` falls back to `node_modules/.bin`.
|
|
336
|
+
// VERIFIED BY EXECUTION: `bun run tsc --version` → `Version 6.0.3`,
|
|
337
|
+
// exit 0; `bun run dev` → `error: Script not found "dev"`, exit 1.
|
|
338
|
+
// Caught LIVE at trial 19. It cost two MEDIUM trials, and BOTH were
|
|
339
|
+
// recovered by rescoring the stored text with no GPU — which is the
|
|
340
|
+
// whole reason the ledger stores `output`.
|
|
341
|
+
// 3. `-f` is a compose file only when a compose token precedes it, and
|
|
342
|
+
// `<runner> run <arg>` must be read for EVERY segment of a compound
|
|
343
|
+
// line, not the first. Found in review: `curl -f <url>` and
|
|
344
|
+
// `git clean -f -d` scored as invented paths (strict), while
|
|
345
|
+
// `bun run lint && bun run dev` scored `real` on `lint` alone and let
|
|
346
|
+
// `dev` — the exact class this cell turns on — through (LOOSE, and the
|
|
347
|
+
// harder half to spot). Caught after the run; rescoring the stored text
|
|
348
|
+
// moved 2/40 trials and left every number on this cell IDENTICAL.
|
|
349
|
+
// No fix moved the offline ceiling (32/45 tasks, 102/118 commands, before
|
|
350
|
+
// and after all three), so none loosened the check against known-good work.
|
|
351
|
+
//
|
|
352
|
+
// WHAT THIS CELL DOES NOT SAY. Medium sits at 20/20, so the axis has no
|
|
353
|
+
// headroom ABOVE it: a future run could not show medium getting worse. And
|
|
354
|
+
// the axis does not score the restart tail this worker is known for — no
|
|
355
|
+
// Trial field records a tool-call count — so "medium is better here" is
|
|
356
|
+
// about the ANSWER, not about the wander.
|
|
357
|
+
//
|
|
358
|
+
// REPRODUCE: `AB_SPECS=research:tooling:20 /abrun/run-group.sh`, then
|
|
359
|
+
// AB_CORPUS=.../mx5-copy bun run scripts/rescore-reasoning-ledger.ts \
|
|
360
|
+
// /home/edgars/hub/ab-grouplab/ledger-research-tooling.jsonl \
|
|
361
|
+
// research:tooling --from-text
|
|
362
|
+
'research:tooling': 'medium',
|
|
185
363
|
// NOT MEASURED. DECIDED BY PRIOR, 2026-08-27 — the same prior that carries
|
|
186
364
|
// every rung-3 cell in this table: thinking that buys nothing measurable is
|
|
187
365
|
// not worth its tokens. `off`.
|
|
@@ -487,39 +665,42 @@ export const DEFAULT_REASONING_TABLE = {
|
|
|
487
665
|
// a stated prior to a measured win. Rescored from the original ledger on 2026-08-25 when the harness
|
|
488
666
|
// moved to a forced two-way verdict; the trials are unchanged.
|
|
489
667
|
//
|
|
490
|
-
//
|
|
491
|
-
//
|
|
492
|
-
//
|
|
493
|
-
//
|
|
494
|
-
//
|
|
495
|
-
//
|
|
496
|
-
//
|
|
497
|
-
//
|
|
498
|
-
//
|
|
499
|
-
//
|
|
500
|
-
//
|
|
501
|
-
//
|
|
502
|
-
//
|
|
503
|
-
//
|
|
504
|
-
//
|
|
505
|
-
//
|
|
506
|
-
//
|
|
507
|
-
//
|
|
508
|
-
//
|
|
509
|
-
// WHAT IS STRUCTURALLY TRUE, and checked: the implementation
|
|
510
|
-
// the USER'S OWN SESSION (see task/implementation-thinking.ts
|
|
511
|
-
// task/implementation-turn.ts), not as a child. It therefore has NO
|
|
668
|
+
// OVERRIDE WITHDRAWN 2026-08-28; THE CELL IS `off` AGAIN, ON ITS OWN
|
|
669
|
+
// MEASUREMENT. Between 2026-08-27 and 2026-08-28 this cell read `medium` by
|
|
670
|
+
// user decision, overriding the rung-2 result above rather than a prior. The
|
|
671
|
+
// decision's stated reason was an implementation turn observed wandering in
|
|
672
|
+
// the mx5-n run, and the cell recorded honestly that the run's .pi-tasks logs
|
|
673
|
+
// had been destroyed before anyone could count the loop.
|
|
674
|
+
//
|
|
675
|
+
// A SECOND mx5-n RUN KEPT ITS LOGS, and they refute the reason. The run
|
|
676
|
+
// executed 10 tasks in 9h22 with this cell at `medium` — thinking ON for
|
|
677
|
+
// every implementation turn — and TASK_0004 wandered anyway: 167 assistant
|
|
678
|
+
// turns, 192 tool calls, 45 files and 2,355 lines committed against a
|
|
679
|
+
// one-file spec, and its own deliverable (src/server/migrate.ts) never
|
|
680
|
+
// written; it arrived four tasks later. Two single responses in that turn
|
|
681
|
+
// were 18,297 and 23,568 output tokens. Thinking did not prevent the
|
|
682
|
+
// rotation, so the lever the override was reaching for is not this one.
|
|
683
|
+
// Measured over the run's 15 implementation sessions: 441,399 output tokens,
|
|
684
|
+
// of which 59.3% by character was thinking (1,314,790 thinking chars vs
|
|
685
|
+
// 902,034 text chars).
|
|
686
|
+
//
|
|
687
|
+
// WHAT IS STRUCTURALLY TRUE, and still true, and checked: the implementation
|
|
688
|
+
// turn runs in the USER'S OWN SESSION (see task/implementation-thinking.ts
|
|
689
|
+
// and task/implementation-turn.ts), not as a child. It therefore has NO
|
|
512
690
|
// LoopDetector and NO StallDetector — only the per-tool-call command
|
|
513
|
-
// watchdog and
|
|
514
|
-
//
|
|
515
|
-
//
|
|
516
|
-
//
|
|
517
|
-
//
|
|
518
|
-
//
|
|
519
|
-
//
|
|
520
|
-
//
|
|
521
|
-
//
|
|
522
|
-
implementation
|
|
691
|
+
// watchdog and superviseImplementation's MAX_COMPACTION_RESUMES, which the
|
|
692
|
+
// runaway above came nowhere near (it took 2 compactions of an allowed 20).
|
|
693
|
+
// So `implementation` remains the one group in this table where a rotation
|
|
694
|
+
// has no guard at all. That is an argument for GIVING it a guard, not for
|
|
695
|
+
// paying 1.8x in thinking that has now been observed not to guard it.
|
|
696
|
+
//
|
|
697
|
+
// THE CELL IS STILL RUNG 2 AND STILL THE WIDEST CI IN THE TABLE ([0.39,
|
|
698
|
+
// 0.78] at 12/20), so it is carried by the clock, not by quality. Nothing
|
|
699
|
+
// measured since has withdrawn the paired-clock win. To put it on more
|
|
700
|
+
// evidence, re-run scripts/live-implementation-thinking-ab.ts at a larger n;
|
|
701
|
+
// the ledger is /home/edgars/hub/ab-implab/impl-ledger.jsonl, 40 rows,
|
|
702
|
+
// b10618 — the odd build out, so a re-run on b10620 is not a replicate.
|
|
703
|
+
implementation: 'off'
|
|
523
704
|
};
|
|
524
705
|
/**
|
|
525
706
|
* A hand-edited or stale mode must not reach {@link resolveReasoning}'s switch as
|
|
@@ -529,6 +710,86 @@ export const DEFAULT_REASONING_TABLE = {
|
|
|
529
710
|
export function sanitizeReasoningMode(value) {
|
|
530
711
|
return REASONING_MODES.includes(value) ? value : 'default';
|
|
531
712
|
}
|
|
713
|
+
/**
|
|
714
|
+
* Child NAME → reasoning group, for every child that goes through
|
|
715
|
+
* `runPhaseChild` / `runPlanningChild`.
|
|
716
|
+
*
|
|
717
|
+
* WHY KEYED ON THE NAME
|
|
718
|
+
* ---------------------
|
|
719
|
+
* The name is the only identifier in scope at all three spawn paths (phases,
|
|
720
|
+
* /task-auto planning, /task-plan), it is what the loader and the debug trail
|
|
721
|
+
* already print, and it is the one thing a reader can check against the phase
|
|
722
|
+
* list without following the call graph. Threading a group parameter through
|
|
723
|
+
* `PhaseDeps` / `AutoDeps` instead would touch both orchestrators' dep bags to
|
|
724
|
+
* express something the call site already says out loud.
|
|
725
|
+
*
|
|
726
|
+
* AN UNMAPPED NAME IS A BUILD FAILURE, not a silent `inherit`.
|
|
727
|
+
* `reasoning-groups.test.ts` scans every literal child name in src/ and fails if
|
|
728
|
+
* it is missing here. A defaulting lookup would let a phase added next year opt
|
|
729
|
+
* itself out of a measured setting without anyone deciding to — which is exactly
|
|
730
|
+
* how `/no_think` ended up applied to eight prompts and read by none of them.
|
|
731
|
+
*
|
|
732
|
+
* The gate and extraction groups are NOT here: those children reach the model
|
|
733
|
+
* through `runWorker` / `focusedChildArgs` at a site with no name in scope, so
|
|
734
|
+
* the group is passed directly. The four RESEARCH workers do have a name — their
|
|
735
|
+
* `spec.label` — and so they are here.
|
|
736
|
+
*/
|
|
737
|
+
export const REASONING_GROUP_BY_CHILD = {
|
|
738
|
+
// ── phase: task/phases.ts + task/title-label.ts ──────────────────────────
|
|
739
|
+
refine: 'phase',
|
|
740
|
+
'verify-tooling': 'phase',
|
|
741
|
+
'grill-auto': 'phase',
|
|
742
|
+
'grill-gen': 'phase',
|
|
743
|
+
compose: 'phase',
|
|
744
|
+
critique: 'phase',
|
|
745
|
+
'critique-triage': 'phase',
|
|
746
|
+
'compress-label': 'phase',
|
|
747
|
+
// ── planning: task/auto-orchestrator.ts ──────────────────────────────────
|
|
748
|
+
'clarify-triage': 'planning',
|
|
749
|
+
'auto-clarify': 'planning',
|
|
750
|
+
'auto-decompose': 'planning',
|
|
751
|
+
'requirement-extract': 'planning',
|
|
752
|
+
'decompose-coverage': 'planning',
|
|
753
|
+
'coverage-map': 'planning',
|
|
754
|
+
'contract-extract': 'planning',
|
|
755
|
+
'launch-extract': 'planning',
|
|
756
|
+
// ── plan: task/plan-orchestrator.ts ──────────────────────────────────────
|
|
757
|
+
'plan-question': 'plan',
|
|
758
|
+
'plan-answer': 'plan',
|
|
759
|
+
// ── research: task/phases.ts `workerSpecs`, keyed on the spec's LABEL ─────
|
|
760
|
+
// These were a second table (`RESEARCH_WORKER_GROUPS`) keyed on the section
|
|
761
|
+
// heading, with a silent `?? 'research'` fallback and a guard that sliced
|
|
762
|
+
// phases.ts source between two string offsets from the config directory.
|
|
763
|
+
// The label is the same name the loader, the debug trail and the A/B ledgers
|
|
764
|
+
// already print, so they belong here with every other named child.
|
|
765
|
+
'worker:files': 'research:files',
|
|
766
|
+
'worker:apis': 'research:apis',
|
|
767
|
+
'worker:context': 'research:context',
|
|
768
|
+
'worker:tooling': 'research:tooling'
|
|
769
|
+
};
|
|
770
|
+
/**
|
|
771
|
+
* The group a named child belongs to.
|
|
772
|
+
*
|
|
773
|
+
* Returns `undefined` for a name the table does not know, and the CALLER decides
|
|
774
|
+
* what that means. `runPhaseChild` treats it as `inherit` — a child that reaches
|
|
775
|
+
* the model with today's argv is always safe — while the test treats it as a
|
|
776
|
+
* failure. That split is deliberate: the guard belongs at build time, where
|
|
777
|
+
* someone can fix it, not at run time, where it would abort a user's task over a
|
|
778
|
+
* missing table row.
|
|
779
|
+
*/
|
|
780
|
+
export function reasoningGroupForChild(name) {
|
|
781
|
+
return REASONING_GROUP_BY_CHILD[name];
|
|
782
|
+
}
|
|
783
|
+
/**
|
|
784
|
+
* For a `research:*` group, the group a stored config falls back to when its own
|
|
785
|
+
* key is missing. Every other group maps to `undefined`.
|
|
786
|
+
*/
|
|
787
|
+
const RESEARCH_SUBGROUP_PARENT = {
|
|
788
|
+
'research:files': 'research',
|
|
789
|
+
'research:apis': 'research',
|
|
790
|
+
'research:context': 'research',
|
|
791
|
+
'research:tooling': 'research'
|
|
792
|
+
};
|
|
532
793
|
/**
|
|
533
794
|
* Always returns a COMPLETE record, never a partial one.
|
|
534
795
|
*
|
|
@@ -541,13 +802,33 @@ export function sanitizeReasoningLevels(value) {
|
|
|
541
802
|
const stored = typeof value === 'object' && value !== null && !Array.isArray(value) ?
|
|
542
803
|
value
|
|
543
804
|
: {};
|
|
805
|
+
const valid = (v) => REASONING_SETTINGS.includes(v);
|
|
544
806
|
const out = {};
|
|
545
807
|
for (const group of REASONING_GROUPS) {
|
|
546
808
|
const stored_ = stored[group];
|
|
809
|
+
if (valid(stored_)) {
|
|
810
|
+
out[group] = stored_;
|
|
811
|
+
continue;
|
|
812
|
+
}
|
|
813
|
+
// A `research:*` key absent from the file falls back to the user's own
|
|
814
|
+
// `research`, NOT to the default table. This is the migration path: a
|
|
815
|
+
// config written before the split carries one `research` level and
|
|
816
|
+
// nothing else, and filling the four sub-cells from the table would
|
|
817
|
+
// silently overrule a choice the user had already made — the four
|
|
818
|
+
// workers are exactly the children that `research` used to set.
|
|
819
|
+
//
|
|
820
|
+
// IT CANNOT TELL A CHOICE FROM A SEED, and that is accepted. Leaving
|
|
821
|
+
// `default`/`on`/`off` for ANY reason makes `applyReasoningLevel` seed
|
|
822
|
+
// every group from `resolveReasoning`, so a user who only ever nudged
|
|
823
|
+
// `gate` still has a `research` on disk they never picked. Inheriting
|
|
824
|
+
// it pins the four workers to the old default and the measured
|
|
825
|
+
// `research:files: 'off'` never reaches them. That is what CUSTOM MODE
|
|
826
|
+
// MEANS: a frozen table, not a subscription to later measurements — the
|
|
827
|
+
// same is true of every other group in the file. `default` mode is
|
|
828
|
+
// where a new reading takes effect, and it takes effect there at once.
|
|
829
|
+
const parent = RESEARCH_SUBGROUP_PARENT[group];
|
|
547
830
|
out[group] =
|
|
548
|
-
|
|
549
|
-
stored_
|
|
550
|
-
: DEFAULT_REASONING_TABLE[group];
|
|
831
|
+
parent && valid(stored[parent]) ? stored[parent] : DEFAULT_REASONING_TABLE[group];
|
|
551
832
|
}
|
|
552
833
|
return out;
|
|
553
834
|
}
|
|
@@ -571,6 +852,24 @@ export function resolveReasoning(group, cfg) {
|
|
|
571
852
|
return DEFAULT_REASONING_TABLE[group];
|
|
572
853
|
}
|
|
573
854
|
}
|
|
855
|
+
/**
|
|
856
|
+
* The WHOLE table, as this config will actually run it.
|
|
857
|
+
*
|
|
858
|
+
* This is the question every caller has — the settings menu when it repaints,
|
|
859
|
+
* the mismatch warning when it scans, the custom-mode seeder when it freezes the
|
|
860
|
+
* table — and each of them used to write the same loop over `resolveReasoning`.
|
|
861
|
+
* A per-key accessor with no whole-table companion is also how a THIRD shape got
|
|
862
|
+
* invented (`Array<{group, setting}>`) and leaked into `reasoningMismatches`.
|
|
863
|
+
*
|
|
864
|
+
* `resolveReasoning` stays: one group is still a fair question, and it is the
|
|
865
|
+
* only place the four modes are interpreted.
|
|
866
|
+
*/
|
|
867
|
+
export function effectiveReasoning(cfg) {
|
|
868
|
+
const out = {};
|
|
869
|
+
for (const group of REASONING_GROUPS)
|
|
870
|
+
out[group] = resolveReasoning(group, cfg);
|
|
871
|
+
return out;
|
|
872
|
+
}
|
|
574
873
|
/**
|
|
575
874
|
* The argv fragment for a setting. `inherit` is the empty fragment — no flag at
|
|
576
875
|
* all — which is what makes an all-`inherit` config byte-identical to the
|
|
@@ -581,8 +880,16 @@ export function thinkingArgs(setting) {
|
|
|
581
880
|
}
|
|
582
881
|
/** One honest sentence per group, for the /task-config rows. */
|
|
583
882
|
export const REASONING_GROUP_HELP = {
|
|
584
|
-
research: 'The
|
|
585
|
-
+ '
|
|
883
|
+
research: 'The pi-worker subagent tool, and the fallback for any research worker below '
|
|
884
|
+
+ 'whose own level is unset. Read-only exploration loops.',
|
|
885
|
+
'research:files': 'Research worker 1 of 4: maps which files the task will touch. Read-heavy. '
|
|
886
|
+
+ 'Measured: the two arms tie, so it runs without thinking.',
|
|
887
|
+
'research:apis': 'Research worker 2 of 4: the symbols and signatures the task must call. '
|
|
888
|
+
+ 'Read-heavy, docs- and search-capable.',
|
|
889
|
+
'research:context': 'Research worker 3 of 4: how the project is put together. One of the two that '
|
|
890
|
+
+ 'burned wall-clock on restarts in the last full run.',
|
|
891
|
+
'research:tooling': 'Research worker 4 of 4: the commands that build, test and run the project. '
|
|
892
|
+
+ 'Measured: thinking wins on both quality and speed.',
|
|
586
893
|
phase: 'Refining your request, generating and answering the clarifying questions, '
|
|
587
894
|
+ 'writing the spec, and critiquing it.',
|
|
588
895
|
planning: "/task-auto's planners: splitting a design document into tasks and extracting "
|