@mjasnikovs/pi-task 0.38.29 → 0.38.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/config.d.ts +70 -70
- package/dist/config/config.js +26 -35
- package/dist/config/extension-list.d.ts +6 -5
- package/dist/config/extension-list.js +3 -2
- package/dist/config/reasoning-args.d.ts +9 -7
- package/dist/config/reasoning-args.js +12 -10
- package/dist/config/reasoning.d.ts +44 -105
- package/dist/config/reasoning.js +27 -704
- package/dist/config/register.d.ts +34 -48
- package/dist/config/register.js +41 -51
- package/dist/config/tool-list.d.ts +16 -16
- package/dist/config/tool-list.js +1 -1
- package/dist/index.js +2 -0
- package/dist/remote/bridge.d.ts +19 -10
- package/dist/remote/bridge.js +3 -2
- package/dist/remote/broadcast.js +3 -1
- package/dist/remote/events.js +12 -11
- package/dist/remote/history.d.ts +1 -1
- package/dist/remote/protocol.d.ts +6 -3
- package/dist/remote/protocol.js +2 -1
- package/dist/remote/push.d.ts +16 -16
- package/dist/remote/push.js +27 -27
- package/dist/remote/register.d.ts +3 -3
- package/dist/remote/register.js +17 -19
- package/dist/remote/server.d.ts +9 -8
- package/dist/remote/server.js +15 -14
- package/dist/remote/session-state.d.ts +5 -4
- package/dist/remote/session-state.js +8 -5
- package/dist/remote/sw.d.ts +7 -6
- package/dist/remote/sw.js +7 -6
- package/dist/remote/tailscale.d.ts +4 -2
- package/dist/remote/tailscale.js +4 -2
- package/dist/remote/ui-highlight.js +6 -5
- package/dist/remote/ui-render.js +4 -4
- package/dist/remote/ui-script.js +24 -24
- package/dist/remote/ui-styles.d.ts +1 -1
- package/dist/remote/ui-styles.js +10 -13
- package/dist/remote/ui-tools.js +9 -6
- package/dist/shared/child-extensions.d.ts +29 -17
- package/dist/shared/child-extensions.js +29 -17
- package/dist/shared/child-output.d.ts +30 -24
- package/dist/shared/child-output.js +25 -17
- package/dist/shared/child-process.d.ts +47 -40
- package/dist/shared/child-process.js +50 -59
- package/dist/shared/command-watchdog.d.ts +85 -16
- package/dist/shared/command-watchdog.js +115 -21
- package/dist/shared/fs-text.d.ts +16 -10
- package/dist/shared/fs-text.js +16 -10
- package/dist/shared/git-runner.d.ts +25 -25
- package/dist/shared/git-runner.js +25 -25
- package/dist/shared/leaked-tool-call.d.ts +17 -11
- package/dist/shared/leaked-tool-call.js +23 -15
- package/dist/shared/model-endpoint.d.ts +29 -16
- package/dist/shared/model-endpoint.js +33 -21
- package/dist/shared/pi-invocation.d.ts +7 -4
- package/dist/shared/pi-invocation.js +12 -7
- package/dist/shared/pkg-version.d.ts +13 -5
- package/dist/shared/pkg-version.js +13 -5
- package/dist/shared/reasoning-capability.d.ts +35 -24
- package/dist/shared/reasoning-capability.js +35 -24
- package/dist/shared/stream-watchdog.d.ts +60 -44
- package/dist/shared/stream-watchdog.js +62 -45
- package/dist/task/accept-debt.d.ts +41 -43
- package/dist/task/accept-debt.js +73 -65
- package/dist/task/api-synthesis.d.ts +24 -21
- package/dist/task/api-synthesis.js +32 -26
- package/dist/task/apis-contract.d.ts +32 -64
- package/dist/task/apis-contract.js +32 -64
- package/dist/task/artifact-closure.d.ts +27 -13
- package/dist/task/artifact-closure.js +95 -67
- package/dist/task/auto-commit.d.ts +46 -35
- package/dist/task/auto-commit.js +51 -38
- package/dist/task/auto-io.d.ts +45 -25
- package/dist/task/auto-io.js +57 -29
- package/dist/task/auto-orchestrator.d.ts +26 -24
- package/dist/task/auto-orchestrator.js +192 -165
- package/dist/task/auto-prompts.d.ts +36 -24
- package/dist/task/auto-prompts.js +40 -26
- package/dist/task/autofix-ledger.d.ts +27 -25
- package/dist/task/autofix-ledger.js +29 -26
- package/dist/task/batch-test-task.d.ts +20 -12
- package/dist/task/batch-test-task.js +67 -60
- package/dist/task/boot-probe.d.ts +60 -44
- package/dist/task/boot-probe.js +91 -72
- package/dist/task/cancel-input.d.ts +30 -16
- package/dist/task/cancel-input.js +20 -11
- package/dist/task/cancel-points.d.ts +27 -20
- package/dist/task/cancel-points.js +30 -22
- package/dist/task/child-runner.d.ts +124 -55
- package/dist/task/child-runner.js +298 -90
- package/dist/task/child-status.d.ts +23 -16
- package/dist/task/child-status.js +23 -16
- package/dist/task/clamp-output.js +12 -5
- package/dist/task/command-run.d.ts +31 -28
- package/dist/task/command-run.js +44 -35
- package/dist/task/command-shrink.d.ts +25 -18
- package/dist/task/command-shrink.js +37 -31
- package/dist/task/command-watchdog.d.ts +9 -6
- package/dist/task/command-watchdog.js +21 -15
- package/dist/task/context-attribution.d.ts +34 -26
- package/dist/task/context-attribution.js +34 -26
- package/dist/task/context-silence.d.ts +39 -29
- package/dist/task/context-silence.js +35 -25
- package/dist/task/context-usage.d.ts +16 -9
- package/dist/task/context-usage.js +16 -9
- package/dist/task/contracts.d.ts +8 -4
- package/dist/task/contracts.js +25 -17
- package/dist/task/coverage-loop.d.ts +22 -18
- package/dist/task/coverage-loop.js +35 -30
- package/dist/task/critique-probes.d.ts +13 -14
- package/dist/task/critique-probes.js +50 -39
- package/dist/task/debug-log.d.ts +13 -5
- package/dist/task/debug-log.js +32 -20
- package/dist/task/decompose-fidelity.d.ts +11 -9
- package/dist/task/decompose-fidelity.js +38 -33
- package/dist/task/decompose-granularity.d.ts +41 -38
- package/dist/task/decompose-granularity.js +41 -38
- package/dist/task/deep-render-check.d.ts +22 -14
- package/dist/task/deep-render-check.js +40 -31
- package/dist/task/dropped-input.d.ts +12 -7
- package/dist/task/dropped-input.js +5 -2
- package/dist/task/enforce-attribution.d.ts +38 -47
- package/dist/task/enforce-attribution.js +46 -52
- package/dist/task/enforce-guidelines.d.ts +31 -20
- package/dist/task/enforce-guidelines.js +32 -21
- package/dist/task/enrichment.d.ts +7 -2
- package/dist/task/enrichment.js +26 -14
- package/dist/task/env-notes.d.ts +16 -7
- package/dist/task/env-notes.js +48 -31
- package/dist/task/env-template-closure.d.ts +4 -4
- package/dist/task/env-template-closure.js +42 -34
- package/dist/task/external-context.d.ts +28 -21
- package/dist/task/external-context.js +17 -12
- package/dist/task/failure-classifier.d.ts +4 -5
- package/dist/task/failure-classifier.js +30 -8
- package/dist/task/file-inventory.d.ts +15 -11
- package/dist/task/file-inventory.js +25 -22
- package/dist/task/final-gate-fix.d.ts +74 -86
- package/dist/task/final-gate-fix.js +97 -116
- package/dist/task/final-gate-progress.d.ts +29 -46
- package/dist/task/final-gate-progress.js +40 -51
- package/dist/task/final-gate.d.ts +64 -97
- package/dist/task/final-gate.js +192 -199
- package/dist/task/fix-child.d.ts +21 -27
- package/dist/task/fix-child.js +21 -27
- package/dist/task/foreign-path.d.ts +6 -5
- package/dist/task/foreign-path.js +0 -0
- package/dist/task/frozen-conflict.d.ts +9 -10
- package/dist/task/frozen-conflict.js +61 -64
- package/dist/task/frozen-path-guard.d.ts +35 -14
- package/dist/task/frozen-path-guard.js +56 -39
- package/dist/task/gate-child.d.ts +27 -28
- package/dist/task/gate-child.js +36 -35
- package/dist/task/gate-deps.d.ts +34 -27
- package/dist/task/gate-deps.js +169 -159
- package/dist/task/gate-tally.d.ts +77 -80
- package/dist/task/gate-tally.js +65 -68
- package/dist/task/git-state-guard.d.ts +15 -11
- package/dist/task/git-state-guard.js +76 -66
- package/dist/task/impl-widget.d.ts +25 -16
- package/dist/task/impl-widget.js +27 -17
- package/dist/task/implementation-guards.d.ts +26 -0
- package/dist/task/implementation-guards.js +177 -0
- package/dist/task/implementation-thinking.d.ts +33 -31
- package/dist/task/implementation-thinking.js +5 -6
- package/dist/task/implementation-turn.d.ts +39 -31
- package/dist/task/implementation-turn.js +41 -28
- package/dist/task/inline-markdown.d.ts +20 -7
- package/dist/task/inline-markdown.js +15 -6
- package/dist/task/launch-config-gap.js +25 -39
- package/dist/task/launch-contract.d.ts +18 -21
- package/dist/task/launch-contract.js +28 -30
- package/dist/task/launch-manifest.d.ts +6 -2
- package/dist/task/launch-manifest.js +35 -34
- package/dist/task/ledger.js +16 -14
- package/dist/task/lint-fix.d.ts +6 -8
- package/dist/task/lint-fix.js +67 -69
- package/dist/task/loop-detector.d.ts +27 -8
- package/dist/task/loop-detector.js +38 -14
- package/dist/task/mid-run-input.d.ts +17 -15
- package/dist/task/mid-run-input.js +17 -15
- package/dist/task/orchestrator.d.ts +24 -28
- package/dist/task/orchestrator.js +89 -66
- package/dist/task/orientation.d.ts +18 -23
- package/dist/task/orientation.js +24 -31
- package/dist/task/owned-freeze-conflict.d.ts +21 -20
- package/dist/task/owned-freeze-conflict.js +52 -85
- package/dist/task/owned-freeze-reassign.d.ts +40 -60
- package/dist/task/owned-freeze-reassign.js +41 -61
- package/dist/task/parsers.d.ts +4 -2
- package/dist/task/parsers.js +4 -4
- package/dist/task/phases.d.ts +41 -48
- package/dist/task/phases.js +196 -252
- package/dist/task/plan-io.d.ts +6 -7
- package/dist/task/plan-io.js +6 -7
- package/dist/task/plan-orchestrator.d.ts +10 -8
- package/dist/task/plan-orchestrator.js +14 -10
- package/dist/task/plan-prompts.d.ts +6 -5
- package/dist/task/plan-prompts.js +6 -5
- package/dist/task/plan-readonly.d.ts +4 -5
- package/dist/task/plan-readonly.js +4 -5
- package/dist/task/plan-rounds.d.ts +17 -29
- package/dist/task/plan-rounds.js +21 -34
- package/dist/task/plan-session.d.ts +58 -72
- package/dist/task/plan-session.js +61 -83
- package/dist/task/probe-gaming.d.ts +28 -27
- package/dist/task/probe-gaming.js +0 -0
- package/dist/task/prohibition-probe.d.ts +14 -16
- package/dist/task/prompts.d.ts +3 -4
- package/dist/task/prompts.js +17 -26
- package/dist/task/qa-transcript.d.ts +15 -22
- package/dist/task/qa-transcript.js +15 -21
- package/dist/task/question-box.d.ts +17 -13
- package/dist/task/question-box.js +19 -15
- package/dist/task/question-dedup.d.ts +6 -7
- package/dist/task/question-dedup.js +13 -14
- package/dist/task/question-dialog.d.ts +22 -32
- package/dist/task/question-dialog.js +22 -32
- package/dist/task/question-source.d.ts +18 -44
- package/dist/task/question-source.js +22 -51
- package/dist/task/refuted-constraint.d.ts +11 -31
- package/dist/task/refuted-constraint.js +27 -51
- package/dist/task/regenerable-artifacts.d.ts +12 -31
- package/dist/task/regenerable-artifacts.js +12 -31
- package/dist/task/render-check.d.ts +11 -22
- package/dist/task/render-check.js +33 -46
- package/dist/task/repo-health-check.d.ts +10 -14
- package/dist/task/repo-health-check.js +17 -23
- package/dist/task/requirements.d.ts +38 -71
- package/dist/task/requirements.js +78 -126
- package/dist/task/research-fanout-budget.d.ts +51 -88
- package/dist/task/research-fanout-budget.js +51 -88
- package/dist/task/research-worker.d.ts +29 -39
- package/dist/task/research-worker.js +37 -61
- package/dist/task/resume-gap.d.ts +14 -15
- package/dist/task/root-cause-repair.d.ts +9 -9
- package/dist/task/root-cause-repair.js +28 -40
- package/dist/task/run-bracket.d.ts +10 -13
- package/dist/task/run-end.d.ts +12 -22
- package/dist/task/run-end.js +8 -16
- package/dist/task/run-final-gate.d.ts +19 -21
- package/dist/task/run-final-gate.js +62 -80
- package/dist/task/runner-globs.d.ts +12 -13
- package/dist/task/runner-globs.js +12 -13
- package/dist/task/runner-resolve.d.ts +9 -9
- package/dist/task/runner-resolve.js +22 -23
- package/dist/task/script-escape.d.ts +10 -12
- package/dist/task/script-escape.js +13 -14
- package/dist/task/serve-entry.d.ts +1 -1
- package/dist/task/serve-entry.js +22 -25
- package/dist/task/service-blocks.js +4 -2
- package/dist/task/shipped-source.d.ts +11 -29
- package/dist/task/shipped-source.js +11 -29
- package/dist/task/skip-escape.js +10 -14
- package/dist/task/spec-urls.d.ts +26 -65
- package/dist/task/spec-urls.js +26 -65
- package/dist/task/spec-validation.d.ts +17 -20
- package/dist/task/spec-validation.js +17 -20
- package/dist/task/stall-detector.d.ts +23 -30
- package/dist/task/stall-detector.js +23 -30
- package/dist/task/stream-watchdog.d.ts +14 -12
- package/dist/task/stream-watchdog.js +14 -12
- package/dist/task/substitution-probe.d.ts +17 -20
- package/dist/task/substitution-probe.js +17 -20
- package/dist/task/task-gates.d.ts +36 -41
- package/dist/task/task-gates.js +95 -106
- package/dist/task/task-io.d.ts +4 -4
- package/dist/task/task-io.js +4 -4
- package/dist/task/task-parsers.js +4 -3
- package/dist/task/task-provenance.d.ts +2 -2
- package/dist/task/task-provenance.js +11 -13
- package/dist/task/task-types.d.ts +4 -3
- package/dist/task/terminal-outcome.d.ts +14 -16
- package/dist/task/terminal-outcome.js +12 -14
- package/dist/task/test-assembly.d.ts +13 -20
- package/dist/task/test-assembly.js +13 -20
- package/dist/task/timings.d.ts +5 -3
- package/dist/task/timings.js +5 -3
- package/dist/task/title-label.d.ts +9 -4
- package/dist/task/title-label.js +9 -4
- package/dist/task/type-only-answer.d.ts +44 -52
- package/dist/task/type-only-answer.js +44 -52
- package/dist/task/unfailable-command.d.ts +18 -24
- package/dist/task/unfailable-command.js +21 -27
- package/dist/task/unknown-routing.d.ts +10 -4
- package/dist/task/unknown-routing.js +10 -4
- package/dist/task/user-directives.d.ts +5 -8
- package/dist/task/user-directives.js +5 -8
- package/dist/task/verify-quality.d.ts +18 -22
- package/dist/task/verify-quality.js +45 -46
- package/dist/task/verify-reconcile.d.ts +15 -10
- package/dist/task/verify-reconcile.js +45 -43
- package/dist/task/verify-resolution.d.ts +24 -20
- package/dist/task/verify-resolution.js +51 -50
- package/dist/task/verify-work.d.ts +59 -66
- package/dist/task/verify-work.js +101 -138
- package/dist/task/widget.d.ts +15 -14
- package/dist/task/widget.js +22 -17
- package/dist/task/wiring-claims.d.ts +25 -32
- package/dist/task/wiring-claims.js +30 -35
- package/dist/task/write-guard.d.ts +39 -39
- package/dist/task/write-guard.js +48 -51
- package/dist/task/yolo.d.ts +34 -30
- package/dist/task/yolo.js +42 -37
- package/dist/workers/abstention.d.ts +21 -41
- package/dist/workers/abstention.js +27 -48
- package/dist/workers/brave-search.d.ts +4 -3
- package/dist/workers/brave-search.js +5 -2
- package/dist/workers/brave-warning.d.ts +7 -4
- package/dist/workers/brave-warning.js +19 -7
- package/dist/workers/ddg-search.d.ts +6 -6
- package/dist/workers/ddg-search.js +18 -12
- package/dist/workers/docs-cache.js +5 -2
- package/dist/workers/docs-chunk.d.ts +30 -37
- package/dist/workers/docs-chunk.js +37 -41
- package/dist/workers/docs-core.d.ts +28 -44
- package/dist/workers/docs-core.js +25 -44
- package/dist/workers/docs-index.js +4 -3
- package/dist/workers/docs-lookup.d.ts +15 -22
- package/dist/workers/docs-lookup.js +12 -21
- package/dist/workers/docs-project.d.ts +15 -9
- package/dist/workers/docs-project.js +17 -10
- package/dist/workers/docs-resolve.d.ts +19 -20
- package/dist/workers/docs-resolve.js +35 -32
- package/dist/workers/docs-retrieve.d.ts +5 -6
- package/dist/workers/docs-retrieve.js +18 -15
- package/dist/workers/exa-search.d.ts +9 -6
- package/dist/workers/exa-search.js +23 -12
- package/dist/workers/fetch-core.d.ts +13 -16
- package/dist/workers/fetch-core.js +23 -23
- package/dist/workers/focused-extractor.d.ts +13 -12
- package/dist/workers/focused-extractor.js +27 -19
- package/dist/workers/html-clean.js +24 -14
- package/dist/workers/http-request.d.ts +28 -20
- package/dist/workers/http-request.js +22 -17
- package/dist/workers/npm-version.d.ts +28 -11
- package/dist/workers/npm-version.js +24 -15
- package/dist/workers/phantom-imports.d.ts +15 -12
- package/dist/workers/phantom-imports.js +30 -24
- package/dist/workers/pi-worker-core.d.ts +65 -96
- package/dist/workers/pi-worker-core.js +93 -181
- package/dist/workers/pi-worker-docs.d.ts +24 -19
- package/dist/workers/pi-worker-docs.js +67 -76
- package/dist/workers/pi-worker-fetch.d.ts +7 -3
- package/dist/workers/pi-worker-fetch.js +27 -19
- package/dist/workers/pi-worker-search.js +12 -8
- package/dist/workers/pi-worker.d.ts +9 -4
- package/dist/workers/pi-worker.js +21 -14
- package/dist/workers/reasoning-warning.d.ts +18 -17
- package/dist/workers/reasoning-warning.js +22 -20
- package/dist/workers/research-cache.js +50 -78
- package/dist/workers/search-core.js +7 -5
- package/dist/workers/search-types.d.ts +10 -9
- package/dist/workers/search-types.js +9 -8
- package/dist/workers/session-hint.d.ts +13 -14
- package/dist/workers/session-hint.js +8 -9
- package/dist/workers/shared.d.ts +21 -25
- package/dist/workers/shared.js +0 -0
- package/dist/workers/single-read-extension.d.ts +14 -7
- package/dist/workers/single-read-extension.js +14 -7
- package/dist/workers/single-read-guard.d.ts +27 -30
- package/dist/workers/single-read-guard.js +36 -36
- package/dist/workers/typeonly-log.d.ts +12 -9
- package/dist/workers/typeonly-log.js +29 -33
- package/dist/workers/worker-channels.d.ts +15 -23
- package/dist/workers/worker-channels.js +15 -23
- package/dist/workers/worker-failure.d.ts +38 -46
- package/dist/workers/worker-failure.js +31 -39
- package/dist/workers/worker-kill.d.ts +25 -26
- package/dist/workers/worker-kill.js +16 -19
- package/dist/workers/worker-profiles.d.ts +54 -56
- package/dist/workers/worker-profiles.js +63 -39
- package/package.json +10 -8
package/dist/config/reasoning.js
CHANGED
|
@@ -13,16 +13,13 @@ export const REASONING_GROUPS = [
|
|
|
13
13
|
];
|
|
14
14
|
export const REASONING_MODES = ['default', 'on', 'off', 'custom'];
|
|
15
15
|
/**
|
|
16
|
-
* The settings offered in /task-config: `inherit` plus pi's
|
|
17
|
-
*
|
|
16
|
+
* The settings offered in /task-config: `inherit` plus the standard part of pi's
|
|
17
|
+
* own thinking cycle.
|
|
18
18
|
*
|
|
19
|
-
* `xhigh` and `max` are DELIBERATELY ABSENT. pi
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* answers an unknown effort with HTTP 500, not a clamp. Offering a level that pi's
|
|
24
|
-
* own UI does not is how you ship a `--thinking` that hard-fails on some models.
|
|
25
|
-
* This machine already reaches xhigh through `"high": "xhigh"` in its map.
|
|
19
|
+
* `xhigh` and `max` are DELIBERATELY ABSENT. pi-ai's `getSupportedThinkingLevels`
|
|
20
|
+
* accepts the standard levels for any reasoning model, but offers `xhigh` and
|
|
21
|
+
* `max` only when the model declares them in its `thinkingLevelMap`. Offering
|
|
22
|
+
* them here would put a level in this UI that pi's own UI may not have.
|
|
26
23
|
*/
|
|
27
24
|
export const REASONING_SETTINGS = [
|
|
28
25
|
'inherit',
|
|
@@ -32,687 +29,36 @@ export const REASONING_SETTINGS = [
|
|
|
32
29
|
'medium',
|
|
33
30
|
'high'
|
|
34
31
|
];
|
|
35
|
-
/**
|
|
36
|
-
* The level mode `on` uses, and the treatment arm of the A/B that fills in
|
|
37
|
-
* {@link DEFAULT_REASONING_TABLE}. ONE constant so the shipped config and the
|
|
38
|
-
* measurement behind it can never drift apart.
|
|
39
|
-
*/
|
|
32
|
+
/** The level mode `on` uses. */
|
|
40
33
|
export const REASONING_ON_LEVEL = 'medium';
|
|
41
34
|
/**
|
|
42
35
|
* The per-group table used by mode `default`.
|
|
43
36
|
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
* {@link REASONING_ON_LEVEL}) carrying its date, model, counts and RUNG in the
|
|
48
|
-
* comment beside it. A cell filled in from intuition is WORSE than `inherit`:
|
|
49
|
-
* `inherit` is honest about knowing nothing, while a wrong cell is a
|
|
50
|
-
* measurement nobody took, wearing the authority of a default.
|
|
51
|
-
*
|
|
52
|
-
* THE RUNG IS PART OF THE RESULT, so read it before trusting a cell. The
|
|
53
|
-
* harness returns a two-way verdict — `off` or {@link REASONING_ON_LEVEL},
|
|
54
|
-
* never a tie — down a three-rung ladder: rung 1 a significant quality
|
|
55
|
-
* difference, rung 2 quality level and a significant speed difference, rung 3
|
|
56
|
-
* nothing separated the arms and the cheaper level carries it by a stated
|
|
57
|
-
* prior. A rung-3 cell is a DECISION, not a finding: it says "no reason found
|
|
58
|
-
* to pay for thinking", not "thinking was shown not to help". At n=20/arm the
|
|
59
|
-
* run is only powered for large effects, so rung 3 is the common outcome and
|
|
60
|
-
* an absent effect and an undetected one look identical from here.
|
|
61
|
-
*
|
|
62
|
-
* NOTHING IS ALREADY ON THE RECORD, and the belief that something was is worth
|
|
63
|
-
* writing down. This docstring used to say magicknumbers.md had measured the
|
|
64
|
-
* decompose child "with one knob, `enable_thinking` — off answered 1/10, on
|
|
65
|
-
* answered 8/8", and treated that as the split the A/B had to reproduce.
|
|
66
|
-
*
|
|
67
|
-
* VERIFIED 2026-08-27: the string "8/8" does not appear in magicknumbers.md, and
|
|
68
|
-
* never has (`git log -S`; the phrase was introduced by THIS file's own commit,
|
|
69
|
-
* `0b91f71`). What that page records is
|
|
70
|
-
*
|
|
71
|
-
* "Measured, captured decompose request, REASONING OFF, n=10 per cell:
|
|
72
|
-
* 1/10 → 7/10 (stall detector) → 8/9 (all three)."
|
|
73
|
-
*
|
|
74
|
-
* — a ladder over THREE GUARDS at CONSTANT reasoning off, from `fea7bbb`. Both
|
|
75
|
-
* endpoints are the same arm. `enable_thinking` is not its knob and never was,
|
|
76
|
-
* and the "8/8" is a misread of the ladder's last cell, `8/9`.
|
|
77
|
-
*
|
|
78
|
-
* So no reasoning effect on `planning` had ever been measured, and the 10/10 vs
|
|
79
|
-
* 10/10 read on the current tree was not a contradiction of a prior result — it
|
|
80
|
-
* was the only reading there had ever been. The controls that page DOES record
|
|
81
|
-
* (Qwen3.6-27B 10/10, Gemma4-12B 10/10) are likewise reasoning off, so they say
|
|
82
|
-
* nothing about a cell flipping by model either.
|
|
83
|
-
*
|
|
84
|
-
* The `planning` cell below is the FIRST measured reading of that knob, taken
|
|
85
|
-
* 2026-08-27 on a citation-fidelity axis built for it. It is also the first cell
|
|
86
|
-
* that is not `off`.
|
|
37
|
+
* `inherit` says nothing was decided for that group and the child keeps the
|
|
38
|
+
* host's level. Any other value is a decision this project made for that
|
|
39
|
+
* group's job, and `/task-config` mode `custom` overrides all of it.
|
|
87
40
|
*/
|
|
88
41
|
export const DEFAULT_REASONING_TABLE = {
|
|
89
|
-
// A/B 2026-08-26, Qwen3.8-27B-NVFP4-MTP-VERY-HIGH.gguf (llama.cpp
|
|
90
|
-
// b10620-0f3b51e03), scripts/live-reasoning-group-ab.ts, n=12/arm over 12
|
|
91
|
-
// distinct mx5 tasks. off 10/12 vs medium 11/12 (p=1.0000); neither arm
|
|
92
|
-
// failed to answer. Wall clock, PAIRED by stimulus over the 10 pairs usable
|
|
93
|
-
// in both arms: p=0.7090, mean off 79.4s vs medium 59.3s.
|
|
94
|
-
// off real+edited-named 95% CI [0.55, 0.95].
|
|
95
|
-
// RUNG 3 — a DECISION, not a finding. Nothing separated the arms on either
|
|
96
|
-
// axis, and off carries it by the stated prior: thinking that buys nothing
|
|
97
|
-
// measurable is not worth its tokens. At 12 pairs an absent effect and an
|
|
98
|
-
// undetected one look the same from here. Ledger: ledger-research.jsonl.
|
|
99
|
-
//
|
|
100
|
-
// THE AXIS IS THE CONJUNCTION, built for this run: every path the answer
|
|
101
|
-
// names is real AND every pre-existing file the task edited is named
|
|
102
|
-
// (`filesAnswered`, scripts/reasoning-ab-files-truth.ts). Recall truth is
|
|
103
|
-
// the shipped tree, not the recorded answer. Screened offline: restricted
|
|
104
|
-
// to pre-existing files the recorded answers score 86.6% (71/82 paths, 29
|
|
105
|
-
// of 38 tasks perfect); counting CREATED files instead drops them to 49.0%
|
|
106
|
-
// and the CHECK loses, so a file that does not exist yet is not truth.
|
|
107
|
-
// It is NOT saturated here — 10/12 and 11/12, with genuine judgement errors
|
|
108
|
-
// in both arms — so the clock was allowed to decide and declined to.
|
|
109
|
-
//
|
|
110
|
-
// THE STIMULI ARE HALF THE AXIS. The superseded precision-only run
|
|
111
|
-
// (ledger-research.PRECISION-ONLY-10rep.jsonl, `--axis precision` to
|
|
112
|
-
// reproduce) tied 10/10 on TASK_0002..0012, the greenfield head of the mx5
|
|
113
|
-
// run — SEVEN OF THOSE TEN EDIT NO PRE-EXISTING FILE AT ALL, so it measured
|
|
114
|
-
// recall where recall does not exist. `filesRecallStimuli` screens for it;
|
|
115
|
-
// these 12 tasks each edit >=2 pre-existing files. That ledger CANNOT be
|
|
116
|
-
// rescored onto this axis and the rescorer abstains rather than return a
|
|
117
|
-
// number.
|
|
118
|
-
//
|
|
119
|
-
// INSTRUMENT NOTE, because this cell read `off 8/12` before it. `filesPaths`
|
|
120
|
-
// split an entry on TWO spaces, which is what the prompt specifies and what
|
|
121
|
-
// all 470 recorded paths use. Three live `off` trials wrote
|
|
122
|
-
// `src/client/main.tsx: App root` — one space after a colon — so the
|
|
123
|
-
// description was taken as part of the path and TASK_0053 scored 0/19 with
|
|
124
|
-
// all 19 paths real. Fixed by `COLON_ENTRY`; rescored from stored text with
|
|
125
|
-
// no GPU, 2 of 24 trials changed side, and the known-good answer is
|
|
126
|
-
// unmoved (53/56 whole-corpus precision, identical path counts). This is
|
|
127
|
-
// [[ab-scorer-must-see-the-same-input]] a second time in one function.
|
|
128
|
-
//
|
|
129
|
-
// THE CONFOUND THE PRECISION RUN EXPOSED IS STILL VISIBLE and still points
|
|
130
|
-
// the other way: off named 212 real paths to medium's 153, while medium
|
|
131
|
-
// named 27/27 edited files to off's 26/27. Off says more and is slower for
|
|
132
|
-
// it. The conjunction was built so that neither half can be gamed alone,
|
|
133
|
-
// and on these numbers it separates neither arm.
|
|
134
|
-
//
|
|
135
|
-
// AXIS RE-AUDITED 2026-08-27, after planning's adjudicator turned out to
|
|
136
|
-
// have four bugs. One more was found here and it is the SAME-INPUT rule
|
|
137
|
-
// broken a third time: `filesPaths` read the child's whole raw turn, while
|
|
138
|
-
// the SCREEN sliced the recorded `## research` down to its FILES block
|
|
139
|
-
// precisely because APIS is symbols and would score as 100% invented. One
|
|
140
|
-
// `off` trial emitted its own `APIS` heading despite the prompt's "No other
|
|
141
|
-
// sections", and three symbols under it — `cn` and two `--*` token lists —
|
|
142
|
-
// were counted as invented paths, failing a trial whose FILES block was
|
|
143
|
-
// 20/20 real. The slice now lives in `filesPaths`, where both callers reach
|
|
144
|
-
// it. The cell is UNCHANGED: that trial still fails, on a genuine recall
|
|
145
|
-
// miss (`src/client/types.d.ts`), and the counts are identical.
|
|
146
|
-
// The other two failures were checked by hand and are genuine — both arms
|
|
147
|
-
// put `AdminPage.spec.tsx` / `.story.tsx` under `src/client/routes/` when
|
|
148
|
-
// they ship under `src/client/pages/`, a wrong-directory prediction made
|
|
149
|
-
// from a real sibling.
|
|
150
|
-
//
|
|
151
|
-
// MEASURED UNDER THE SERVER'S GLOBAL SAMPLER, which is the THINKING preset,
|
|
152
|
-
// so the `off` arm decodes on sampling tuned for the `on` arm. That is the
|
|
153
|
-
// regime this machine really runs pi-task in, so the result is
|
|
154
|
-
// ecologically valid — it is NOT a clean comparison.
|
|
155
|
-
//
|
|
156
|
-
// OVERRIDDEN TO `medium` BY USER DECISION, 2026-08-27. THIS IS NOT A NEW
|
|
157
|
-
// MEASUREMENT — the trials above are unchanged and still say the arms tied.
|
|
158
|
-
// What changed is the prior. Rung 3 says "no reason found to pay for
|
|
159
|
-
// thinking"; it does not say thinking is worthless, and at n=12 an absent
|
|
160
|
-
// effect and an undetected one look identical.
|
|
161
|
-
//
|
|
162
|
-
// WHAT THE SAME LEDGER SAYS ABOUT THE CLOCK, re-read for this decision:
|
|
163
|
-
// medium is the FASTER arm on the mean, 62.9s vs 105.6s over the 12 paired
|
|
164
|
-
// stimuli (median 51.7s vs 87.7s), and it is faster on 6 of 12 by 1-265s
|
|
165
|
-
// while slower on 6 by 0-43s. The mean gap is entirely `off`'s TAIL: off's
|
|
166
|
-
// two slowest trials are 289s and 367s, medium's slowest is 131s. So the
|
|
167
|
-
// typical trial is a coin flip worth under a minute and the difference the
|
|
168
|
-
// arms actually have is in the tail.
|
|
169
|
-
//
|
|
170
|
-
// AND THE TAIL IS THE PATHOLOGY THIS CHANGE IS AIMED AT. mx5-n 2026-08-27:
|
|
171
|
-
// 40.7 of the research phase's 81.4 wall-clock minutes were thrown away on
|
|
172
|
-
// restarts, all of them in `worker:tooling` and `worker:context` — `files`
|
|
173
|
-
// and `apis` never restarted once. The worst was 550 tool calls over 20
|
|
174
|
-
// distinct files in 20 minutes for 25s of useful work.
|
|
175
|
-
//
|
|
176
|
-
// THE HONEST LIMIT OF THIS DECISION: no axis in
|
|
177
|
-
// scripts/live-reasoning-group-ab.ts scores tool-call count or repetition —
|
|
178
|
-
// the Trial record has no such field and RunWorkerResult exposes no count —
|
|
179
|
-
// so the ledger CANNOT say whether thinking prevents the wander. It was
|
|
180
|
-
// never asked. And the ledger's stimuli are the FILES worker, not the two
|
|
181
|
-
// that looped. Anyone flipping this back should note that the evidence for
|
|
182
|
-
// `off` is equally silent on the question.
|
|
183
|
-
//
|
|
184
|
-
// The structural fix for the wander shipped separately and does not depend
|
|
185
|
-
// on this cell: StallDetector is now wired into runWorker (it was in phase
|
|
186
|
-
// children only), so a rotation is killed ~8 calls into its second lap
|
|
187
|
-
// rather than at the 20-minute ceiling.
|
|
188
42
|
research: 'medium',
|
|
189
|
-
//
|
|
190
|
-
//
|
|
191
|
-
// WHY THE SPLIT EXISTS, from `research`'s own evidence. The override to
|
|
192
|
-
// `medium` above was aimed at ONE pathology — the restart tail — and that
|
|
193
|
-
// tail is not spread evenly. mx5-n 2026-08-27: 40.7 of the research phase's
|
|
194
|
-
// 81.4 wall-clock minutes were lost to restarts, ALL of them in
|
|
195
|
-
// `worker:tooling` and `worker:context`; `worker:files` and `worker:apis`
|
|
196
|
-
// never restarted once. With one cell, paying for thinking in the two that
|
|
197
|
-
// wander means paying for it in the two that do not.
|
|
198
|
-
//
|
|
199
|
-
// ── research:files — MEASURED, `off`. WRITTEN 2026-08-28 ──
|
|
200
|
-
//
|
|
201
|
-
// A/B 2026-08-26, Qwen3.8-27B-NVFP4-MTP-VERY-HIGH.gguf (llama.cpp
|
|
202
|
-
// b10620-0f3b51e03), scripts/live-reasoning-group-ab.ts, n=12/arm over 12
|
|
203
|
-
// distinct mx5 tasks. off 10/12 vs medium 11/12 (p=1.0000); clock PAIRED
|
|
204
|
-
// over the 10 usable pairs p=0.7090, mean off 79.4s vs medium 59.3s.
|
|
205
|
-
// off 95% CI [0.55, 0.95]. RUNG 3 — a DECISION, not a finding.
|
|
206
|
-
//
|
|
207
|
-
// THIS IS NOT A NEW RUN. It is ledger-research.jsonl, rescored under the
|
|
208
|
-
// group's new name:
|
|
209
|
-
//
|
|
210
|
-
// AB_CORPUS=/home/edgars/hub/ab-grouplab/mx5-copy bun run \
|
|
211
|
-
// scripts/rescore-reasoning-ledger.ts \
|
|
212
|
-
// /home/edgars/hub/ab-grouplab/ledger-research.jsonl research:files --from-text
|
|
213
|
-
//
|
|
214
|
-
// That ledger was ALWAYS a FILES worker — `child: 'worker:files'`, prompt
|
|
215
|
-
// `RESEARCH_FILES_PROMPT`, run in each task's own before-tree — so the split
|
|
216
|
-
// did not strand it, it named it. The axis is the CONJUNCTION: every path
|
|
217
|
-
// named is real AND every pre-existing file the task edited is named. The
|
|
218
|
-
// full derivation, the stimulus screen, the parser fix and the confound sit
|
|
219
|
-
// on the `research` cell above; they are that ledger's, and this is that
|
|
220
|
-
// ledger.
|
|
221
|
-
//
|
|
222
|
-
// WHY THIS CELL DOES NOT FOLLOW `research`'s OVERRIDE. That override was
|
|
223
|
-
// aimed at the restart tail, and the tail is not here: `worker:files` never
|
|
224
|
-
// restarted once. The reason for paying does not apply to this worker, and
|
|
225
|
-
// its own ledger says the arms tied.
|
|
43
|
+
// The four research workers, one cell each.
|
|
226
44
|
'research:files': 'off',
|
|
227
|
-
// NOT MEASURED. DECIDED BY PRIOR: a worker with no trial of its own keeps
|
|
228
|
-
// whatever `research` was decided to run at. No trial in this repo has ever
|
|
229
|
-
// run apis, context or tooling as its own arm, so a cell here that differed
|
|
230
|
-
// from `research` would be intuition wearing the authority of a default.
|
|
231
|
-
//
|
|
232
|
-
// AND AN AXIS IS THE HARD PART, not the GPU. Two candidates were screened
|
|
233
|
-
// offline and DIED there — scripts/research-worker-axes-step0.ts, no model,
|
|
234
|
-
// no GPU. For APIS: "every dotted symbol named is present in the tree"
|
|
235
|
-
// scores the recorded answers 28/28 tasks and 81/81 items. SATURATED, and
|
|
236
|
-
// loose with it — the check greps the symbol's LAST SEGMENT, so
|
|
237
|
-
// `Hono.c.json` passes on the word `json`. Tightening it has nothing to
|
|
238
|
-
// bite on, because APIS names are model-composed pseudo-symbols
|
|
239
|
-
// (`Hono.c.var`, `UUID regex`); that is the same wall
|
|
240
|
-
// [[apis-contract-stage3-refuted]] hit from the other side.
|
|
241
|
-
// TO REPLACE THIS WITH A MEASUREMENT, the axis must be an EXECUTION or a
|
|
242
|
-
// production adjudication, the way gate's and planning's are — not a
|
|
243
|
-
// property of the text.
|
|
244
|
-
//
|
|
245
|
-
// A COST RUN WAS DONE ANYWAY, 2026-08-28, and it found nothing to act on.
|
|
246
|
-
// n=20/arm on the same corpus and trees, production's tools, extensions,
|
|
247
|
-
// search hint and the task's own recorded FILES map: off 19/20 answered vs
|
|
248
|
-
// medium 20/20, and the PAIRED clock over 19 stimuli is off 127.2s vs
|
|
249
|
-
// medium 144.4s, p=0.5262. So thinking is neither better nor cheaper here
|
|
250
|
-
// by anything this run can see, and there is no cost argument for moving
|
|
251
|
-
// the cell either way. Ledger: ledger-research-apis.jsonl.
|
|
252
|
-
// THAT RUN CANNOT WRITE THIS CELL. Its axis is TERMINATION — "did it answer
|
|
253
|
-
// at all" — so its `off [rung 3]` line is the standing prior speaking with
|
|
254
|
-
// no quality reading behind it. See README-research-cost-runs.txt.
|
|
255
|
-
// ITS BUILD IS b10665-ca3d5a3e1, not the b10620-0f3b51e03 every other cell
|
|
256
|
-
// was measured on: llama.cpp was rebuilt mid-session. Internally paired, so
|
|
257
|
-
// off-vs-medium is fair within it; its SECONDS are not comparable to the
|
|
258
|
-
// files or tooling ledgers.
|
|
259
45
|
'research:apis': 'medium',
|
|
260
|
-
// NOT MEASURED. DECIDED BY PRIOR — see the block above `research:apis`.
|
|
261
|
-
// Its candidate axis died offline too: "every backticked project path in a
|
|
262
|
-
// CONTEXT bullet exists in the tree" scores the recorded answers 6/53 tasks
|
|
263
|
-
// and 213/391 items. THE CHECK LOSES, on exactly the residue the phase
|
|
264
|
-
// path-axis audit catalogued — npm specifiers (`@hono/zod-validator`),
|
|
265
|
-
// import paths (`hono/cookie`, `../server/auth`), bare filenames
|
|
266
|
-
// (`schema.ts`) and `src/` prefix elision.
|
|
267
|
-
// TO REPLACE THIS WITH A MEASUREMENT, note that this worker is one of the
|
|
268
|
-
// two the restart tail lives in, so the honest axis is the wander itself —
|
|
269
|
-
// and no Trial field records a tool-call count today.
|
|
270
|
-
//
|
|
271
|
-
// A COST RUN WAS DONE ANYWAY, 2026-08-28, n=20/arm, production's `read,grep`
|
|
272
|
-
// and no extensions. It found no clock difference — medians 81.2s vs 80.8s,
|
|
273
|
-
// paired p=0.8594 over 16 stimuli — but it did see the wander directly:
|
|
274
|
-
//
|
|
275
|
-
// died in a loop off 3/20 vs medium 0/20 p=0.2308
|
|
276
|
-
// emitted bullets off 17/20 vs medium 20/20 p=0.2308
|
|
277
|
-
//
|
|
278
|
-
// Neither reaches significance at n=20, and the second is TERMINATION, not
|
|
279
|
-
// quality. But this is the first time the restart tail has appeared INSIDE
|
|
280
|
-
// a measured run rather than in a production log, and it appeared only in
|
|
281
|
-
// the `off` arm. off's higher MEAN (97.8s vs 69.3s) is those three deaths,
|
|
282
|
-
// not slower work. THIS RUN CANNOT WRITE THIS CELL either: an underpowered
|
|
283
|
-
// termination signal is not a measurement, and its `off [rung 3]` verdict
|
|
284
|
-
// line is the prior, not a reading. THE CELL IS UNCHANGED. Ledger: ledger-research-context.jsonl,
|
|
285
|
-
// README-research-cost-runs.txt, build b10665-ca3d5a3e1.
|
|
286
|
-
//
|
|
287
|
-
// INSTRUMENT NOTE, because that run ABSTAINED first with 40 good answers
|
|
288
|
-
// already stored. It scored CONTEXT with `hasAnswerContent` — the FILES
|
|
289
|
-
// `name<gap>description` shape — and this worker emits a BULLET list of
|
|
290
|
-
// prose sentences, which `isEntryLine` rejects for ending in a full stop.
|
|
291
|
-
// Both arms read 0/20. Fixed by `contextEmittedBullets` and the stored
|
|
292
|
-
// trials rescored with no GPU. Fourth instance of
|
|
293
|
-
// [[ab-scorer-must-match-the-real-prompt]] in one harness.
|
|
294
|
-
// AND THE REPLACEMENT WAS WRONG BY ONE, found in review the same day: it
|
|
295
|
-
// asked for >=2 bullets where production's `classifyContextSilence` calls
|
|
296
|
-
// >=1 PRODUCTIVE, so a one-bullet answer production accepts read UNUSABLE
|
|
297
|
-
// in both arms. `countBullets` — production's own function, called not
|
|
298
|
-
// restated — moved 37/40 stored trials and the emitted-bullets line above
|
|
299
|
-
// from off 16/20 to off 17/20. The verdict, the cell and the clock are
|
|
300
|
-
// unchanged: still an underpowered termination signal, still a prior.
|
|
301
46
|
'research:context': 'medium',
|
|
302
|
-
// ── research:tooling — MEASURED, `medium`. RUNG 1. WRITTEN 2026-08-28 ──
|
|
303
|
-
//
|
|
304
|
-
// A/B 2026-08-28, Qwen3.8-27B-NVFP4-MTP-VERY-HIGH.gguf (llama.cpp
|
|
305
|
-
// b10620-0f3b51e03), scripts/live-reasoning-group-ab.ts, n=20/arm over 20
|
|
306
|
-
// distinct mx5 tasks, one rep each so the clock can pair.
|
|
307
|
-
//
|
|
308
|
-
// every command runs off 13/20 vs medium 20/20 p=0.0083
|
|
309
|
-
// wall clock (paired) off 49.5s vs medium 15.9s p=0.0400
|
|
310
|
-
// off 95% CI [0.43, 0.82]. Ledger: ledger-research-tooling.jsonl.
|
|
311
|
-
//
|
|
312
|
-
// RUNG 1 — quality decided it, and this is only the SECOND cell in the table
|
|
313
|
-
// a quality axis has ever decided. THE FIRST WAS extraction, which went the
|
|
314
|
-
// other way. Note there is no trade here: medium is both better AND 3.1x
|
|
315
|
-
// faster, so the sampler caveat below cannot be what produced it.
|
|
316
|
-
//
|
|
317
|
-
// THE AXIS: every command the checker can adjudicate must RESOLVE in the
|
|
318
|
-
// tree the worker inspected (scripts/reasoning-ab-tooling-truth.ts). Not a
|
|
319
|
-
// property of the text — the same kind of truth as gate's executed VERIFY,
|
|
320
|
-
// which is why it lives where four `phase` text axes died. `unknown` is a
|
|
321
|
-
// first-class answer and is scored neither way: three of the four command
|
|
322
|
-
// shapes can only return `real` or `unknown`, because an undeclared binary
|
|
323
|
-
// may still be on PATH and `bun test` resolves through the runtime.
|
|
324
|
-
//
|
|
325
|
-
// OFF'S FAILURE IS ONE CLASS, AUDITED ROW BY ROW. All seven name a
|
|
326
|
-
// dev-server command against `src/server/index.ts` — a file that exists in
|
|
327
|
-
// NEITHER the before-tree nor the after-tree of any of those seven tasks;
|
|
328
|
-
// it first appears around TASK_0057. The worker's own prompt says "If a
|
|
329
|
-
// tool isn't present in the repo, omit it — don't invent." Medium omits it.
|
|
330
|
-
//
|
|
331
|
-
// THE CHECKER WAS WRONG THREE TIMES FIRST, and all three are pinned in
|
|
332
|
-
// reasoning-ab-tooling-truth.test.ts:
|
|
333
|
-
// 1. `bun run <file.ts>` is a file invocation, not a script lookup.
|
|
334
|
-
// Caught offline; four of twelve reported failures were this bug.
|
|
335
|
-
// 2. `bun run <installed-bin>` falls back to `node_modules/.bin`.
|
|
336
|
-
// VERIFIED BY EXECUTION: `bun run tsc --version` → `Version 6.0.3`,
|
|
337
|
-
// exit 0; `bun run dev` → `error: Script not found "dev"`, exit 1.
|
|
338
|
-
// Caught LIVE at trial 19. It cost two MEDIUM trials, and BOTH were
|
|
339
|
-
// recovered by rescoring the stored text with no GPU — which is the
|
|
340
|
-
// whole reason the ledger stores `output`.
|
|
341
|
-
// 3. `-f` is a compose file only when a compose token precedes it, and
|
|
342
|
-
// `<runner> run <arg>` must be read for EVERY segment of a compound
|
|
343
|
-
// line, not the first. Found in review: `curl -f <url>` and
|
|
344
|
-
// `git clean -f -d` scored as invented paths (strict), while
|
|
345
|
-
// `bun run lint && bun run dev` scored `real` on `lint` alone and let
|
|
346
|
-
// `dev` — the exact class this cell turns on — through (LOOSE, and the
|
|
347
|
-
// harder half to spot). Caught after the run; rescoring the stored text
|
|
348
|
-
// moved 2/40 trials and left every number on this cell IDENTICAL.
|
|
349
|
-
// No fix moved the offline ceiling (32/45 tasks, 102/118 commands, before
|
|
350
|
-
// and after all three), so none loosened the check against known-good work.
|
|
351
|
-
//
|
|
352
|
-
// WHAT THIS CELL DOES NOT SAY. Medium sits at 20/20, so the axis has no
|
|
353
|
-
// headroom ABOVE it: a future run could not show medium getting worse. And
|
|
354
|
-
// the axis does not score the restart tail this worker is known for — no
|
|
355
|
-
// Trial field records a tool-call count — so "medium is better here" is
|
|
356
|
-
// about the ANSWER, not about the wander.
|
|
357
|
-
//
|
|
358
|
-
// REPRODUCE: `AB_SPECS=research:tooling:20 /abrun/run-group.sh`, then
|
|
359
|
-
// AB_CORPUS=.../mx5-copy bun run scripts/rescore-reasoning-ledger.ts \
|
|
360
|
-
// /home/edgars/hub/ab-grouplab/ledger-research-tooling.jsonl \
|
|
361
|
-
// research:tooling --from-text
|
|
362
47
|
'research:tooling': 'medium',
|
|
363
|
-
// NOT MEASURED. DECIDED BY PRIOR, 2026-08-27 — the same prior that carries
|
|
364
|
-
// every rung-3 cell in this table: thinking that buys nothing measurable is
|
|
365
|
-
// not worth its tokens. `off`.
|
|
366
|
-
//
|
|
367
|
-
// WHY A DECISION AND NOT `inherit`. All eight phase children run UNATTENDED
|
|
368
|
-
// — refine, grill-gen, grill-auto, verify-tooling, compose, critique,
|
|
369
|
-
// critique-triage, compress-label. Nobody is watching to turn a knob, so
|
|
370
|
-
// `inherit` does not defer to a judgement, it defers to whatever
|
|
371
|
-
// ~/.pi/agent/settings.json happens to hold. That is the one case this
|
|
372
|
-
// table exists to remove. (`plan` is the opposite case and keeps `inherit`
|
|
373
|
-
// for exactly the opposite reason — see that cell.)
|
|
374
|
-
//
|
|
375
|
-
// WHY `off` RATHER THAN `medium`. The one cell decided on QUALITY is
|
|
376
|
-
// extraction at rung 1, where off beat medium 20/20 vs 15/20 and medium's
|
|
377
|
-
// failure mode was STITCHING — concatenating non-contiguous passages into
|
|
378
|
-
// something that reads verbatim. refine, compose and critique all emit prose
|
|
379
|
-
// that downstream phases consume as fact, which is the same hazard.
|
|
380
|
-
// `planning` went `medium`, and it is the nearest neighbour by job, but it
|
|
381
|
-
// is NOT evidence that thinking helps this class: off led that cell on
|
|
382
|
-
// quality too, 28/30 vs 26/30, and lost only the clock, because it wrote
|
|
383
|
-
// more. Nothing measured anywhere in this table shows thinking improving a
|
|
384
|
-
// synthesis child's output.
|
|
385
|
-
//
|
|
386
|
-
// WHY IT COULD NOT BE MEASURED. Three candidate axes were each scored
|
|
387
|
-
// against refine's OWN RECORDED OUTPUT and rejected before any GPU:
|
|
388
|
-
// `validateRefineShape` scores 55/56 — saturated; "EXTERNAL-DEPENDENCIES
|
|
389
|
-
// names a real package" finds 4 distinct packages across all 56 tasks — no
|
|
390
|
-
// signal; "every backticked path exists" scored 56.2%, 6/56 perfect — a bar
|
|
391
|
-
// the known-good answer could not clear.
|
|
392
|
-
//
|
|
393
|
-
// THAT THIRD AXIS WAS RE-AUDITED 2026-08-27, because planning's citation
|
|
394
|
-
// axis was rejected the same way at 59.2% and reached 97.1% once four
|
|
395
|
-
// adjudicator bugs were fixed. THE CHECK WAS INDEED LOSING.
|
|
396
|
-
// scripts/phase-path-axis-audit.ts walks the ladder:
|
|
397
|
-
//
|
|
398
|
-
// NAIVE 60.1% of paths, 5/55 tasks perfect
|
|
399
|
-
// CATEGORY-CLEAN 83.8% of paths, 34/52 tasks perfect
|
|
400
|
-
// FINAL-TREE 93.4% of paths, 43/52 tasks perfect
|
|
401
|
-
//
|
|
402
|
-
// CATEGORY-CLEAN drops the spans that were never repo paths: npm specifiers
|
|
403
|
-
// (`@hono/zod-validator`, `hono/client`), doc URLs, MIME types, dotted code
|
|
404
|
-
// expressions (`c.var.user` — nine of them), bare filenames with no
|
|
405
|
-
// directory, and `./`/`../` import specifiers. FINAL-TREE additionally counts
|
|
406
|
-
// a path real if it exists anywhere in the tree the RUN shipped, which stops
|
|
407
|
-
// marking a CORRECT PREDICTION wrong — 16 of the 27 remaining misses are
|
|
408
|
-
// files a later task really creates.
|
|
409
|
-
//
|
|
410
|
-
// AND IT IS STILL NOT A USABLE AXIS. At 93.4% the eleven residual misses are
|
|
411
|
-
// almost all PATH-PREFIX ELISIONS — `server/index.ts` for
|
|
412
|
-
// `src/server/index.ts`, `client/main.tsx` for `src/client/main.tsx`.
|
|
413
|
-
// Loosen enough to accept a suffix match and it saturates; keep them and the
|
|
414
|
-
// A/B measures whether refine writes the `src/` prefix. Formatting, not
|
|
415
|
-
// reasoning. DO NOT RE-DERIVE THIS.
|
|
416
|
-
//
|
|
417
|
-
// ledger-phase.VOID-wrong-scorer.jsonl stores no output text, so its apparent
|
|
418
|
-
// off 2/14 vs medium 6/14 is an artefact that cannot be rescored. DO NOT READ
|
|
419
|
-
// A RESULT INTO IT.
|
|
420
|
-
//
|
|
421
|
-
// TO REPLACE THIS WITH A MEASUREMENT: find a bar refine's own recorded output
|
|
422
|
-
// clears while still leaving headroom, and that is not a property of its
|
|
423
|
-
// formatting. Four candidates are now dead; a fifth needs a different kind of
|
|
424
|
-
// truth — an EXECUTION, the way gate and implementation are scored, not a
|
|
425
|
-
// property of the text.
|
|
426
48
|
phase: 'off',
|
|
427
|
-
// A/B 2026-08-27, Qwen3.8-27B-NVFP4-MTP-VERY-HIGH.gguf (llama.cpp
|
|
428
|
-
// b10620-0f3b51e03), scripts/live-reasoning-group-ab.ts, n=30/arm over the
|
|
429
|
-
// committed mx5 fixture replayed 30 times. Ledger: ledger-planning.jsonl.
|
|
430
|
-
//
|
|
431
|
-
// THE FIRST CELL THAT IS NOT `off`, and the only one where the two axes
|
|
432
|
-
// point in opposite directions. Read both.
|
|
433
|
-
//
|
|
434
|
-
// off 28/30 faithful plans vs medium 26/30 (p=0.6707); neither arm ever
|
|
435
|
-
// failed to terminate. Wall clock, UNPAIRED — one fixture, so there are no
|
|
436
|
-
// distinct stimuli to pair by and `pairByStimulus` refuses rather than
|
|
437
|
-
// invent a pairing: mean off 207.3s vs medium 143.6s, p=0.0241.
|
|
438
|
-
// off faithful-plan 95% CI [0.79, 0.98].
|
|
439
|
-
// RUNG 2 — quality is level on an axis with headroom, and medium is 1.4x
|
|
440
|
-
// faster at equal quality. The ladder is symmetric: `off` losing the clock
|
|
441
|
-
// is not privileged over `medium` losing it.
|
|
442
|
-
//
|
|
443
|
-
// THE CLOCK CAVEAT, because the direction is the opposite of every other
|
|
444
|
-
// cell. off's mean is tail-driven — medians are 157.6s vs 148.2s, ~6% apart
|
|
445
|
-
// — and off emits a LONGER plan for the same title count (median 6840 vs
|
|
446
|
-
// 4988 chars, 13 titles both arms). Per character off is the faster arm,
|
|
447
|
-
// 23.0 vs 29.7 ms/char. So what the clock records here is "off writes more",
|
|
448
|
-
// not "thinking is free". Production waits for the whole plan, so the cost
|
|
449
|
-
// is real; it is not evidence that thinking decodes faster.
|
|
450
|
-
//
|
|
451
|
-
// THE AXIS IS `planningPlanFaithful`, built for this run:
|
|
452
|
-
// a plan is FAITHFUL when it lists >= 2 titles AND EVERY source clause it
|
|
453
|
-
// emitted — counted in the RAW TITLE — comes back grounded.
|
|
454
|
-
// It is production's own adjudicator (`extractTitleSource`) with no model in
|
|
455
|
-
// the loop, and the source doc is a committed fixture, so every stored trial
|
|
456
|
-
// stays rescorable with no corpus:
|
|
457
|
-
// bun run scripts/rescore-reasoning-ledger.ts \
|
|
458
|
-
// ab-grouplab/ledger-planning.jsonl planning --from-text
|
|
459
|
-
// Counting the clauses in the RAW title is load-bearing: a malformed clause
|
|
460
|
-
// (a missing closing quote — measured live, 4 trials) stops the peel, and a
|
|
461
|
-
// scorer that counted only what it peeled reads everything before the break
|
|
462
|
-
// as a clean sweep.
|
|
463
|
-
//
|
|
464
|
-
// THE OLD AXIS WAS A SHAPE CHECK. `parseDecomposeList >= 2` read 10/10 in
|
|
465
|
-
// both arms at n=10/arm — saturated, the same death as gate's and phase's
|
|
466
|
-
// first scorers. ledger-planning.SHAPE-AXIS-no-extension-10rep.jsonl is that
|
|
467
|
-
// run; it is a PRIOR, NOT A REPLICATE, because it also ran a different child
|
|
468
|
-
// (no single-read extension). Do not pool it.
|
|
469
|
-
//
|
|
470
|
-
// THE ADJUDICATOR HAD TO BE FIXED FOUR TIMES BEFORE IT COULD JUDGE, and
|
|
471
|
-
// every fix was found by auditing failures row by row, not by reading code:
|
|
472
|
-
// 1. GREEDY REGEX across multi-clause titles (25% of real titles carry
|
|
473
|
-
// more than one clause) — two real citations became one superstring.
|
|
474
|
-
// 2. MARKDOWN EMPHASIS/LIST MARKERS counted as content.
|
|
475
|
-
// 3. CODE BACKTICKS counted as content — the larger half of (2), and
|
|
476
|
-
// found only after this run. A code span renders as bare text, so
|
|
477
|
-
// `Invites — create/validate/redeem, /join/:token page.` is a verbatim
|
|
478
|
-
// copy of a line the file stores with backticks. Screening EVERY spec
|
|
479
|
-
// line of the fixture in its rendered form: 107/216 grounded before,
|
|
480
|
-
// 216/216 after, floor 0/216.
|
|
481
|
-
// 4. BACKSLASH-ESCAPED QUOTES. The clause is double-quoted, so a spec line
|
|
482
|
-
// containing a double quote comes back as `\"`. The backslash is the
|
|
483
|
-
// delimiter's artefact, not content.
|
|
484
|
-
// The live run scored off 21/30 vs medium 24/30 and printed `medium`; fixes
|
|
485
|
-
// 3 and 4 moved 9 of 60 trials and the QUALITY ORDER REVERSED, to off 28/30
|
|
486
|
-
// vs medium 26/30. The verdict survives only because the clock decides at
|
|
487
|
-
// rung 2. scripts/decompose-fidelity-screen.ts is the standing screen that
|
|
488
|
-
// would have caught 3 and 4 before the GPU ran.
|
|
489
|
-
//
|
|
490
|
-
// THE REMAINING 6 FAILURES ARE GENUINE, checked by hand: 4 malformed clauses
|
|
491
|
-
// (missing closing quote or bracket), 1 word substituted in a real line
|
|
492
|
-
// ("invokes" for "invites"), 1 single-quoted where the doc has double.
|
|
493
|
-
//
|
|
494
|
-
// THE HARNESS SPAWNS PRODUCTION'S CHILD. `phaseDeps()` passed no
|
|
495
|
-
// `childExtensions`, so auto-decompose ran WITHOUT the single-read guard
|
|
496
|
-
// production hands every planning child (auto-orchestrator.ts). Fixed before
|
|
497
|
-
// this run; `loadableSingleReadExtension()` maps src->dist and ABSTAINS if
|
|
498
|
-
// neither exists, so `bun run build` is a precondition.
|
|
499
|
-
//
|
|
500
|
-
// THE TARGET THIS CELL ONCE CHASED DOES NOT EXIST. "off ~1/10 vs on ~8/8
|
|
501
|
-
// from magicknumbers.md" was cited here and in this file's header as the
|
|
502
|
-
// split a positive control had to reproduce. Verified 2026-08-27: that page
|
|
503
|
-
// has no "8/8" in it and never did, and its `1/10 -> 7/10 -> 8/9` ladder is
|
|
504
|
-
// labelled REASONING OFF, n=10 per cell — three GUARDS from `fea7bbb`, one
|
|
505
|
-
// arm throughout. See the header.
|
|
506
|
-
//
|
|
507
|
-
// A pre-`fea7bbb` control tree WAS built and verified anyway
|
|
508
|
-
// (ab-grouplab/make-preguard-tree.sh + preguard-probe.ts, all four guard
|
|
509
|
-
// behaviours rolled back and asserted both ways). It is the only planning
|
|
510
|
-
// regime known to have headroom at reasoning off — 1/10 there — but it is a
|
|
511
|
-
// regime we deleted, so a cell decided in it would not be writable here.
|
|
512
|
-
// Left unrun on purpose.
|
|
513
|
-
//
|
|
514
|
-
// MEASURED UNDER THE SERVER'S GLOBAL SAMPLER, which is the THINKING preset.
|
|
515
|
-
// Here that cuts AGAINST the written cell rather than for it: the `off` arm
|
|
516
|
-
// decoded on sampling tuned for `medium` and still led on quality, so a
|
|
517
|
-
// clean comparison could only widen off's quality margin — which the clock
|
|
518
|
-
// then has to overcome. Ecologically valid, not clean.
|
|
519
49
|
planning: 'medium',
|
|
520
|
-
// `inherit` ON PURPOSE, and this is the one cell where that is an ANSWER
|
|
521
|
-
// rather than an absence of one.
|
|
522
|
-
//
|
|
523
|
-
// /task-plan is INTERACTIVE. The user is sitting in the loop, reading each
|
|
524
|
-
// question and steering the next one, and how much thinking that wants is
|
|
525
|
-
// theirs to judge per session — a quick sketch and a hard architectural
|
|
526
|
-
// plan are the same command. Every other group in this table runs
|
|
527
|
-
// unattended, where nobody is there to turn a knob and the table has to
|
|
528
|
-
// decide. Here there is, and it should not be overridden.
|
|
529
|
-
//
|
|
530
|
-
// `inherit` means the child gets whatever `~/.pi/agent/settings.json` holds
|
|
531
|
-
// — that is the general objection to it, and here it is exactly the point:
|
|
532
|
-
// the setting the user chose is the setting the user gets.
|
|
533
|
-
//
|
|
534
|
-
// It is also unmeasured: /task-plan has never executed inside the A/B
|
|
535
|
-
// corpus, so there is no recorded child turn to replay. But that is not why
|
|
536
|
-
// the cell reads `inherit`. Recording a run and measuring the group would
|
|
537
|
-
// not change it.
|
|
538
50
|
plan: 'inherit',
|
|
539
|
-
// A/B 2026-08-26, Qwen3.8-27B-NVFP4-MTP-VERY-HIGH.gguf (llama.cpp
|
|
540
|
-
// b10620-0f3b51e03), scripts/live-reasoning-group-ab.ts, n=30/arm.
|
|
541
|
-
//
|
|
542
|
-
// SCORED AGAINST THE TREE, NOT THE ANSWER'S SHAPE. Ten mx5 tasks were
|
|
543
|
-
// screened by EXECUTING each one's own VERIFY with no model in the loop:
|
|
544
|
-
// it fails on the before-tree and passes on the after-tree. So the child
|
|
545
|
-
// faces 20 (task, tree) pairs whose correct verdict is a fact — FAIL before,
|
|
546
|
-
// PASS after — balanced, so always-PASS and always-FAIL each score exactly
|
|
547
|
-
// 50%. The verdict is read by production's own `parseVerifyVerdict`.
|
|
548
|
-
// UNOBSERVED scores WRONG: the harness just executed the evidence, so
|
|
549
|
-
// declining to look is a failure to do the job.
|
|
550
|
-
//
|
|
551
|
-
// off 28/30 correct verdicts vs medium 28/30 (p=1.0000); neither arm ever
|
|
552
|
-
// failed to emit a verdict. Wall clock, PAIRED by stimulus over the 27
|
|
553
|
-
// pairs usable in both arms: off is faster in 23 of them, geometric mean
|
|
554
|
-
// 0.61x, p=0.0019. off correct-verdict 95% CI [0.79, 0.98].
|
|
555
|
-
// RUNG 2 — quality is level on an axis with headroom, and off is 1.6x
|
|
556
|
-
// faster at equal correctness. Not a prior: the axis recorded four genuine
|
|
557
|
-
// judgement errors across the run, so it could have separated the arms and
|
|
558
|
-
// did not.
|
|
559
|
-
//
|
|
560
|
-
// TWO INSTRUMENT NOTES, because this cell read differently before both.
|
|
561
|
-
// (1) The clock test was UNPAIRED on a matched design. Every stimulus runs
|
|
562
|
-
// once per arm, and the stimulus dominates: the same child is ~25s on a
|
|
563
|
-
// before-tree and 100-500s on an after-tree. Pooling buried the arm
|
|
564
|
-
// effect. On these numbers the unpaired test reads p=0.3408 and the
|
|
565
|
-
// paired one p=0.0019. Fixed in `pairedPermutationP`.
|
|
566
|
-
// (2) The 20-rep run alone scored 20/20 vs 20/20 — SATURATED, and the
|
|
567
|
-
// ladder now refuses rung 2 there. The 10-rep run over the same 20
|
|
568
|
-
// stimuli, same fingerprint, same scorer, scored 8/10 vs 8/10. Pooled
|
|
569
|
-
// (ledger-gate.POOLED-10rep+20rep.jsonl, pairs keyed by run so the
|
|
570
|
-
// pairing stays within-run) the axis has headroom and n=30/arm.
|
|
571
|
-
// Superseded ledgers kept beside it: VOID-synthetic-prompt (hand-written
|
|
572
|
-
// prompt, prose-matching scorer) and VOID-saturated-shape-axis.
|
|
573
|
-
//
|
|
574
|
-
// AXIS RE-AUDITED 2026-08-27, all four mismatches read by hand. The PARSER
|
|
575
|
-
// is clean — `verdictWord` extracted the stated word correctly every time.
|
|
576
|
-
// THE TRUTH IS NOT, on one stimulus. TASK_0009/after is `PASS` because its
|
|
577
|
-
// own VERIFY script passes there, and BOTH arms answered FAIL with the same
|
|
578
|
-
// checkable reason: the acceptance list requires phone validation to REJECT
|
|
579
|
-
// `+1234567`, while the spec's own mandated regex `/^\+[1-9]\d{6,14}$/`
|
|
580
|
-
// accepts it (`+` `1` then six digits, and the range starts at six), and the
|
|
581
|
-
// shipped test omits the case. Verified by reading the regex, not by
|
|
582
|
-
// trusting the model. So "the task's VERIFY passes" is not the same fact as
|
|
583
|
-
// "the acceptance criteria are met", and on that stimulus the child was
|
|
584
|
-
// right and the axis was wrong.
|
|
585
|
-
// THE CELL IS UNCHANGED: both arms fail it identically, so dropping it
|
|
586
|
-
// leaves off 28/29 vs medium 28/29 — still level, still rung 2. Recorded
|
|
587
|
-
// because the next axis built on executed VERIFY should expect this gap.
|
|
588
51
|
gate: 'off',
|
|
589
|
-
// A/B 2026-08-26, Qwen3.8-27B-NVFP4-MTP-VERY-HIGH.gguf (llama.cpp
|
|
590
|
-
// b10620-0f3b51e03), scripts/live-reasoning-group-ab.ts, n=20/arm over 20
|
|
591
|
-
// recorded docs queries across 10 packages. off 20/20 usable vs medium
|
|
592
|
-
// 15/20 (p=0.0471); neither arm failed to answer. off usable 95% CI
|
|
593
|
-
// [0.84, 1.00]. Wall clock, PAIRED over the 15 stimuli usable in both arms:
|
|
594
|
-
// off 5.2s vs medium 11.1s, p=0.0010.
|
|
595
|
-
// RUNG 1 — the ONLY cell in this table decided by a quality difference, and
|
|
596
|
-
// the clock agrees with it rather than carrying it. Ledger:
|
|
597
|
-
// ledger-extraction.jsonl.
|
|
598
|
-
//
|
|
599
|
-
// THE PROMPT IS PRODUCTION'S. The harness used to hand-write it, because
|
|
600
|
-
// the two production builders frame content as a named npm package's docs
|
|
601
|
-
// or an anchored web page and the material it had — a recorded `## research`
|
|
602
|
-
// section — is neither. The fix was the right MATERIAL, not a better frame:
|
|
603
|
-
// `.pi-tasks/research-cache.json` records 190 distinct `pi-worker-docs`
|
|
604
|
-
// (package, query) pairs the run really asked, over 31 packages installed in
|
|
605
|
-
// the corpus copy, and all 190 replay through `docsRaw` + `buildPrompt` with
|
|
606
|
-
// no model and no network. See scripts/reasoning-ab-extraction-truth.ts.
|
|
607
|
-
//
|
|
608
|
-
// THE AXIS IS THE CONJUNCTION: a non-empty answer AND a citation that is
|
|
609
|
-
// really in the content the child was shown. Production gates on the first
|
|
610
|
-
// half only and carries `excerptVerified` as metadata, so this is a HARDER
|
|
611
|
-
// bar than production's own — legitimate for an A/B, and necessary because
|
|
612
|
-
// the shape half is the ceiling that returned 10/10 in both arms for gate,
|
|
613
|
-
// research and planning. SCREENED offline over all 190 recorded answers
|
|
614
|
-
// before any GPU:
|
|
615
|
-
// production's own excerptVerified 189/190 — clears the bar
|
|
616
|
-
// every backticked SPAN in the content 51/190 — the CHECK loses
|
|
617
|
-
// code-shaped identifiers only 123/190 — the CHECK loses
|
|
618
|
-
// Both grounding rules die the way phase's three did, for a nameable reason:
|
|
619
|
-
// real type names the model correctly knows — `ResponseInit`,
|
|
620
|
-
// `ArrayBufferView`, `DataTransfer` — are simply not in the retrieved
|
|
621
|
-
// chunks, so a correct answer is marked wrong.
|
|
622
|
-
//
|
|
623
|
-
// WHAT MEDIUM ACTUALLY DOES WRONG, audited row by row rather than trusted.
|
|
624
|
-
// All five failures are genuine and none is a normaliser gap: re-checked
|
|
625
|
-
// with whitespace and case squashed out, ZERO of them appear in the content.
|
|
626
|
-
// The failure mode is STITCHING — the arm concatenates non-contiguous
|
|
627
|
-
// passages into one quote block that reads as verbatim and is not. One
|
|
628
|
-
// matched 457 of its 522 squashed characters before diverging; another glued
|
|
629
|
-
// three separate `ts fences and a trailing `export type` line. It is not
|
|
630
|
-
// inventing the package, it is inventing the CONTIGUITY, which is exactly
|
|
631
|
-
// what a citation asserts.
|
|
632
|
-
//
|
|
633
|
-
// A RESCORE OF THIS GROUP RE-RETRIEVES, and that is a trap the ledger now
|
|
634
|
-
// guards. Two trials that verified in the container failed when rescored on
|
|
635
|
-
// the host, both quoting a real bun declaration (`@deprecated Prefer
|
|
636
|
-
// {@link Bun.sql}`) the host's newer bun does not ship — enough to move the
|
|
637
|
-
// cell from rung 1 to rung 2 on an artefact of WHERE the rescorer ran.
|
|
638
|
-
// Retrieval is deterministic within one environment and NOT across two, so
|
|
639
|
-
// the row now carries `verifyHash` and rescore-reasoning-ledger.ts ABSTAINS
|
|
640
|
-
// on a mismatch. This ledger predates the field, so the live judgements —
|
|
641
|
-
// made against exactly the bytes each child was shown — are authoritative.
|
|
642
|
-
//
|
|
643
|
-
// MEASURED UNDER THE SERVER'S GLOBAL SAMPLER, which is the THINKING preset,
|
|
644
|
-
// so the `off` arm decodes on sampling tuned for the `on` arm. That is the
|
|
645
|
-
// regime this machine really runs pi-task in, so the result is
|
|
646
|
-
// ecologically valid — it is NOT a clean comparison. Here it makes the
|
|
647
|
-
// finding CONSERVATIVE: the arm that wins is the one running on the other
|
|
648
|
-
// arm's sampler.
|
|
649
52
|
extraction: 'off',
|
|
650
|
-
// A/B 2026-08-25, Qwen3.8-27B-NVFP4-MTP-VERY-HIGH.gguf (llama.cpp
|
|
651
|
-
// b10618-1efd800e9), scripts/live-implementation-thinking-ab.ts, n=20/arm
|
|
652
|
-
// over 20 distinct mx5 specs, scored by each task's OWN recorded VERIFY
|
|
653
|
-
// block against the tree the turn produced. off 12/20 pass vs medium 12/20
|
|
654
|
-
// (p=1.0000); turn died 6/20 both arms. off pass 95% CI [0.39, 0.78].
|
|
655
|
-
// RUNG 2, and it was read as rung 3 until 2026-08-26. The quality axes are
|
|
656
|
-
// identical to the trial, so the clock decides — and the clock test was
|
|
657
|
-
// UNPAIRED on a design that runs each spec once per arm. Pooled, the mean
|
|
658
|
-
// wall clock of passing turns is off 237s vs medium 298s, p=0.6150. Paired
|
|
659
|
-
// by spec over the 12 specs that pass in BOTH arms: off faster in 9,
|
|
660
|
-
// geometric mean 0.55x, p=0.0166.
|
|
661
|
-
// Quality is still the widest CI in the table — at 12/20 the true pass rate
|
|
662
|
-
// is anywhere from 39% to 78%, so an ordinary quality difference would have
|
|
663
|
-
// been invisible — but 12/20 is not a ceiling, so the clock is allowed to
|
|
664
|
-
// carry the cell. The value is unchanged; only its standing improved, from
|
|
665
|
-
// a stated prior to a measured win. Rescored from the original ledger on 2026-08-25 when the harness
|
|
666
|
-
// moved to a forced two-way verdict; the trials are unchanged.
|
|
667
|
-
//
|
|
668
|
-
// OVERRIDE WITHDRAWN 2026-08-28; THE CELL IS `off` AGAIN, ON ITS OWN
|
|
669
|
-
// MEASUREMENT. Between 2026-08-27 and 2026-08-28 this cell read `medium` by
|
|
670
|
-
// user decision, overriding the rung-2 result above rather than a prior. The
|
|
671
|
-
// decision's stated reason was an implementation turn observed wandering in
|
|
672
|
-
// the mx5-n run, and the cell recorded honestly that the run's .pi-tasks logs
|
|
673
|
-
// had been destroyed before anyone could count the loop.
|
|
674
|
-
//
|
|
675
|
-
// A SECOND mx5-n RUN KEPT ITS LOGS, and they refute the reason. The run
|
|
676
|
-
// executed 10 tasks in 9h22 with this cell at `medium` — thinking ON for
|
|
677
|
-
// every implementation turn — and TASK_0004 wandered anyway: 167 assistant
|
|
678
|
-
// turns, 192 tool calls, 45 files and 2,355 lines committed against a
|
|
679
|
-
// one-file spec, and its own deliverable (src/server/migrate.ts) never
|
|
680
|
-
// written; it arrived four tasks later. Two single responses in that turn
|
|
681
|
-
// were 18,297 and 23,568 output tokens. Thinking did not prevent the
|
|
682
|
-
// rotation, so the lever the override was reaching for is not this one.
|
|
683
|
-
// Measured over the run's 15 implementation sessions: 441,399 output tokens,
|
|
684
|
-
// of which 59.3% by character was thinking (1,314,790 thinking chars vs
|
|
685
|
-
// 902,034 text chars).
|
|
686
|
-
//
|
|
687
|
-
// WHAT IS STRUCTURALLY TRUE, and still true, and checked: the implementation
|
|
688
|
-
// turn runs in the USER'S OWN SESSION (see task/implementation-thinking.ts
|
|
689
|
-
// and task/implementation-turn.ts), not as a child. It therefore has NO
|
|
690
|
-
// LoopDetector and NO StallDetector — only the per-tool-call command
|
|
691
|
-
// watchdog and superviseImplementation's MAX_COMPACTION_RESUMES, which the
|
|
692
|
-
// runaway above came nowhere near (it took 2 compactions of an allowed 20).
|
|
693
|
-
// So `implementation` remains the one group in this table where a rotation
|
|
694
|
-
// has no guard at all. That is an argument for GIVING it a guard, not for
|
|
695
|
-
// paying 1.8x in thinking that has now been observed not to guard it.
|
|
696
|
-
//
|
|
697
|
-
// THE CELL IS STILL RUNG 2 AND STILL THE WIDEST CI IN THE TABLE ([0.39,
|
|
698
|
-
// 0.78] at 12/20), so it is carried by the clock, not by quality. Nothing
|
|
699
|
-
// measured since has withdrawn the paired-clock win. To put it on more
|
|
700
|
-
// evidence, re-run scripts/live-implementation-thinking-ab.ts at a larger n;
|
|
701
|
-
// the ledger is /home/edgars/hub/ab-implab/impl-ledger.jsonl, 40 rows,
|
|
702
|
-
// b10618 — the odd build out, so a re-run on b10620 is not a replicate.
|
|
703
53
|
implementation: 'off'
|
|
704
54
|
};
|
|
705
|
-
/**
|
|
706
|
-
* A hand-edited or stale mode must not reach {@link resolveReasoning}'s switch as
|
|
707
|
-
* an unknown string — the `default:` arm would silently absorb it and the user
|
|
708
|
-
* would see "custom" in the file and the default table in behaviour.
|
|
709
|
-
*/
|
|
55
|
+
/** A stored mode, or `default` when the value is not one. */
|
|
710
56
|
export function sanitizeReasoningMode(value) {
|
|
711
57
|
return REASONING_MODES.includes(value) ? value : 'default';
|
|
712
58
|
}
|
|
713
59
|
/**
|
|
714
|
-
* Child NAME → reasoning group, for every child
|
|
715
|
-
* `runPhaseChild`
|
|
60
|
+
* Child NAME → reasoning group, for every child spawned under a name:
|
|
61
|
+
* `runPhaseChild`, `runPlanningChild`, and the research workers' `spec.label`.
|
|
716
62
|
*
|
|
717
63
|
* WHY KEYED ON THE NAME
|
|
718
64
|
* ---------------------
|
|
@@ -724,15 +70,14 @@ export function sanitizeReasoningMode(value) {
|
|
|
724
70
|
* express something the call site already says out loud.
|
|
725
71
|
*
|
|
726
72
|
* AN UNMAPPED NAME IS A BUILD FAILURE, not a silent `inherit`.
|
|
727
|
-
* `reasoning-groups.test.ts` scans every literal child name
|
|
728
|
-
* it is missing here. A defaulting lookup would let a phase added
|
|
729
|
-
* itself out of
|
|
730
|
-
* how `/no_think` ended up applied to eight prompts and read by none of them.
|
|
73
|
+
* `reasoning-groups.test.ts` scans every literal child name under `src/task`
|
|
74
|
+
* and fails if it is missing here. A defaulting lookup would let a phase added
|
|
75
|
+
* later opt itself out of the table without anyone deciding to.
|
|
731
76
|
*
|
|
732
77
|
* The gate and extraction groups are NOT here: those children reach the model
|
|
733
|
-
* through `
|
|
734
|
-
*
|
|
735
|
-
* `spec.label` —
|
|
78
|
+
* through `groupThinkingArgs('gate' | 'extraction')` at call sites with no name
|
|
79
|
+
* in scope (gate-deps.ts, fetch-core.ts, docs-core.ts, pi-worker-docs.ts). The
|
|
80
|
+
* four research workers do have a name — their `spec.label` — so they are here.
|
|
736
81
|
*/
|
|
737
82
|
export const REASONING_GROUP_BY_CHILD = {
|
|
738
83
|
// ── phase: task/phases.ts + task/title-label.ts ──────────────────────────
|
|
@@ -757,11 +102,6 @@ export const REASONING_GROUP_BY_CHILD = {
|
|
|
757
102
|
'plan-question': 'plan',
|
|
758
103
|
'plan-answer': 'plan',
|
|
759
104
|
// ── research: task/phases.ts `workerSpecs`, keyed on the spec's LABEL ─────
|
|
760
|
-
// These were a second table (`RESEARCH_WORKER_GROUPS`) keyed on the section
|
|
761
|
-
// heading, with a silent `?? 'research'` fallback and a guard that sliced
|
|
762
|
-
// phases.ts source between two string offsets from the config directory.
|
|
763
|
-
// The label is the same name the loader, the debug trail and the A/B ledgers
|
|
764
|
-
// already print, so they belong here with every other named child.
|
|
765
105
|
'worker:files': 'research:files',
|
|
766
106
|
'worker:apis': 'research:apis',
|
|
767
107
|
'worker:context': 'research:context',
|
|
@@ -810,22 +150,9 @@ export function sanitizeReasoningLevels(value) {
|
|
|
810
150
|
out[group] = stored_;
|
|
811
151
|
continue;
|
|
812
152
|
}
|
|
813
|
-
// A `research:*` key
|
|
814
|
-
// `research
|
|
815
|
-
//
|
|
816
|
-
// nothing else, and filling the four sub-cells from the table would
|
|
817
|
-
// silently overrule a choice the user had already made — the four
|
|
818
|
-
// workers are exactly the children that `research` used to set.
|
|
819
|
-
//
|
|
820
|
-
// IT CANNOT TELL A CHOICE FROM A SEED, and that is accepted. Leaving
|
|
821
|
-
// `default`/`on`/`off` for ANY reason makes `applyReasoningLevel` seed
|
|
822
|
-
// every group from `resolveReasoning`, so a user who only ever nudged
|
|
823
|
-
// `gate` still has a `research` on disk they never picked. Inheriting
|
|
824
|
-
// it pins the four workers to the old default and the measured
|
|
825
|
-
// `research:files: 'off'` never reaches them. That is what CUSTOM MODE
|
|
826
|
-
// MEANS: a frozen table, not a subscription to later measurements — the
|
|
827
|
-
// same is true of every other group in the file. `default` mode is
|
|
828
|
-
// where a new reading takes effect, and it takes effect there at once.
|
|
153
|
+
// A `research:*` key the stored config never had falls back to its
|
|
154
|
+
// parent `research` value, so a config written before the split keeps
|
|
155
|
+
// meaning what it meant.
|
|
829
156
|
const parent = RESEARCH_SUBGROUP_PARENT[group];
|
|
830
157
|
out[group] =
|
|
831
158
|
parent && valid(stored[parent]) ? stored[parent] : DEFAULT_REASONING_TABLE[group];
|
|
@@ -855,14 +182,10 @@ export function resolveReasoning(group, cfg) {
|
|
|
855
182
|
/**
|
|
856
183
|
* The WHOLE table, as this config will actually run it.
|
|
857
184
|
*
|
|
858
|
-
*
|
|
859
|
-
*
|
|
860
|
-
*
|
|
861
|
-
*
|
|
862
|
-
* invented (`Array<{group, setting}>`) and leaked into `reasoningMismatches`.
|
|
863
|
-
*
|
|
864
|
-
* `resolveReasoning` stays: one group is still a fair question, and it is the
|
|
865
|
-
* only place the four modes are interpreted.
|
|
185
|
+
* The question the mismatch scan and the custom-mode seeder both ask. One
|
|
186
|
+
* accessor, so they cannot each invent their own shape for it. The settings menu
|
|
187
|
+
* asks the single-group question instead, through `resolveReasoning` — which is
|
|
188
|
+
* the only place the four modes are interpreted.
|
|
866
189
|
*/
|
|
867
190
|
export function effectiveReasoning(cfg) {
|
|
868
191
|
const out = {};
|