devlyn-cli 2.10.1 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -9
- package/CLAUDE.md +17 -17
- package/README.md +33 -9
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +25 -0
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +10 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +30 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +32 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +44 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +16 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +15 -0
- package/bin/devlyn.js +9 -2
- package/config/skills/_shared/.ruff.toml +2 -0
- package/config/skills/_shared/adapters/claude.md +9 -1
- package/config/skills/_shared/adapters/codex.md +5 -0
- package/config/skills/_shared/adapters/grok.md +4 -2
- package/config/skills/_shared/archive_run.py +738 -56
- package/config/skills/_shared/codex-config.md +19 -4
- package/config/skills/_shared/codex-monitored.sh +81 -12
- package/config/skills/_shared/collect-codex-findings.py +311 -128
- package/config/skills/_shared/engine-preflight.md +8 -7
- package/config/skills/_shared/expected.schema.json +61 -0
- package/config/skills/_shared/finish-gate.py +12 -11
- package/config/skills/_shared/grok-anchor-guard.py +18 -1
- package/config/skills/_shared/invocation-receipt.py +1041 -0
- package/config/skills/_shared/judge-output-parser.py +200 -0
- package/config/skills/_shared/judge-role-evidence.py +273 -0
- package/config/skills/_shared/process-evidence.py +909 -0
- package/config/skills/_shared/resolve-bootstrap.py +776 -84
- package/config/skills/_shared/resolve-stop-hook.py +33 -1
- package/config/skills/_shared/role-config.py +552 -0
- package/config/skills/_shared/run-bounded.py +1 -1
- package/config/skills/_shared/spec-verify-check.py +749 -181
- package/config/skills/_shared/state-phase-write.py +1838 -58
- package/config/skills/_shared/terminal-claim-check.py +15 -0
- package/config/skills/_shared/verify-merge-findings.py +958 -51
- package/config/skills/devlyn:engines/SKILL.md +8 -16
- package/config/skills/devlyn:ideate/SKILL.md +2 -2
- package/config/skills/devlyn:ideate/references/spec-template.md +6 -5
- package/config/skills/devlyn:queue/SKILL.md +7 -4
- package/config/skills/devlyn:resolve/SKILL.md +93 -35
- package/config/skills/devlyn:resolve/references/free-form-mode.md +1 -0
- package/config/skills/devlyn:resolve/references/outer-loop.md +15 -0
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +31 -2
- package/config/skills/devlyn:resolve/references/phases/implement.md +3 -2
- package/config/skills/devlyn:resolve/references/phases/verify.md +120 -84
- package/config/skills/devlyn:resolve/references/state-schema.md +22 -8
- package/package.json +5 -1
- package/scripts/lint-skills.sh +314 -285
package/AGENTS.md
CHANGED
|
@@ -4,7 +4,7 @@ devlyn-cli installs `/devlyn:ideate` (optional) and `/devlyn:resolve` (required)
|
|
|
4
4
|
|
|
5
5
|
## North Star
|
|
6
6
|
|
|
7
|
-
This contract serves one goal: any capable engine — Claude, GPT/Codex, or a future adapter-equipped model — takes a user's intent (prompt, spec, or queue entry) end-to-end to shipped, engineer-quality software, hands-free, with consistent quality across engines.
|
|
7
|
+
This contract serves one goal: any capable engine — Claude, GPT/Codex, or a future adapter-equipped model — takes a user's intent (prompt, spec, or queue entry) end-to-end to shipped, engineer-quality software, hands-free, with consistent quality across engines. When rules below conflict or feel ambiguous in context, resolve toward this goal.
|
|
8
8
|
|
|
9
9
|
## Core principles
|
|
10
10
|
|
|
@@ -15,7 +15,7 @@ Seven rules govern every change. Cite them by name when a decision touches one.
|
|
|
15
15
|
3. **No guesswork** — verify with the actual files, logs, diffs, and run output before forming conclusions. State the falsifiable prediction BEFORE the experiment; record raw results AFTER.
|
|
16
16
|
4. **Worldclass** — code that survives review at a non-trivial codebase. Zero CRITICAL, zero HIGH security/design findings on the shippable path.
|
|
17
17
|
5. **Best practice** — idiomatic for the language and framework. Use standard primitives; do not hand-roll what the library already provides.
|
|
18
|
-
6. **Optimized** — efficient
|
|
18
|
+
6. **Optimized** — use efficient code and avoid unnecessary work without sacrificing correctness or the user's requirements.
|
|
19
19
|
7. **Production ready** — error states are explicit and visible; behavior under failure is what the user expects, not silent corruption.
|
|
20
20
|
|
|
21
21
|
Three discipline rules govern HOW the principles are applied:
|
|
@@ -36,9 +36,13 @@ ideate (optional) -> resolve -> ship
|
|
|
36
36
|
|
|
37
37
|
Each skill's `SKILL.md` is the source of truth for flags and workflow. Do not duplicate.
|
|
38
38
|
|
|
39
|
-
Engine roles: orchestrator = whichever CLI the user opened (this contract is symmetric with CLAUDE.md
|
|
39
|
+
Engine roles: orchestrator = whichever CLI the user opened (this contract is symmetric with CLAUDE.md. If you are Codex orchestrating `/devlyn:resolve`, the phase machinery is mandatory regardless of task size; if you will not run it, say so explicitly and stop — do not silently degrade to ad-hoc execution); legacy executor (with optional separate worker/primary/pair profiles) uses the canonical skill's orchestrator-supported default, overridable per run with `--engine <name>` or durably via machine-local `.devlyn/engines.json` `{"executor": "<name>", "pair_judge_priority": ["<name>", ...]}`; PLAN is orchestrator-fixed and never inherits `--engine` or the executor pin; pair judge = first available OTHER engine, default for VERIFY whenever available; risk probes remain conditional and --no-pair opts out. Pins fail closed (`BLOCKED:<engine>-unavailable` / `BLOCKED:invalid-engine-config`); new engines plug in by shipping `_shared/adapters/<name>.md`. `/devlyn:engines` (no args) shows the role table + detected engines; `executor <name>` / `pair <name>,...` / `clear` manage the pins. The executor pin binds the orchestrator in plain conversation too, not only inside a skill run: when you would implement directly and executor is pinned to a non-default engine, route that work through the pin — via `/devlyn:resolve` (reads it at PHASE 0) or by delegating to that engine; no pin / no file → use the canonical skill's orchestrator-supported default. The pair-judge pin stays pipeline-scoped. `/devlyn:engines role` configures explicit engine/model/effort for supported worker and judge routes; `/devlyn:resolve --role-config <path>` overrides roles for one run. Canonical skills define precedence, evidence and unsupported-route errors; absent profiles preserve existing defaults.
|
|
40
40
|
|
|
41
|
-
Conversational
|
|
41
|
+
Conversational entry: Before writes, inspect requested files and relevant callers/tests. Default to direct execution only for clear, local, reversible, low-risk work with a decisive existing or small task-specific acceptance check. Preserve requested behavior, scope, explicit constraints and the executor pin; make scoped edits, run required checks, review the final diff, and report changes with evidence. One concise route/check explanation suffices; ordinary direct work needs no spec, pipeline state, phase workers or extra approval.
|
|
42
|
+
|
|
43
|
+
Use full `/devlyn:resolve` for material ambiguity, subsystem/design work, security/auth/payment/persistence/concurrency/public-API-contract changes, or inconclusive inspection/checks. Counts of lines, files or words alone do not establish low risk. Explicit resolve (even small), formal spec workflows and queue drains retain all canonical phases, independent verification, pins and failure handling. If new risk appears mid-edit, preserve the delta in the full run's scope/evidence; do not hide it in a new baseline or reconfirm existing authorization.
|
|
44
|
+
|
|
45
|
+
Full-route handoff + loop engineering: the orchestrating model — not the user — invokes the skills. `/devlyn:queue` fronts the intent queue (`docs/specs/queue.md`): no args = status, `add <intent>`, `drain` = serial unattended drain per the contract below. Large agreed work is written to `docs/specs/<id>/spec.md` (always a spec file — the user's reviewed contract; free-form large assumes-and-logs instead of halting, zero-scope-signal goals still halt to ideate), summarized once for user review, then run hands-free via `--spec`. Outer loop per task: before the first full run, commit the scoped queue-item delta (when applicable) and linked spec bundle; findings-backed verdicts (NEEDS_WORK, verify/build-gate exhaustion) get a committed spec amendment before re-run, max 3 iterations; infrastructure / invalid-input / engine-availability / implement-empty BLOCKED verdicts surface immediately. Unattended queue drain (`docs/specs/queue.md`, strictly serial): assumptions may only take scope-narrowing, reversible, non-user-visible defaults — material ambiguity marks the item `[F] needs-review`; commit every terminal `[x]` / `[F]` transition before advancing. The canonical commit order is `config/skills/devlyn:resolve/references/outer-loop.md`.
|
|
42
46
|
|
|
43
47
|
## Subtractive-First Editing
|
|
44
48
|
|
|
@@ -103,13 +107,8 @@ Exclude vague claims. They produce vague fixes.
|
|
|
103
107
|
- Never revert user changes unless explicitly asked.
|
|
104
108
|
- Use `rg` / `rg --files` for search.
|
|
105
109
|
- Keep changes scoped to the task; stop when the core request is answered or the change is verified.
|
|
106
|
-
- For installer or skill edits inside the devlyn-cli source repo, run `bash scripts/lint-skills.sh`.
|
|
107
110
|
- Treat the project's own `docs/VISION.md`, `docs/ROADMAP.md`, `docs/roadmap/**`, and any local `AGENTS.md` / `CLAUDE.md` overrides as authoritative when present.
|
|
108
111
|
|
|
109
|
-
### Evolution loop — orchestrator-neutral continuation
|
|
110
|
-
|
|
111
|
-
Any orchestrating engine (Claude, Codex, or future) continuing harness / loop-engineering work cold-starts from `autoresearch/HANDOFF.md` (read order inside), then `autoresearch/NORTH-STAR.md` (goal + floor/ceiling contracts + ops-test gates) and `autoresearch/PRINCIPLES.md`. Instruments and gates are scripts + files, never model memory. The pair partner is the strongest available OTHER engine — the protocol is direction-symmetric (iter-0060). On any model/version change, re-certify engine seats with the seat-fitness instrument before re-pinning `.devlyn/engines.json` (NORTH-STAR ceiling contract; iter-0064).
|
|
112
|
-
|
|
113
112
|
## Communication
|
|
114
113
|
|
|
115
114
|
- Lead with objective evidence before opinion.
|
package/CLAUDE.md
CHANGED
|
@@ -4,7 +4,7 @@ devlyn-cli installs `/devlyn:ideate` (optional) and `/devlyn:resolve` (required)
|
|
|
4
4
|
|
|
5
5
|
## North Star
|
|
6
6
|
|
|
7
|
-
This contract serves one goal: any capable engine — Claude, GPT/Codex, or a future adapter-equipped model — takes a user's intent (prompt, spec, or queue entry) end-to-end to shipped, engineer-quality software, hands-free, with consistent quality across engines.
|
|
7
|
+
This contract serves one goal: any capable engine — Claude, GPT/Codex, or a future adapter-equipped model — takes a user's intent (prompt, spec, or queue entry) end-to-end to shipped, engineer-quality software, hands-free, with consistent quality across engines. When rules below conflict or feel ambiguous in context, resolve toward this goal.
|
|
8
8
|
|
|
9
9
|
## Core principles
|
|
10
10
|
|
|
@@ -15,7 +15,7 @@ Seven rules govern every change. Cite them by name when a decision touches one.
|
|
|
15
15
|
3. **No guesswork** — verify with the actual files, logs, diffs, and run output before forming conclusions. State the falsifiable prediction BEFORE the experiment; record raw results AFTER. Retroactive prediction edits are dishonest.
|
|
16
16
|
4. **Worldclass** — code that survives review at a non-trivial codebase. Zero CRITICAL, zero HIGH security/design findings on the shippable path.
|
|
17
17
|
5. **Best practice** — idiomatic for the language and framework. Use standard primitives; do not hand-roll what the library already provides.
|
|
18
|
-
6. **Optimized** — efficient
|
|
18
|
+
6. **Optimized** — use efficient code and avoid unnecessary work without sacrificing correctness or the user's requirements.
|
|
19
19
|
7. **Production ready** — error states are explicit and visible; behavior under failure is what the user expects, not silent corruption.
|
|
20
20
|
|
|
21
21
|
Three discipline rules govern HOW the principles are applied:
|
|
@@ -39,26 +39,30 @@ Each skill's `SKILL.md` is the source of truth for its flags and workflow — do
|
|
|
39
39
|
|
|
40
40
|
| Role | Default | Manual override |
|
|
41
41
|
|---|---|---|
|
|
42
|
-
| Orchestrator — conversation, handoff, loop driving | whichever CLI you open (contract is symmetric: CLAUDE.md ↔ AGENTS.md)
|
|
43
|
-
|
|
|
42
|
+
| Orchestrator — conversation, handoff, loop driving | whichever CLI you open (contract is symmetric: CLAUDE.md ↔ AGENTS.md) | switch CLIs; the file artifacts (spec/queue/state) carry over |
|
|
43
|
+
| Legacy executor — worker/primary unless separately profiled; PLAN is orchestrator-fixed and never inherits `--engine` or an executor pin | canonical skill's orchestrator-supported default | `--engine <name>` per run, or `/devlyn:engines executor <name>` (durable pin) |
|
|
44
44
|
| Pair judge — default for VERIFY; conditional for risk probes | first available OTHER engine (claude↔codex) | `/devlyn:engines pair <name>,...`; `--no-pair` opts out |
|
|
45
45
|
|
|
46
46
|
`/devlyn:engines` with no args shows the current role table, detected engines, and how to pin or clear — the pins live in `.devlyn/engines.json`.
|
|
47
47
|
|
|
48
|
-
`.devlyn/engines.json` is machine-local — not committed, not archived. Pins are promises: a pinned unavailable engine stops with `BLOCKED:<engine>-unavailable`; a name without a `_shared/adapters/<name>.md` adapter stops with `BLOCKED:invalid-engine-config`. New engines (GLM, pi-agent backends) plug in by shipping an adapter file — no skill changes.
|
|
48
|
+
`.devlyn/engines.json` is machine-local — not committed, not archived. Pins are promises: a pinned unavailable engine stops with `BLOCKED:<engine>-unavailable`; a name without a `_shared/adapters/<name>.md` adapter stops with `BLOCKED:invalid-engine-config`. New engines (GLM, pi-agent backends) plug in by shipping an adapter file — no skill changes.
|
|
49
49
|
|
|
50
|
-
**The executor pin binds the orchestrator in plain conversation too, not only inside a skill run.** When you would do implementation work directly and executor is pinned to a non-default engine, route that work through the pin — run it via `/devlyn:resolve` (which reads the pin at PHASE 0), or delegate to that engine
|
|
50
|
+
**The executor pin binds the orchestrator in plain conversation too, not only inside a skill run.** When you would do implementation work directly and executor is pinned to a non-default engine, route that work through the pin — run it via `/devlyn:resolve` (which reads the pin at PHASE 0), or delegate to that engine. No pin / no `.devlyn/engines.json` → use the canonical skill's orchestrator-supported default. The pair-judge pin stays pipeline-scoped. `/devlyn:engines role` configures explicit engine/model/effort for supported worker and judge routes; `/devlyn:resolve --role-config <path>` overrides roles for one run. Canonical skills define precedence, evidence and unsupported-route errors; absent profiles preserve existing defaults.
|
|
51
51
|
|
|
52
|
-
### Conversational
|
|
52
|
+
### Conversational entry + full-route handoff
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
Before writes, inspect requested files and relevant callers/tests. Default to direct execution only for clear, local, reversible, low-risk work with a decisive existing or small task-specific acceptance check. Preserve requested behavior, scope, explicit constraints and the executor pin; make scoped edits, run required checks, review the final diff, and report changes with evidence. One concise route/check explanation suffices; ordinary direct work needs no spec, pipeline state, phase workers or extra approval.
|
|
55
|
+
|
|
56
|
+
Use full `/devlyn:resolve` for material ambiguity, subsystem/design work, security/auth/payment/persistence/concurrency/public-API-contract changes, or inconclusive inspection/checks. Counts of lines, files or words alone do not establish low risk. Explicit resolve (even small), formal spec workflows and queue drains retain all canonical phases, independent verification, pins and failure handling. If new risk appears mid-edit, preserve the delta in the full run's scope/evidence; do not hide it in a new baseline or reconfirm existing authorization.
|
|
57
|
+
|
|
58
|
+
The user does not invoke skills manually; the orchestrating model does. Large tasks agreed in conversation:
|
|
55
59
|
|
|
56
60
|
1. Write the agreed contract to `docs/specs/<id>/spec.md` (+ `spec.expected.json` when mechanical verifications exist). Always a spec file for large work — a written spec is the user's reviewed contract; free-form large now assumes-and-logs instead of halting (zero-scope-signal goals still halt to ideate).
|
|
57
61
|
2. Present a one-screen plan-contract summary — the user's single review checkpoint, BEFORE the pipeline starts.
|
|
58
|
-
3. On go-ahead, invoke `/devlyn:resolve --spec <path>` hands-free to completion.
|
|
59
|
-
4. **Per-task outer loop**: read the terminal verdict. PASS → done. Verdicts backed by spec/verification findings (NEEDS_WORK, verify/build-gate exhaustion) → adjudicate the findings, amend the spec (recorded in the spec file; spec stays read-only inside a run), re-invoke — at most 3 outer iterations, then surface with the findings trail. Infrastructure, invalid-input, engine-availability, and implement-empty BLOCKED verdicts are not spec-amendable: surface immediately. Every iteration re-enters through durable artifacts (spec, findings, run archive), never conversation memory.
|
|
62
|
+
3. On go-ahead, commit the scoped queue-item delta (when applicable) and linked spec bundle, then invoke `/devlyn:resolve --spec <path>` hands-free to completion.
|
|
63
|
+
4. **Per-task outer loop**: read the terminal verdict. PASS → done. Verdicts backed by spec/verification findings (NEEDS_WORK, verify/build-gate exhaustion) → adjudicate the findings, amend the spec (recorded in the spec file; spec stays read-only inside a run), commit that scoped amendment, then re-invoke — at most 3 outer iterations, then surface with the findings trail. Infrastructure, invalid-input, engine-availability, and implement-empty BLOCKED verdicts are not spec-amendable: surface immediately. Every iteration re-enters through committed durable artifacts (spec, findings, run archive), never conversation memory.
|
|
60
64
|
|
|
61
|
-
**Intent queue (unattended drain)**: the user stacks intents in `docs/specs/queue.md` (ordered checklist; `/devlyn:queue` is the front-end — no args shows status, `add <intent>` appends, `drain` starts the serial drain). Drain strictly serially — per item: spec it, run the outer loop, mark `[x]` done or `[F]` blocked with reason, continue; a blocked item never halts the queue. The queue entry is the user's go-ahead, so assume-and-log replaces the interactive checkpoint — but unattended assumptions may only take scope-narrowing, reversible, non-user-visible defaults. Material ambiguity (user-visible behavior, data/state semantics, new files/scripts/flags, implementation surface) → mark `[F] needs-review
|
|
65
|
+
**Intent queue (unattended drain)**: the user stacks intents in `docs/specs/queue.md` (ordered checklist; `/devlyn:queue` is the front-end — no args shows status, `add <intent>` appends, `drain` starts the serial drain). Drain strictly serially — per item: spec it, commit the scoped owner baseline, run the outer loop, mark `[x]` done or `[F]` blocked with reason, commit that terminal transition, then continue; a blocked item never halts the queue. The queue entry is the user's go-ahead, so assume-and-log replaces the interactive checkpoint — but unattended assumptions may only take scope-narrowing, reversible, non-user-visible defaults. Material ambiguity (user-visible behavior, data/state semantics, new files/scripts/flags, implementation surface) → mark `[F] needs-review`, commit it, and move on. End the drain with a per-item verdict + assumptions report. The canonical commit order is `config/skills/devlyn:resolve/references/outer-loop.md`.
|
|
62
66
|
|
|
63
67
|
## Subtractive-first editing — perfection = nothing left to remove
|
|
64
68
|
<!-- runtime-principles:section=subtractive-first:begin -->
|
|
@@ -127,10 +131,6 @@ This rule exists because LLMs (including you) are trained to be helpful, compreh
|
|
|
127
131
|
**Stopping rule.** A task is done when the user's stated goal is closed AND no off-path work was added. If you find yourself hesitating because "I should also do Z" — Z is drift. Note it for follow-up, do not execute.
|
|
128
132
|
<!-- runtime-principles:section=goal-locked:end -->
|
|
129
133
|
|
|
130
|
-
## Evolution loop — orchestrator-neutral continuation
|
|
131
|
-
|
|
132
|
-
Any orchestrating engine (Claude, Codex, or future) continuing harness / loop-engineering work cold-starts from `autoresearch/HANDOFF.md` (read order inside), then `autoresearch/NORTH-STAR.md` (goal + floor/ceiling contracts + ops-test gates) and `autoresearch/PRINCIPLES.md`. Instruments and gates are scripts + files, never model memory. The pair partner is the strongest available OTHER engine — the protocol is direction-symmetric (iter-0060). On any model/version change, re-certify engine seats with the seat-fitness instrument before re-pinning `.devlyn/engines.json` (NORTH-STAR ceiling contract; iter-0064).
|
|
133
|
-
|
|
134
134
|
## Error Handling Philosophy
|
|
135
135
|
|
|
136
136
|
**No silent fallbacks.** Handle errors explicitly and show the user what happened.
|
|
@@ -158,7 +158,7 @@ A finding without one of these forms is excluded. Vague findings produce vague f
|
|
|
158
158
|
|
|
159
159
|
## Codex invocation
|
|
160
160
|
|
|
161
|
-
|
|
161
|
+
For Codex model selection, receipts, isolation and availability, follow the invoked skill's sibling `_shared/codex-config.md` and `_shared/engine-preflight.md` (`config/skills/_shared/` in this repository).
|
|
162
162
|
|
|
163
163
|
## Working Mode
|
|
164
164
|
|
|
@@ -169,7 +169,7 @@ When `/devlyn:resolve` or `/devlyn:ideate` route a phase to Codex (`--engine cod
|
|
|
169
169
|
|
|
170
170
|
## Skill Boundary Policy
|
|
171
171
|
|
|
172
|
-
The runtime pipeline surface is two skills — `/devlyn:resolve` and `/devlyn:ideate` — plus `/devlyn:design-ui` for creative UI exploration and two utilities
|
|
172
|
+
The runtime pipeline surface is two skills — `/devlyn:resolve` and `/devlyn:ideate` — plus `/devlyn:design-ui` for creative UI exploration and two utilities: `/devlyn:engines` (engine-role config) and `/devlyn:queue` (intent-queue status/add/drain). `/devlyn:resolve` runs PLAN → IMPLEMENT → BUILD_GATE → CLEANUP → VERIFY inline; verification, cleanup, and security review (delegated to the native `security-review` Claude Code skill from BUILD_GATE) all live inside the pipeline. There are no standalone `/devlyn:review`, `/devlyn:evaluate`, or `/devlyn:team-resolve` surfaces. `/devlyn:design-ui` spawns a 5-specialist design team (Creative Director, Product Designer, Visual Designer, Interaction Designer, Accessibility Designer). `/devlyn:reap` is an optional user-invoked skill in `optional-skills/`; resolve never delegates to it.
|
|
173
173
|
|
|
174
174
|
Browser validation runs directly from BUILD_GATE using whichever toolchain is available (Chrome MCP, Playwright, or curl-tier fallback) — there is no separate `/devlyn:browser-validate` skill.
|
|
175
175
|
|
package/README.md
CHANGED
|
@@ -27,7 +27,7 @@ If devlyn-cli saved you time, [give it a star](https://github.com/fysoul17/devly
|
|
|
27
27
|
npx devlyn-cli
|
|
28
28
|
```
|
|
29
29
|
|
|
30
|
-
That's it. The installer opens with a single **agent selector** — pick any combination of **Claude Code**, **Codex CLI**, **oh-my-pi (omp)**, **Pi**, or **Grok Build CLI**, and devlyn installs into every one you choose in a single pass. Claude Code and any agent already present on your machine are pre-checked, so the common case is just Enter. Skill-capable agents receive the `devlyn:resolve`, `devlyn:ideate`, and `devlyn:design-ui` skills — plus the `devlyn:engines` and `devlyn:queue` utilities — in the directory each one loads from: Codex → `~/.codex/skills/`, Grok → `~/.grok/skills/`, while **omp and Pi share `~/.agents/skills/`** — the cross-agent standard both read — so the bundle is written there once, not duplicated per agent. In Codex / omp / Pi, invoke them as skills (`$devlyn:resolve`, `$devlyn:ideate`, `$devlyn:design-ui`); in Claude Code and Grok Build CLI they're slash commands (`/devlyn:resolve`).
|
|
30
|
+
That's it. The installer opens with a single **agent selector** — pick any combination of **Claude Code**, **Codex CLI**, **oh-my-pi (omp)**, **Pi**, or **Grok Build CLI**, and devlyn installs into every one you choose in a single pass. Claude Code and any agent already present on your machine are pre-checked, so the common case is just Enter. Skill-capable agents receive the `devlyn:resolve`, `devlyn:ideate`, and `devlyn:design-ui` skills — plus the `devlyn:engines` and `devlyn:queue` utilities — in the directory each one loads from: Codex → `~/.codex/skills/`, Grok → `~/.grok/skills/`, while **omp and Pi share `~/.agents/skills/`** — the cross-agent standard both read — so the bundle is written there once, not duplicated per agent. In Codex / omp / Pi, invoke them as skills (`$devlyn:resolve`, `$devlyn:ideate`, `$devlyn:design-ui`); in Claude Code and Grok Build CLI they're slash commands (`/devlyn:resolve`). Rerunning refreshes skills for selected agents, preserves existing AGENTS.md without legacy managed markers (legacy appended blocks are still cleaned up), and replaces CLAUDE.md when Claude Code is selected. Use the bundled AGENTS.md template path printed by the installer to compare and merge relevant shared runtime instructions into your preserved AGENTS.md, retaining your content and keeping engine-specific instructions distinct. (`npx devlyn-cli -y` installs the Claude core non-interactively; `npx devlyn-cli agents <cli>` adds one agent later.)
|
|
31
31
|
|
|
32
32
|
---
|
|
33
33
|
|
|
@@ -62,7 +62,7 @@ Skip ideate entirely if you have a spec or just want to describe the work — `/
|
|
|
62
62
|
|
|
63
63
|
### Step 2 — Resolve with `/devlyn:resolve`
|
|
64
64
|
|
|
65
|
-
|
|
65
|
+
Before writes, inspect requested files and relevant callers/tests. Clear, local, reversible, low-risk conversational edits with a decisive acceptance check run directly, honoring scope, constraints and executor pins; finish required checks and diff review, then report evidence. Material ambiguity, subsystem/design work, security/auth/payment/persistence/concurrency/public-API-contract changes or no decisive check use full `/devlyn:resolve`. Explicit resolve (including small tasks), formal specs and queue drains keep the full workflow below. Pass a spec, a free-form goal, or a diff to verify.
|
|
66
66
|
|
|
67
67
|
```
|
|
68
68
|
/devlyn:resolve "fix the login bug" # free-form
|
|
@@ -81,7 +81,7 @@ PLAN → IMPLEMENT → BUILD_GATE → CLEANUP → VERIFY (fresh subagent
|
|
|
81
81
|
- **VERIFY** runs in a fresh subagent context with no code-mutation tools — findings only, structurally independent.
|
|
82
82
|
- Git checkpoints at every phase for safe rollback. Fix-loop budget shared across BUILD_GATE and VERIFY (`--max-rounds N`, default 4).
|
|
83
83
|
|
|
84
|
-
Common flags: `--engine claude|codex|
|
|
84
|
+
Common flags: `--engine claude|codex|omp` (default: the orchestrator-supported default), `--role-config <path>` (one-run worker/judge profiles, see below), `--bypass build-gate,cleanup`, `--pair-verify` (force pair-mode JUDGE in VERIFY), `--no-pair` (intentional solo VERIFY), `--risk-probes` / `--no-risk-probes`, `--perf` (per-phase timing).
|
|
85
85
|
`--pair-verify` and `--no-pair` are mutually exclusive; using both stops with `BLOCKED:invalid-flags`.
|
|
86
86
|
|
|
87
87
|
Free-form goals that ask for benchmark evidence, pair-evidence, risk-probe
|
|
@@ -106,20 +106,44 @@ Stack tasks to run back-to-back without supervision. `/devlyn:queue add "<intent
|
|
|
106
106
|
devlyn separates three engine roles:
|
|
107
107
|
|
|
108
108
|
- **Orchestrator** — the CLI you opened (Claude Code, Codex, or omp) that drives the conversation and loop. The contract is symmetric (`CLAUDE.md` ↔ `AGENTS.md`), so the same phase-gated pipeline runs whichever you launch; the file artifacts (spec, queue, state) carry over if you switch.
|
|
109
|
-
- **Executor** — IMPLEMENT / CLEANUP
|
|
109
|
+
- **Executor** (legacy) — the worker for IMPLEMENT / CLEANUP and their repair rounds, and the primary VERIFY judge, unless either is separately profiled (below). Defaults to the orchestrator-supported default. PLAN is orchestrator-fixed and never inherits `--engine` or an executor pin. An absent primary profile follows the legacy executor, not an opt-in worker profile.
|
|
110
110
|
- **Pair judge** — the first available *other* engine, default for VERIFY and conditional for risk probes.
|
|
111
111
|
|
|
112
|
-
`--engine
|
|
112
|
+
`--engine <name>` sets the legacy executor for one run as an engine-only entry (it clears any lower-precedence profile's model/effort). BUILD_GATE and risk-probe derivation keep their existing routes; PLAN always runs on the orchestrator's native worker. VERIFY/JUDGE runs pair mode by default when the OTHER engine is available.
|
|
113
113
|
|
|
114
|
-
Pin roles durably with `/devlyn:engines` (no args shows the role table + detected engines; `executor <name>` / `pair <name>,...` / `clear` manage the pins, stored machine-local in `.devlyn/engines.json`). Pins fail closed: an unavailable pinned engine stops with `BLOCKED:<engine>-unavailable`, and a name with no `_shared/adapters/<name>.md` adapter stops with `BLOCKED:invalid-engine-config`. New engines plug in by shipping an adapter file — no skill changes.
|
|
114
|
+
Pin roles durably with `/devlyn:engines` (no args shows the role table + detected engines; `executor <name>` / `pair <name>,...` / `role <worker|primary_judge|pair_judge> <JSON object>` / `role <name> clear` manage the pins, stored machine-local in `.devlyn/engines.json`; `clear` removes all legacy and role pins, preserves unrelated keys, and deletes an empty file). Pins fail closed: an unavailable pinned engine stops with `BLOCKED:<engine>-unavailable`, and a name with no `_shared/adapters/<name>.md` adapter stops with `BLOCKED:invalid-engine-config`. New engines plug in by shipping an adapter file — no skill changes.
|
|
115
115
|
|
|
116
|
-
|
|
116
|
+
#### Explicit worker and judge profiles
|
|
117
|
+
|
|
118
|
+
`.devlyn/engines.json` may add a `roles` object with `worker`, `primary_judge`, and `pair_judge` entries. Each entry requires `engine` and accepts an optional exact `model` ID and `effort`; unknown fields, aliases (`default`, `auto`, `opus`, `sonnet`, `fable`, `codex` as a model), null/empty values, and duplicate keys stop with `BLOCKED:invalid-engine-config`. Only the project's own `.devlyn/engines.json` is read — no parent or global lookup. Example (an example, not a default or ranking):
|
|
119
|
+
|
|
120
|
+
```json
|
|
121
|
+
{
|
|
122
|
+
"executor": "codex",
|
|
123
|
+
"pair_judge_priority": ["claude"],
|
|
124
|
+
"roles": {
|
|
125
|
+
"worker": {"engine": "codex", "model": "gpt-6-astra", "effort": "xhigh"},
|
|
126
|
+
"primary_judge": {"engine": "codex", "model": "gpt-6-astra", "effort": "high"},
|
|
127
|
+
"pair_judge": {"engine": "claude", "model": "claude-fable-5-1", "effort": "medium"}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`--role-config <path>` supplies the same `{"roles": {...}}` object for one run only, with no other top-level keys; the flag may appear once.
|
|
133
|
+
|
|
134
|
+
- **Precedence** — an entry replaces the lower one as a whole; model/effort never merge across engines. Worker and primary judge: run `--role-config` entry > `--engine` (engine-only) > project role entry > legacy executor/default. Pair judge: run entry > project entry > `pair_judge_priority` / the OTHER-engine complement. A missing `model`/`effort` keeps that route's current default and is shown as inherited/unresolved.
|
|
135
|
+
- **Boundaries** — `worker` covers IMPLEMENT / CLEANUP and their repair rounds; `primary_judge` and `pair_judge` cover VERIFY only. PLAN stays orchestrator-fixed, BUILD_GATE keeps its existing route, and risk-probe derivation stays outside these controls. OTHER must differ from the primary by engine: an explicit same-engine pair is `BLOCKED:invalid-engine-config`. `--no-pair` is unchanged — explicit solo VERIFY; an unused pair entry is neither dispatched nor availability-checked.
|
|
136
|
+
- **Initial explicit routes** — Codex worker/primary/pair accept exact `model` + `effort` (dispatched as `-m` / `-c model_reasoning_effort=`, validated against the installed Codex CLI's native model metadata for its version). Claude primary/pair judges pass an exact `model` as `--model` and validate native result identity; model-only selection needs no source update for a new ID. Explicit `effort` separately requires the adapter's version/model support declaration. The Claude worker stays the native `Agent` route and is engine-only: explicit `model`/`effort` on a Claude worker is `BLOCKED:unsupported-role-option` — a supported-route boundary, not a claim that Claude cannot implement. Other adapters (omp, grok) are engine-only for their eligible roles.
|
|
137
|
+
- **Errors** — `BLOCKED:unsupported-role-option` names the role, engine and field with guidance; there is no clamp, fallback, substituted model, or weaker retry. An explicitly selected engine (flag, pin, or profile) that is unavailable stops with `BLOCKED:<engine>-unavailable`. A native warning that a requested option was ignored is a failed promise even at exit 0.
|
|
138
|
+
- **Requested vs observed** — `/devlyn:engines` status prints each role's requested engine/model/effort, source and dispatch channel before dispatch; it describes a request and does not prove a native invocation happened. Effective model is reported only with native evidence; a Codex worker receipt binds requested dispatch and leaves effective model unknown. A reported worker model reroute blocks completion even at exit 0. SURFACE_CLOSE inherits native Claude model settings and records the model from its required JSON result. Effective effort stays unknown unless native evidence establishes it — invocation arguments prove dispatch, not provider-internal reasoning.
|
|
139
|
+
|
|
140
|
+
`--engine codex` routes IMPLEMENT / CLEANUP to Codex as a supported engine-only route (exact model/effort via the profiles above). Historical record: iter-0020 closed Codex BUILD/IMPLEMENT below the quality floor on the 9-fixture suite (L2 vs L1 = −3.6, 3/8 gated fixtures cleared the +5 margin floor — release-readiness FAIL). Separately, PLAN-pair remains research-only: iter-0033g + iter-0034 closed it with explicit unblock conditions (container/sandbox infra OR production telemetry capturing positive evidence of subagent introspection). Install the Codex CLI (https://platform.openai.com/docs/codex) and pass the flag explicitly to opt in:
|
|
117
141
|
|
|
118
142
|
```
|
|
119
|
-
/devlyn:resolve "fix the auth bug" --engine codex
|
|
143
|
+
/devlyn:resolve "fix the auth bug" --engine codex
|
|
120
144
|
```
|
|
121
145
|
|
|
122
|
-
If
|
|
146
|
+
If an engine is absent when explicitly selected by flag, pin, or role profile, or OTHER engine is absent under `--pair-verify`, the harness stops with `BLOCKED:<engine>-unavailable` and prints setup guidance. Automatic VERIFY absence is a reported solo route. Use `--no-pair` only when intentionally accepting solo VERIFY; use `--no-risk-probes` only when intentionally disabling automatic high-risk probes.
|
|
123
147
|
|
|
124
148
|
### Benchmark score runs
|
|
125
149
|
|
package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"run_id": "20260507-f10-f11-tier1-full-pipeline",
|
|
3
|
+
"rule": "at least 2 candidate fixtures; each must satisfy bare <= 60 and solo_claude <= 80, with both arms clean",
|
|
4
|
+
"verdict": "FAIL",
|
|
5
|
+
"fixtures_total": 2,
|
|
6
|
+
"fixtures_passed": 0,
|
|
7
|
+
"min_fixtures": 2,
|
|
8
|
+
"fixture_count_ok": true,
|
|
9
|
+
"rows": [
|
|
10
|
+
{
|
|
11
|
+
"fixture": "F10-persist-write-collision",
|
|
12
|
+
"status": "FAIL",
|
|
13
|
+
"bare_score": 75,
|
|
14
|
+
"solo_score": 94,
|
|
15
|
+
"reason": "bare score 75 > 60; solo_claude score 94 > 80"
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"fixture": "F11-batch-import-all-or-nothing",
|
|
19
|
+
"status": "FAIL",
|
|
20
|
+
"bare_score": 98,
|
|
21
|
+
"solo_score": 97,
|
|
22
|
+
"reason": "bare score 98 > 60; solo_claude score 97 > 80"
|
|
23
|
+
}
|
|
24
|
+
]
|
|
25
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Headroom Gate — 20260507-f10-f11-tier1-full-pipeline
|
|
2
|
+
|
|
3
|
+
Verdict: **FAIL**
|
|
4
|
+
|
|
5
|
+
Rule: at least 2 fixtures; bare <= 60, solo_claude <= 80, both arms clean.
|
|
6
|
+
|
|
7
|
+
| Fixture | Bare | Solo | Status | Reason |
|
|
8
|
+
|---|---:|---:|---|---|
|
|
9
|
+
| F10-persist-write-collision | 75 | 94 | FAIL | bare score 75 > 60; solo_claude score 94 > 80 |
|
|
10
|
+
| F11-batch-import-all-or-nothing | 98 | 97 | FAIL | bare score 98 > 60; solo_claude score 97 > 80 |
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"run_id": "20260508-f22-exact-error-headroom",
|
|
3
|
+
"rule": "at least 2 candidate fixtures; each must satisfy bare <= 60 and solo_claude <= 80, with both arms clean",
|
|
4
|
+
"verdict": "FAIL",
|
|
5
|
+
"fixtures_total": 1,
|
|
6
|
+
"fixtures_passed": 0,
|
|
7
|
+
"min_fixtures": 2,
|
|
8
|
+
"fixture_count_ok": false,
|
|
9
|
+
"rows": [
|
|
10
|
+
{
|
|
11
|
+
"fixture": "F22-cli-ledger-close",
|
|
12
|
+
"status": "FAIL",
|
|
13
|
+
"bare_score": 94,
|
|
14
|
+
"solo_score": 98,
|
|
15
|
+
"reason": "bare score 94 > 60; solo_claude score 98 > 80"
|
|
16
|
+
}
|
|
17
|
+
]
|
|
18
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Headroom Gate — 20260508-f22-exact-error-headroom
|
|
2
|
+
|
|
3
|
+
Verdict: **FAIL**
|
|
4
|
+
|
|
5
|
+
Rule: at least 2 fixtures; bare <= 60, solo_claude <= 80, both arms clean.
|
|
6
|
+
|
|
7
|
+
| Fixture | Bare | Solo | Status | Reason |
|
|
8
|
+
|---|---:|---:|---|---|
|
|
9
|
+
| F22-cli-ledger-close | 94 | 98 | FAIL | bare score 94 > 60; solo_claude score 98 > 80 |
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"run_id": "20260508-f26-headroom",
|
|
3
|
+
"rule": "at least 2 candidate fixtures; each must satisfy bare <= 60 and solo_claude <= 80, with both arms clean",
|
|
4
|
+
"verdict": "FAIL",
|
|
5
|
+
"fixtures_total": 1,
|
|
6
|
+
"fixtures_passed": 0,
|
|
7
|
+
"min_fixtures": 2,
|
|
8
|
+
"fixture_count_ok": false,
|
|
9
|
+
"rows": [
|
|
10
|
+
{
|
|
11
|
+
"fixture": "F26-cli-payout-ledger-rules",
|
|
12
|
+
"status": "FAIL",
|
|
13
|
+
"bare_score": 25,
|
|
14
|
+
"solo_score": 98,
|
|
15
|
+
"reason": "solo_claude score 98 > 80"
|
|
16
|
+
}
|
|
17
|
+
]
|
|
18
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Headroom Gate — 20260508-f26-headroom
|
|
2
|
+
|
|
3
|
+
Verdict: **FAIL**
|
|
4
|
+
|
|
5
|
+
Rule: at least 2 fixtures; bare <= 60, solo_claude <= 80, both arms clean.
|
|
6
|
+
|
|
7
|
+
| Fixture | Bare | Solo | Status | Reason |
|
|
8
|
+
|---|---:|---:|---|---|
|
|
9
|
+
| F26-cli-payout-ledger-rules | 25 | 98 | FAIL | solo_claude score 98 > 80 |
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"fixture": "F16-cli-quote-tax-rules",
|
|
3
|
+
"arm": "l2_risk_probes",
|
|
4
|
+
"run_id": "20260507-f16-riskprobes-v15-pricing-diagnostic",
|
|
5
|
+
"disqualifier": false,
|
|
6
|
+
"verify_score": 1.0,
|
|
7
|
+
"commands_passed": 4,
|
|
8
|
+
"commands_total": 4,
|
|
9
|
+
"diff_bytes": 8961,
|
|
10
|
+
"files_changed": 2,
|
|
11
|
+
"elapsed_seconds": 1164,
|
|
12
|
+
"timed_out": false,
|
|
13
|
+
"environment_contamination": false,
|
|
14
|
+
"environment_contamination_reason": null,
|
|
15
|
+
"invoke_exit": 0,
|
|
16
|
+
"invoke_failure": false,
|
|
17
|
+
"invoke_failure_reason": null,
|
|
18
|
+
"terminal_verdict": "PASS",
|
|
19
|
+
"verify_verdict": "PASS",
|
|
20
|
+
"pair_trigger": {
|
|
21
|
+
"eligible": true,
|
|
22
|
+
"reasons": [
|
|
23
|
+
"complexity.high"
|
|
24
|
+
],
|
|
25
|
+
"skipped_reason": null
|
|
26
|
+
},
|
|
27
|
+
"pair_mode": true
|
|
28
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"fixture": "F23-cli-fulfillment-wave",
|
|
3
|
+
"arm": "l2_risk_probes",
|
|
4
|
+
"run_id": "20260508-f23-riskprobes-v17-rollback-probe",
|
|
5
|
+
"disqualifier": false,
|
|
6
|
+
"verify_score": 1.0,
|
|
7
|
+
"commands_passed": 3,
|
|
8
|
+
"commands_total": 3,
|
|
9
|
+
"diff_bytes": 13562,
|
|
10
|
+
"files_changed": 2,
|
|
11
|
+
"elapsed_seconds": 1805,
|
|
12
|
+
"timed_out": false,
|
|
13
|
+
"environment_contamination": false,
|
|
14
|
+
"environment_contamination_reason": null,
|
|
15
|
+
"invoke_exit": 0,
|
|
16
|
+
"invoke_failure": false,
|
|
17
|
+
"invoke_failure_reason": null,
|
|
18
|
+
"terminal_verdict": "PASS_WITH_ISSUES",
|
|
19
|
+
"verify_verdict": "PASS_WITH_ISSUES",
|
|
20
|
+
"pair_trigger": {
|
|
21
|
+
"eligible": true,
|
|
22
|
+
"reasons": [
|
|
23
|
+
"complexity.high"
|
|
24
|
+
],
|
|
25
|
+
"skipped_reason": null
|
|
26
|
+
},
|
|
27
|
+
"pair_mode": true
|
|
28
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"fixture": "F25-cli-cart-promotion-rules",
|
|
3
|
+
"arm": "l2_risk_probes",
|
|
4
|
+
"run_id": "20260509-f25-cartprobe-v2",
|
|
5
|
+
"disqualifier": false,
|
|
6
|
+
"oracle_disqualifier": false,
|
|
7
|
+
"oracle_findings_count": 0,
|
|
8
|
+
"verify_score": 1.0,
|
|
9
|
+
"commands_passed": 4,
|
|
10
|
+
"commands_total": 4,
|
|
11
|
+
"diff_bytes": 12108,
|
|
12
|
+
"files_changed": 2,
|
|
13
|
+
"elapsed_seconds": 1177,
|
|
14
|
+
"timed_out": false,
|
|
15
|
+
"environment_contamination": false,
|
|
16
|
+
"environment_contamination_reason": null,
|
|
17
|
+
"invoke_exit": 0,
|
|
18
|
+
"invoke_failure": false,
|
|
19
|
+
"invoke_failure_reason": null,
|
|
20
|
+
"terminal_verdict": "PASS",
|
|
21
|
+
"verify_verdict": "PASS",
|
|
22
|
+
"pair_trigger": {
|
|
23
|
+
"eligible": true,
|
|
24
|
+
"reasons": [
|
|
25
|
+
"complexity.high"
|
|
26
|
+
],
|
|
27
|
+
"skipped_reason": null
|
|
28
|
+
},
|
|
29
|
+
"pair_mode": true
|
|
30
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"run_id": "20260511-f12-webhook-headroom",
|
|
3
|
+
"rule": "at least 1 candidate fixtures; each must satisfy bare <= 60 with headroom >= 5, solo_claude <= 80 with headroom >= 5, with both baseline arms evidence-complete",
|
|
4
|
+
"verdict": "FAIL",
|
|
5
|
+
"fixtures_total": 1,
|
|
6
|
+
"fixtures_passed": 0,
|
|
7
|
+
"min_fixtures": 1,
|
|
8
|
+
"bare_max": 60,
|
|
9
|
+
"solo_max": 80,
|
|
10
|
+
"min_bare_headroom_required": 5,
|
|
11
|
+
"min_solo_headroom_required": 5,
|
|
12
|
+
"fixture_count_ok": true,
|
|
13
|
+
"avg_bare_headroom": -25.0,
|
|
14
|
+
"min_bare_headroom": -25,
|
|
15
|
+
"avg_solo_headroom": -19.0,
|
|
16
|
+
"min_solo_headroom": -19,
|
|
17
|
+
"rows": [
|
|
18
|
+
{
|
|
19
|
+
"fixture": "F12-webhook-raw-body-signature",
|
|
20
|
+
"status": "FAIL",
|
|
21
|
+
"bare_score": 85,
|
|
22
|
+
"solo_score": 99,
|
|
23
|
+
"bare_headroom": -25,
|
|
24
|
+
"solo_headroom": -19,
|
|
25
|
+
"reason": "bare score 85 > 60; solo_claude score 99 > 80"
|
|
26
|
+
}
|
|
27
|
+
]
|
|
28
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Headroom Gate — 20260511-f12-webhook-headroom
|
|
2
|
+
|
|
3
|
+
Verdict: **FAIL**
|
|
4
|
+
|
|
5
|
+
Fixtures passed: 0/1 (minimum required: 1)
|
|
6
|
+
|
|
7
|
+
Rule: at least 1 fixtures; bare <= 60 with headroom >= 5, solo_claude <= 80 with headroom >= 5, both baseline arms evidence-complete.
|
|
8
|
+
Average bare headroom: -25.0
|
|
9
|
+
Minimum bare headroom: -25
|
|
10
|
+
Average solo headroom: -19.0
|
|
11
|
+
Minimum solo headroom: -19
|
|
12
|
+
|
|
13
|
+
| Fixture | Bare | Bare headroom | Solo | Solo headroom | Status | Reason |
|
|
14
|
+
|---|---:|---:|---:|---:|---|---|
|
|
15
|
+
| F12-webhook-raw-body-signature | 85 | -25 | 99 | -19 | FAIL | bare score 85 > 60; solo_claude score 99 > 80 |
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"run_id": "20260511-f15-concurrency-headroom",
|
|
3
|
+
"rule": "at least 1 candidate fixtures; each must satisfy bare <= 60 with headroom >= 5, solo_claude <= 80 with headroom >= 5, with both baseline arms evidence-complete",
|
|
4
|
+
"verdict": "FAIL",
|
|
5
|
+
"fixtures_total": 1,
|
|
6
|
+
"fixtures_passed": 0,
|
|
7
|
+
"min_fixtures": 1,
|
|
8
|
+
"bare_max": 60,
|
|
9
|
+
"solo_max": 80,
|
|
10
|
+
"min_bare_headroom_required": 5,
|
|
11
|
+
"min_solo_headroom_required": 5,
|
|
12
|
+
"fixture_count_ok": true,
|
|
13
|
+
"avg_bare_headroom": -39.0,
|
|
14
|
+
"min_bare_headroom": -39,
|
|
15
|
+
"avg_solo_headroom": -14.0,
|
|
16
|
+
"min_solo_headroom": -14,
|
|
17
|
+
"rows": [
|
|
18
|
+
{
|
|
19
|
+
"fixture": "F15-frozen-diff-race-review",
|
|
20
|
+
"status": "FAIL",
|
|
21
|
+
"bare_score": 99,
|
|
22
|
+
"solo_score": 94,
|
|
23
|
+
"bare_headroom": -39,
|
|
24
|
+
"solo_headroom": -14,
|
|
25
|
+
"reason": "bare score 99 > 60; solo_claude score 94 > 80"
|
|
26
|
+
}
|
|
27
|
+
]
|
|
28
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Headroom Gate — 20260511-f15-concurrency-headroom
|
|
2
|
+
|
|
3
|
+
Verdict: **FAIL**
|
|
4
|
+
|
|
5
|
+
Fixtures passed: 0/1 (minimum required: 1)
|
|
6
|
+
|
|
7
|
+
Rule: at least 1 fixtures; bare <= 60 with headroom >= 5, solo_claude <= 80 with headroom >= 5, both baseline arms evidence-complete.
|
|
8
|
+
Average bare headroom: -39.0
|
|
9
|
+
Minimum bare headroom: -39
|
|
10
|
+
Average solo headroom: -14.0
|
|
11
|
+
Minimum solo headroom: -14
|
|
12
|
+
|
|
13
|
+
| Fixture | Bare | Bare headroom | Solo | Solo headroom | Status | Reason |
|
|
14
|
+
|---|---:|---:|---:|---:|---|---|
|
|
15
|
+
| F15-frozen-diff-race-review | 99 | -39 | 94 | -14 | FAIL | bare score 99 > 60; solo_claude score 94 > 80 |
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"fixture": "F21-cli-scheduler-priority",
|
|
3
|
+
"arm": "l2_risk_probes",
|
|
4
|
+
"run_id": "20260511-f21-current-riskprobes-v1",
|
|
5
|
+
"disqualifier": false,
|
|
6
|
+
"oracle_disqualifier": false,
|
|
7
|
+
"oracle_findings_count": 0,
|
|
8
|
+
"verify_score": 1.0,
|
|
9
|
+
"commands_passed": 3,
|
|
10
|
+
"commands_total": 3,
|
|
11
|
+
"diff_bytes": 14846,
|
|
12
|
+
"files_changed": 2,
|
|
13
|
+
"elapsed_seconds": 1112,
|
|
14
|
+
"timed_out": false,
|
|
15
|
+
"environment_contamination": false,
|
|
16
|
+
"environment_contamination_reason": null,
|
|
17
|
+
"invoke_exit": 0,
|
|
18
|
+
"invoke_failure": false,
|
|
19
|
+
"invoke_failure_reason": null,
|
|
20
|
+
"terminal_verdict": "PASS",
|
|
21
|
+
"verify_verdict": "PASS",
|
|
22
|
+
"pair_trigger": {
|
|
23
|
+
"eligible": true,
|
|
24
|
+
"reasons": [
|
|
25
|
+
"complexity.high",
|
|
26
|
+
"risk_profile.high_risk",
|
|
27
|
+
"risk_probes_enabled"
|
|
28
|
+
],
|
|
29
|
+
"skipped_reason": null
|
|
30
|
+
},
|
|
31
|
+
"pair_mode": true
|
|
32
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"run_id": "20260511-f21-current-riskprobes-v1",
|
|
3
|
+
"rule": "headroom candidates only; bare headroom >= 5; solo_claude headroom >= 5; l2_risk_probes must be evidence-clean, pair_mode true, pair_trigger eligible with a canonical reason, and beat solo_claude by the configured margin",
|
|
4
|
+
"verdict": "PASS",
|
|
5
|
+
"fixtures_total": 1,
|
|
6
|
+
"fixtures_passed": 1,
|
|
7
|
+
"min_fixtures": 1,
|
|
8
|
+
"fixture_count_ok": true,
|
|
9
|
+
"bare_max": 60,
|
|
10
|
+
"solo_max": 80,
|
|
11
|
+
"min_bare_headroom_required": 5,
|
|
12
|
+
"min_solo_headroom_required": 5,
|
|
13
|
+
"min_pair_margin": 5,
|
|
14
|
+
"pair_arm": "l2_risk_probes",
|
|
15
|
+
"max_pair_solo_wall_ratio": 3.0,
|
|
16
|
+
"require_hypothesis_trigger": false,
|
|
17
|
+
"max_observed_pair_solo_wall_ratio": 1.4728476821192054,
|
|
18
|
+
"avg_pair_margin": 33.0,
|
|
19
|
+
"avg_pair_solo_wall_ratio": 1.4728476821192054,
|
|
20
|
+
"rows": [
|
|
21
|
+
{
|
|
22
|
+
"fixture": "F21-cli-scheduler-priority",
|
|
23
|
+
"status": "PASS",
|
|
24
|
+
"bare_score": 33,
|
|
25
|
+
"bare_headroom": 27,
|
|
26
|
+
"solo_score": 66,
|
|
27
|
+
"solo_headroom": 14,
|
|
28
|
+
"pair_score": 99,
|
|
29
|
+
"pair_margin": 33,
|
|
30
|
+
"pair_mode": true,
|
|
31
|
+
"pair_trigger_eligible": true,
|
|
32
|
+
"pair_trigger_reasons": [
|
|
33
|
+
"complexity.high",
|
|
34
|
+
"risk.high",
|
|
35
|
+
"risk_probes.enabled",
|
|
36
|
+
"spec.solo_headroom_hypothesis"
|
|
37
|
+
],
|
|
38
|
+
"pair_trigger_has_canonical_reason": true,
|
|
39
|
+
"pair_trigger_has_hypothesis_reason": true,
|
|
40
|
+
"pair_solo_wall_ratio": 1.4728476821192054,
|
|
41
|
+
"reason": ""
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
}
|