devlyn-cli 2.10.1 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -9
- package/CLAUDE.md +17 -17
- package/README.md +33 -9
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +25 -0
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +10 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +30 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +32 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +44 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +16 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +15 -0
- package/bin/devlyn.js +9 -2
- package/config/skills/_shared/.ruff.toml +2 -0
- package/config/skills/_shared/adapters/claude.md +9 -1
- package/config/skills/_shared/adapters/codex.md +5 -0
- package/config/skills/_shared/adapters/grok.md +4 -2
- package/config/skills/_shared/archive_run.py +738 -56
- package/config/skills/_shared/codex-config.md +19 -4
- package/config/skills/_shared/codex-monitored.sh +81 -12
- package/config/skills/_shared/collect-codex-findings.py +311 -128
- package/config/skills/_shared/engine-preflight.md +8 -7
- package/config/skills/_shared/expected.schema.json +61 -0
- package/config/skills/_shared/finish-gate.py +12 -11
- package/config/skills/_shared/grok-anchor-guard.py +18 -1
- package/config/skills/_shared/invocation-receipt.py +1041 -0
- package/config/skills/_shared/judge-output-parser.py +200 -0
- package/config/skills/_shared/judge-role-evidence.py +273 -0
- package/config/skills/_shared/process-evidence.py +909 -0
- package/config/skills/_shared/resolve-bootstrap.py +776 -84
- package/config/skills/_shared/resolve-stop-hook.py +33 -1
- package/config/skills/_shared/role-config.py +552 -0
- package/config/skills/_shared/run-bounded.py +1 -1
- package/config/skills/_shared/spec-verify-check.py +749 -181
- package/config/skills/_shared/state-phase-write.py +1838 -58
- package/config/skills/_shared/terminal-claim-check.py +15 -0
- package/config/skills/_shared/verify-merge-findings.py +958 -51
- package/config/skills/devlyn:engines/SKILL.md +8 -16
- package/config/skills/devlyn:ideate/SKILL.md +2 -2
- package/config/skills/devlyn:ideate/references/spec-template.md +6 -5
- package/config/skills/devlyn:queue/SKILL.md +7 -4
- package/config/skills/devlyn:resolve/SKILL.md +93 -35
- package/config/skills/devlyn:resolve/references/free-form-mode.md +1 -0
- package/config/skills/devlyn:resolve/references/outer-loop.md +15 -0
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +31 -2
- package/config/skills/devlyn:resolve/references/phases/implement.md +3 -2
- package/config/skills/devlyn:resolve/references/phases/verify.md +120 -84
- package/config/skills/devlyn:resolve/references/state-schema.md +22 -8
- package/package.json +5 -1
- package/scripts/lint-skills.sh +314 -285
|
@@ -6,6 +6,35 @@ Per-engine adapter header is prepended at runtime. BUILD_GATE is mechanical / de
|
|
|
6
6
|
Run language-specific gates and the spec literal-match verification. Emit findings; the orchestrator's fix loop consumes them.
|
|
7
7
|
</role>
|
|
8
8
|
|
|
9
|
+
<capability_contract>
|
|
10
|
+
A Codex BUILD_GATE runs on the parent-provided CI-equivalent route. Required
|
|
11
|
+
filesystem, subprocess, loopback, PTY, and network capabilities are part of the
|
|
12
|
+
gate input, not permissions this worker may widen.
|
|
13
|
+
|
|
14
|
+
If the parent route or an authoritative tool response explicitly denies a
|
|
15
|
+
required operation before the command can produce product output:
|
|
16
|
+
|
|
17
|
+
1. Preserve the exact original command, denied operation, and raw denial through
|
|
18
|
+
the shared `process-evidence.py` manifest for this BUILD_GATE run/round. Use
|
|
19
|
+
its `capability_denied` classification; a matching stderr phrase is never
|
|
20
|
+
sufficient classification.
|
|
21
|
+
2. Stop with `BLOCKED:build-env-underprovisioned`, citing
|
|
22
|
+
`operation=<filesystem|subprocess|loopback|pty|network>`, the command, and the
|
|
23
|
+
manifest path plus evidence id.
|
|
24
|
+
3. Do not emit a product finding, run a substitute command, narrow the command,
|
|
25
|
+
retry it on another route, or treat the denial as test/lint output.
|
|
26
|
+
|
|
27
|
+
The state writer derives the phase verdict floor from this sealed manifest. A
|
|
28
|
+
capability denial can complete only as `BLOCKED`; a failed product expectation
|
|
29
|
+
cannot complete as `PASS` or `PASS_WITH_ISSUES` even if a mutable results or
|
|
30
|
+
findings file claims otherwise.
|
|
31
|
+
|
|
32
|
+
An executed command that exits nonzero remains a genuine product result even if
|
|
33
|
+
its stdout/stderr contains text such as `permission denied` or `operation not
|
|
34
|
+
permitted`. Preserve the existing finding rules below. Only explicit route/tool
|
|
35
|
+
capability evidence produces the blocker.
|
|
36
|
+
</capability_contract>
|
|
37
|
+
|
|
9
38
|
<detection>
|
|
10
39
|
Detect the project shape from files in `state.base_ref.sha`:
|
|
11
40
|
|
|
@@ -22,7 +51,7 @@ Run in this order; each emits findings into `.devlyn/build_gate.findings.jsonl`:
|
|
|
22
51
|
1. **Type check** (TypeScript / mypy / etc.). Each error → one finding, severity `HIGH`, rule `correctness.type-check`.
|
|
23
52
|
2. **Lint**. Run a recognized configured language linter (for example, ESLint with a config, `ruff`, or `clippy`) and/or the exact `package.json` `scripts.lint` command. A declared specialized `lint:*` script (for example, `lint:json`) is not the general language lint gate on the strength of its declaration alone. If the spec's verification commands explicitly name such a specialized command, step 4 executes it and preserves its real failure severity. If no linter is applicable, log an explicit SKIP in `.devlyn/build_gate.log.md` and emit zero lint findings. Each error → finding, severity `MEDIUM`, rule `quality.lint`. Warnings stay LOW unless the spec elevates them.
|
|
24
53
|
3. **Test suite** (npm test / pytest / go test / cargo test). Each failing test → finding, severity `HIGH`, rule `correctness.test-failure`. Include the failing test's file:line and the assertion.
|
|
25
|
-
4. **Spec literal verification + risk probes**: `python3 "$DEVLYN_SHARED_DIR/spec-verify-check.py" --include-risk-probes`. The script self-stages from sibling `spec.expected.json` next to `state.source.spec_path`, or the legacy inline carrier when the sibling is absent; benchmark-prestaged `.devlyn/spec-verify.json` still wins. It appends `.devlyn/risk-probes.jsonl` when present, and requires that file when `state.risk_profile.risk_probes_enabled == true`. Malformed `state.risk_profile` is also CRITICAL because it can hide enabled risk probes. When probes are enabled or `.devlyn/risk-probes.jsonl` exists, missing or mismatched top-level `state.risk_probes_digest` emits `correctness.risk-probe-integrity` CRITICAL; the digest binds `.devlyn/risk-probes.jsonl` plus referenced `.devlyn/probes/` script bytes. Command or risk-probe mismatch → CRITICAL finding. Missing required risk probes, missing/malformed generated carrier, or malformed sibling expected file → `correctness.spec-verify-malformed` CRITICAL. The same invocation also enforces PLAN's declared `authorized_surface` (the json block under `.devlyn/plan.md`'s `<!-- devlyn:authorized-surface -->` section, located by that sentinel regardless of the heading's language) against this run's diff and untracked delta: any changed path or created-during-run untracked path outside the declared surface → `scope.out-of-scope-file` CRITICAL (`.devlyn/` exempt) — the only sanctioned fix is removing the file, never widening the declared surface, since that would let the same worker that leaked scope self-authorize the leak; missing sentinel, malformed `plan.md`/surface block, or missing `.devlyn/untracked.baseline` → `scope.authorized-surface-malformed` CRITICAL. This scope check runs only here (BUILD_GATE), not when VERIFY MECHANICAL reuses this script post-CLEANUP, because CLEANUP's own allowlist legitimately touches paths PLAN never declared.
|
|
54
|
+
4. **Spec literal verification + risk probes**: `python3 "$DEVLYN_SHARED_DIR/spec-verify-check.py" --include-risk-probes`. The script routes every literal command and risk probe through `process-evidence.py`, preserving exact command, outcome, raw stdout/stderr, and digests in the BUILD_GATE run/round manifest. It self-stages from sibling `spec.expected.json` next to `state.source.spec_path`, or the legacy inline carrier when the sibling is absent; benchmark-prestaged `.devlyn/spec-verify.json` still wins. It appends `.devlyn/risk-probes.jsonl` when present, and requires that file when `state.risk_profile.risk_probes_enabled == true`. Malformed `state.risk_profile` is also CRITICAL because it can hide enabled risk probes. When probes are enabled or `.devlyn/risk-probes.jsonl` exists, missing or mismatched top-level `state.risk_probes_digest` emits `correctness.risk-probe-integrity` CRITICAL; the digest binds `.devlyn/risk-probes.jsonl` plus referenced `.devlyn/probes/` script bytes. Command or risk-probe mismatch → CRITICAL finding. Missing required risk probes, missing/malformed generated carrier, or malformed sibling expected file → `correctness.spec-verify-malformed` CRITICAL. The same invocation also enforces PLAN's declared `authorized_surface` (the json block under `.devlyn/plan.md`'s `<!-- devlyn:authorized-surface -->` section, located by that sentinel regardless of the heading's language) against this run's diff and untracked delta: any changed path or created-during-run untracked path outside the declared surface → `scope.out-of-scope-file` CRITICAL (`.devlyn/` exempt) — the only sanctioned fix is removing the file, never widening the declared surface, since that would let the same worker that leaked scope self-authorize the leak; missing sentinel, malformed `plan.md`/surface block, or missing `.devlyn/untracked.baseline` → `scope.authorized-surface-malformed` CRITICAL. This scope check runs only here (BUILD_GATE), not when VERIFY MECHANICAL reuses this script post-CLEANUP, because CLEANUP's own allowlist legitimately touches paths PLAN never declared.
|
|
26
55
|
5. **Browser** (only when diff touches `*.tsx`, `*.jsx`, `*.vue`, `*.svelte`, `page.*`, `layout.*`, `route.*`, `*.css`, `*.html`): start the dev server and run the repo's existing browser checks, or a minimal curl/HTML check when no browser test harness exists. Each failed check → finding, severity `HIGH`, rule `correctness.browser-flow-failed`.
|
|
27
56
|
|
|
28
57
|
Append all findings; do not stop on the first failure.
|
|
@@ -31,7 +60,7 @@ Append all findings; do not stop on the first failure.
|
|
|
31
60
|
<output>
|
|
32
61
|
- `.devlyn/build_gate.findings.jsonl` — JSONL stream, one finding per line. Schema: `{id, rule_id, severity, file, line, message, fix_hint, criterion_ref}`.
|
|
33
62
|
- `.devlyn/build_gate.log.md` — human-readable summary of which gates ran and their raw output.
|
|
34
|
-
- Report
|
|
63
|
+
- Report `PASS` if zero CRITICAL/HIGH findings, `FAIL` for genuine product findings, or `BLOCKED:build-env-underprovisioned` only for the evidence-backed capability path above. A capability blocker leaves no product finding. Do not edit `pipeline.state.json` yourself — the orchestrator records it via `state-phase-write.py` from these artifacts.
|
|
35
64
|
</output>
|
|
36
65
|
|
|
37
66
|
<quality_bar>
|
|
@@ -14,7 +14,8 @@ You execute the plan. Constrained design judgment within PLAN's invariants — w
|
|
|
14
14
|
|
|
15
15
|
<output>
|
|
16
16
|
- Code changes implementing every Requirement. Verify with `git diff`.
|
|
17
|
-
- Tests added or updated for changed behavior. Run the
|
|
17
|
+
- Tests added or updated for changed behavior. Run the focused development tests needed to establish that behavior; BUILD_GATE and VERIFY own the post-implementation full suite.
|
|
18
|
+
- For every sibling `spec.expected.json.process_evidence[]` item whose `phase` is `implement`, run `python3 "$DEVLYN_SHARED_DIR/process-evidence.py" --devlyn-dir .devlyn run --phase implement --id '<id>'`. The runner must report `expectation_met: true`; cite its manifest path in the phase reply.
|
|
18
19
|
- For each criterion satisfied, set `state.criteria[i].status: "implemented"` with an `evidence` record `{"file": "...", "line": N, "note": "brief"}`.
|
|
19
20
|
- Report your verdict in this reply: `PASS` on success; `BLOCKED` if a criterion cannot be satisfied (missing external dep, blocking ambiguity in the spec) — never silently `pending`. Do not edit `pipeline.state.json` yourself — the orchestrator records it via `state-phase-write.py`.
|
|
20
21
|
</output>
|
|
@@ -22,7 +23,7 @@ You execute the plan. Constrained design judgment within PLAN's invariants — w
|
|
|
22
23
|
<quality_bar>
|
|
23
24
|
- Spec is the contract. The plan is the path. If they disagree, surface the conflict and follow the spec.
|
|
24
25
|
- Bugs: write the failing test first, then fix. Features: follow existing patterns, then write tests. Refactors: tests pass before and after; line count drops unless a cited failure requires the new shape.
|
|
25
|
-
-
|
|
26
|
+
- Do not execute the spec's `verification_commands` in IMPLEMENT; VERIFY MECHANICAL executes those literal commands once after CLEANUP. Declared IMPLEMENT process obligations, including red-first commands, run only through the shared evidence runner above.
|
|
26
27
|
- Tooling-generated artifacts (`test-results/`, `playwright-report/`, `.last-run.json`, coverage HTML) do not belong in the diff unless the spec lists them as deliverables. Configure tools to emit to gitignored paths.
|
|
27
28
|
- Existing tests are contract. Do not replace real HTTP / filesystem / subprocess calls with mocks. Do not skip or disable tests. Do not reduce assertion count on behavior still in scope.
|
|
28
29
|
- Files not in PLAN's list are off-limits. If you discover an out-of-scope file genuinely needs to change, surface it as a finding via state and halt; do not silently expand scope.
|
|
@@ -11,19 +11,28 @@ Independent quality layer. You answer one question: did the diff deliver what th
|
|
|
11
11
|
- `spec.expected.json` — the mechanical acceptance contract per `_shared/expected.schema.json`.
|
|
12
12
|
- The cumulative diff against `state.base_ref.sha`.
|
|
13
13
|
- The source hash (`state.source.spec_sha256` for spec mode, `state.source.criteria_sha256` for generated free-form mode) — re-read the source contract from disk and confirm the hash matches; if it does not, write `state.phases.verify.verdict: "BLOCKED"` with reason `source_sha256_mismatch` and stop.
|
|
14
|
+
- `.devlyn/spec-verify.results.json` plus the validated VERIFY
|
|
15
|
+
process-evidence manifest and raw stdout/stderr streams named by its
|
|
16
|
+
`process_evidence` carrier. These are immutable MECHANICAL results; rehash
|
|
17
|
+
every named byte before JUDGE spawn and again during merge.
|
|
14
18
|
|
|
15
|
-
You do NOT receive: PLAN, IMPLEMENT's reasoning, BUILD_GATE's findings, CLEANUP's allowlist negotiations. Reading those would compromise independence.
|
|
19
|
+
You do NOT receive: PLAN, IMPLEMENT's reasoning, BUILD_GATE's findings, CLEANUP's allowlist negotiations. Reading those would compromise independence. Inspect authorized source, diff and sealed evidence using native read/search tools or non-mutating shell commands; executable verification belongs exclusively to MECHANICAL.
|
|
16
20
|
</input>
|
|
17
21
|
|
|
18
22
|
<sub_phases>
|
|
19
23
|
|
|
20
24
|
### MECHANICAL (deterministic)
|
|
21
25
|
|
|
22
|
-
|
|
26
|
+
The orchestrator completes these checks before spawning either JUDGE,
|
|
27
|
+
independent of BUILD_GATE's earlier run:
|
|
23
28
|
|
|
24
|
-
1. `SPEC_VERIFY_PHASE=verify_mechanical SPEC_VERIFY_FINDINGS_FILE=verify-mechanical.findings.jsonl SPEC_VERIFY_FINDING_PREFIX=VERIFY-MECH python3 "$DEVLYN_SHARED_DIR/spec-verify-check.py" --include-risk-probes` against the post-CLEANUP code. In spec mode, sibling `spec.expected.json` wins; a malformed sibling is CRITICAL, not a fallback. When `state.risk_profile.risk_probes_enabled == true`, missing `.devlyn/risk-probes.jsonl` is also CRITICAL. The script also checks `forbidden_patterns`, `required_files`, `forbidden_files`, and `max_deps_added`.
|
|
29
|
+
1. `SPEC_VERIFY_PHASE=verify_mechanical SPEC_VERIFY_FINDINGS_FILE=verify-mechanical.findings.jsonl SPEC_VERIFY_FINDING_PREFIX=VERIFY-MECH python3 "$DEVLYN_SHARED_DIR/spec-verify-check.py" --include-risk-probes` against the post-CLEANUP code. In spec mode, sibling `spec.expected.json` wins; a malformed sibling is CRITICAL, not a fallback. When `state.risk_profile.risk_probes_enabled == true`, missing `.devlyn/risk-probes.jsonl` is also CRITICAL. The script also checks `forbidden_patterns`, `required_files`, `forbidden_files`, and `max_deps_added`, and writes `.devlyn/spec-verify.results.json` with a VERIFY process-evidence carrier.
|
|
25
30
|
|
|
26
|
-
|
|
31
|
+
2. Validate the carrier and every named raw stream. A missing, altered,
|
|
32
|
+
escaping, duplicate, or expectation-mismatched record is a CRITICAL mechanical
|
|
33
|
+
blocker. Freeze the validated results, manifest, and streams as JUDGE inputs.
|
|
34
|
+
|
|
35
|
+
Emit findings to `.devlyn/verify-mechanical.findings.jsonl`. Each match = one finding. Severity from the pattern's `severity` field (disqualifier → CRITICAL, warning → MEDIUM). A verdict-binding MECHANICAL result skips both JUDGEs.
|
|
27
36
|
|
|
28
37
|
### JUDGE (fresh-context grading)
|
|
29
38
|
|
|
@@ -34,6 +43,14 @@ Grade the diff against the spec on rubric axes:
|
|
|
34
43
|
- **Quality** — does the implementation follow the framework's idiomatic patterns, or are there hand-rolled helpers replacing standard primitives? `design.unidiomatic-pattern` MEDIUM if so.
|
|
35
44
|
- **Consistency** — internal style (naming, error shape, module boundaries) consistent with the surrounding code.
|
|
36
45
|
|
|
46
|
+
**Bounded primary review**: the primary JUDGE makes one broad pass over the
|
|
47
|
+
source contract, sealed MECHANICAL carrier, and cumulative diff, covering all
|
|
48
|
+
four rubric axes and every binding Requirement clause. It then makes one
|
|
49
|
+
targeted interaction pass over the clauses the broad pass left unresolved.
|
|
50
|
+
Before any third pass, emit the required terminal result. If R1–R8 or another
|
|
51
|
+
spec axis remains uncovered, emit a verdict-binding BLOCKED coverage finding
|
|
52
|
+
instead of continuing or assuming PASS.
|
|
53
|
+
|
|
37
54
|
For each finding, write file:line evidence. Do not paraphrase code; quote it.
|
|
38
55
|
|
|
39
56
|
**Clause-level check**: split each Requirement into its binding clauses before
|
|
@@ -54,8 +71,9 @@ coexist, compose them in the stated order instead of inventing a stronger global
|
|
|
54
71
|
ordering. A finding based on a widened invariant is a false positive and must
|
|
55
72
|
not drive the fix loop.
|
|
56
73
|
|
|
57
|
-
**
|
|
58
|
-
|
|
74
|
+
**Targeted interaction pass**: trace interactions among the clauses the broad
|
|
75
|
+
pass left unresolved. For high-complexity specs, one-axis examples are not
|
|
76
|
+
enough: construct at least one adversarial scenario that combines two or more
|
|
59
77
|
explicit verification bullets. Prioritize combinations such as
|
|
60
78
|
ordering/priority + blocked interval/failure, ordering/priority +
|
|
61
79
|
all-or-nothing rollback + later entity state, validation/error-priority +
|
|
@@ -63,22 +81,29 @@ stdout/stderr contract, or auth/idempotency + duplicate/replay ordering. If the
|
|
|
63
81
|
implementation only passes isolated examples but fails the combined scenario,
|
|
64
82
|
emit a HIGH finding tied to all relevant spec clauses.
|
|
65
83
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
**Coverage check**: before declaring done, confirm you have evidence for every spec axis. If
|
|
74
|
-
|
|
75
|
-
**Verdict-binding
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
84
|
+
JUDGE does not execute literal verification, lint, test, build, risk-probe, or
|
|
85
|
+
newly invented interaction commands. For high-complexity behavior, executable
|
|
86
|
+
coverage must already be declared in sibling `spec.expected.json` or derived
|
|
87
|
+
risk probes and present in the sealed MECHANICAL evidence. Missing coverage is
|
|
88
|
+
a verdict-binding finding; review the implementation's clause and code order
|
|
89
|
+
without inventing a replacement command.
|
|
90
|
+
|
|
91
|
+
**Coverage check**: before declaring done, confirm you have sealed evidence and code-order support for every spec axis. If an axis lacks declared MECHANICAL coverage, or the diff does not touch the code that produces it, set `state.verify.coverage_failed: true` and surface the missing-evidence finding rather than passing on assumption.
|
|
92
|
+
|
|
93
|
+
**Verdict-binding check**: a demonstrated violation of an applicable mandatory
|
|
94
|
+
task, public, or existing test contract is binding, including unmet new
|
|
95
|
+
requirements and incorrect customer documentation. Quote the exact applicable
|
|
96
|
+
clause and concrete file:line evidence. Small impact, rare inputs, or a
|
|
97
|
+
pre-existing defect do not excuse that violation. A changed line or requirement
|
|
98
|
+
reference alone does not establish applicability.
|
|
99
|
+
|
|
100
|
+
Emit HIGH/CRITICAL as appropriate, or MEDIUM with literal `verdict_binding: true`.
|
|
101
|
+
Do not label a binding finding LOW/INFO: the merge honors HIGH/CRITICAL and
|
|
102
|
+
MEDIUM/true; confidence is reported but adds no merge threshold. Preferences,
|
|
103
|
+
style, stronger inferred invariants, genuinely ambiguous clauses, and unrelated
|
|
104
|
+
pre-existing issues are not demonstrated mandatory violations. Name the actual
|
|
105
|
+
ambiguity or evidence limitation; do not substitute an impact argument.
|
|
106
|
+
Advisory MEDIUM findings remain non-binding and produce `PASS_WITH_ISSUES`.
|
|
82
107
|
|
|
83
108
|
**Anti-self-filter rule**: report every finding you observe, including ones you consider low-severity or low-confidence. Tag each with `confidence: high|medium|low` and let the harness's downstream filter rank them. Filtering at this stage suppresses recall.
|
|
84
109
|
|
|
@@ -99,7 +124,8 @@ round and merge by worst source verdict without vote counting. An orchestrator
|
|
|
99
124
|
without foreground parallel dispatch runs the same two required judges
|
|
100
125
|
sequentially, with no flag or extra state marker. This is a dispatch-shape
|
|
101
126
|
fallback, not outcome-dependent escalation; the primary cannot skip the pair.
|
|
102
|
-
|
|
127
|
+
Both JUDGEs inspect the same immutable evidence; neither runs probes or mutates
|
|
128
|
+
shared state.
|
|
103
129
|
|
|
104
130
|
Keep these reasons as telemetry; they no longer gate the second spawn:
|
|
105
131
|
`mode.pair-verify`, `mode.verify-only`, `complexity.high`,
|
|
@@ -148,74 +174,80 @@ solo. Never synthesize pair findings.
|
|
|
148
174
|
|
|
149
175
|
When eligible and the orchestrator spawns a second VERIFY agent with the OTHER engine's adapter, both judgments are merged:
|
|
150
176
|
- Any HIGH/CRITICAL finding either model surfaces is verdict-binding.
|
|
151
|
-
- Any
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
duplicate/idempotent/order-preservation regressions and real warning/error
|
|
155
|
-
behavior changes. Do not downgrade these to advisory simply because they are
|
|
156
|
-
not HIGH.
|
|
177
|
+
- Any MEDIUM finding with literal `verdict_binding: true` is also binding.
|
|
178
|
+
Apply the same mandatory-clause, applicability and evidence check above;
|
|
179
|
+
neither impact nor pre-existing origin makes an applicable violation advisory.
|
|
157
180
|
- Other lower-severity disagreements are logged but do not change the verdict.
|
|
158
181
|
- The orchestrator handles merge; you only emit your own findings.
|
|
159
182
|
- The second judge's job is adversarial complement, not a duplicate summary:
|
|
160
|
-
|
|
183
|
+
review the two highest-risk explicit `## Verification` bullets that cross
|
|
161
184
|
state mutation, all-or-nothing rollback, ordering, idempotency, auth, or
|
|
162
185
|
error-priority clauses. The primary judge owns broad coverage; the pair judge
|
|
163
186
|
is a bounded adversarial complement. Do not read `.claude/skills`,
|
|
164
187
|
`.codex/skills`, `CLAUDE.md`, `AGENTS.md`, or other harness docs unless the
|
|
165
188
|
orchestrator pasted a specific excerpt into the prompt. Use only the spec,
|
|
166
|
-
diff, implementation files, tests, and
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
the
|
|
177
|
-
|
|
178
|
-
(stdout/stderr/exit
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
189
|
+
diff, implementation files, tests, and sealed MECHANICAL evidence. Complete at
|
|
190
|
+
most two targeted reviews before first output; inspection remains read-only.
|
|
191
|
+
Pair-JUDGE output: emit JSONL findings then a bare terminal verdict line, or
|
|
192
|
+
emit only `PASS` when clean. `_shared/judge-output-parser.py` is the single
|
|
193
|
+
acceptance rule for pair output: JSONL findings, then a `# SUMMARY {json}`
|
|
194
|
+
line or a bare verdict line (`PASS` alone when clean); it ignores bare
|
|
195
|
+
code-fence lines and unwraps the registered Codex JSON envelope including its
|
|
196
|
+
narrated-preamble recovery (iter-0082), and binds a whole-message NDJSON
|
|
197
|
+
capture only through its uniquely attested terminal `end_turn` assistant
|
|
198
|
+
message (iter-0106); every other non-empty line blocks the pair source.
|
|
199
|
+
If the spec includes a solo-headroom hypothesis, one targeted review must use
|
|
200
|
+
the hypothesis's backticked observable command as its exact anchor and inspect
|
|
201
|
+
the complete sealed result (stdout/stderr/exit plus the full parsed output
|
|
202
|
+
object). Missing matching evidence is a coverage finding; do not substitute or
|
|
203
|
+
run a neighboring edge case. When the spec names exact keys, row shapes, JSON
|
|
204
|
+
object shape, or an exact error body, compare the sealed parsed key sets/deep
|
|
205
|
+
equality so aliased keys, missing keys, and extra keys are verdict-binding
|
|
206
|
+
failures. For priority/stateful specs, trace implementation code order for an
|
|
207
|
+
earlier input entity that would succeed under input-order processing, a later
|
|
208
|
+
higher-priority entity that consumes or blocks the critical resource, and a
|
|
209
|
+
failure/blocked/rollback edge that determines a later entity's state. Scope
|
|
210
|
+
qualifiers are binding for the pair judge too: do not reinterpret `inside a
|
|
211
|
+
warehouse`, `per resource`, or line-scoped rules as global rules. When both
|
|
212
|
+
priority ordering and rollback/blocked-interval behavior appear, perform this
|
|
213
|
+
dominance-loss code-order review first and confirm the sealed evidence covers
|
|
214
|
+
complete accepted/scheduled and rejected output ordering.
|
|
215
|
+
|
|
216
|
+
Both resolved JUDGEs are read-only. Capture the primary reply as
|
|
217
|
+
`.devlyn/<primary-engine>-judge.stdout` and its stderr sibling; never write the
|
|
218
|
+
generic `.devlyn/verify-judge.stdout`. When the primary engine is Codex, invoke
|
|
219
|
+
it only through this distinct monitored route:
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
CODEX_MONITORED_ISOLATED=1 CODEX_MONITORED_TIMEOUT_SEC=600 bash "$CODEX_MONITORED_PATH" -C "$PWD" -s read-only -c model_reasoning_effort=high "<primary prompt>" >.devlyn/codex-judge.stdout 2>.devlyn/codex-judge.stderr
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
This command supplies unconfigured defaults. Explicit judge profiles use
|
|
226
|
+
`SKILL.md#explicit-role-dispatch` for validated model/effort options and
|
|
227
|
+
round-scoped evidence. Omit bypass flags; do not pipe either stream. On primary exit
|
|
228
|
+
124, write `.devlyn/verify.primary.timeout.json` with exactly
|
|
229
|
+
`{"engine": "<resolved-primary-engine>", "budget_seconds": 600}` before merge. The
|
|
230
|
+
marker is primary-only authority: malformed, wrong-engine, or wrong-budget
|
|
231
|
+
content BLOCKs; a valid marker preserves all canonical primary findings and
|
|
232
|
+
floors `judge` at `BLOCKED`, including when findings/stdout are empty. It never
|
|
233
|
+
produces `PASS`, a solo verdict, or pair-style `TIMEOUT`; without the marker,
|
|
234
|
+
existing missing/invalid primary-output behavior remains fail-closed.
|
|
235
|
+
|
|
236
|
+
Codex pair-JUDGE keeps the monitored
|
|
206
237
|
`codex-monitored.sh` route with
|
|
207
238
|
`CODEX_MONITORED_ISOLATED=1 CODEX_MONITORED_TIMEOUT_SEC=600` and
|
|
208
|
-
`-c model_reasoning_effort=medium`; isolation blocks user config, AGENTS.md,
|
|
239
|
+
unconfigured default `-c model_reasoning_effort=medium`; isolation blocks user config, AGENTS.md,
|
|
209
240
|
hooks, and project rules from hidden context/tool side effects. Do not pipe it
|
|
210
241
|
to `tail`, `head`, `grep`, `sed`, or `awk`; capture stdout/stderr directly.
|
|
211
242
|
Every other resolved OTHER engine follows `_shared/adapters/<name>.md`
|
|
212
|
-
`## Invocation`. Capture the
|
|
243
|
+
`## Invocation`. Capture the pair reply as `.devlyn/<other-engine>-judge.stdout`, then run
|
|
213
244
|
`python3 "$DEVLYN_SHARED_DIR/collect-codex-findings.py" --devlyn-dir
|
|
214
|
-
"<abs repo>/.devlyn" --stdout-file <
|
|
245
|
+
"<abs repo>/.devlyn" --stdout-file <other-engine>-judge.stdout` before merge.
|
|
215
246
|
The orchestrator writes the canonical `.devlyn/verify.pair.findings.jsonl`.
|
|
216
247
|
The pair prompt must include a bounded-output contract: no harness-doc reads,
|
|
217
|
-
maximum two targeted
|
|
218
|
-
verdict-binding finding, and emit PASS immediately after the bounded
|
|
248
|
+
read-only inspection, maximum two targeted reviews before first output, stop on
|
|
249
|
+
the first verdict-binding finding, and emit PASS immediately after the bounded
|
|
250
|
+
reviews pass.
|
|
219
251
|
Raw stdout is diagnostic-only. A non-zero collector exit must write no
|
|
220
252
|
canonical findings file: set the pair source to `BLOCKED` for
|
|
221
253
|
`verify.pair.emission-contract`, and do not merge as though unparsed stdout were
|
|
@@ -228,12 +260,14 @@ nested inside, normalized string `sub_verdicts`. A judge subprocess exit
|
|
|
228
260
|
124 is a budget abort: before merge, the orchestrator writes
|
|
229
261
|
`.devlyn/verify.pair.timeout.json` with `{"engine": "<resolved name>",
|
|
230
262
|
"budget_seconds": 600}`. Three cases are binding: marker plus no canonical pair
|
|
231
|
-
findings and no
|
|
232
|
-
|
|
233
|
-
`
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
263
|
+
findings and no stdout findings — including a capture the parser rejects solely
|
|
264
|
+
because its message stream was truncated before the terminal result — records
|
|
265
|
+
`pair_judge: "TIMEOUT"`, computes the merged verdict from mechanical plus
|
|
266
|
+
primary judge, and surfaces `solo verdict after pair TIMEOUT` in the report
|
|
267
|
+
header; marker plus canonical findings or parseable stdout findings binds those
|
|
268
|
+
findings exactly as today, and every other parser rejection stays `BLOCKED`
|
|
269
|
+
under the stdout emission contract; no marker preserves the existing `BLOCKED`
|
|
270
|
+
contract for missing, empty, or uncaptured pair output. A budget
|
|
237
271
|
abort is not an availability fallback; explicit-route availability still fails
|
|
238
272
|
closed.
|
|
239
273
|
|
|
@@ -247,8 +281,8 @@ This deterministic merge is the routing source of truth for VERIFY. It writes
|
|
|
247
281
|
`.devlyn/verify-merged.findings.jsonl`, `.devlyn/verify-merge.summary.json`, and
|
|
248
282
|
updates `state.phases.verify.{verdict,sub_verdicts,merged}`. Branch on the
|
|
249
283
|
merged state verdict, not on either model's prose verdict. Any HIGH/CRITICAL
|
|
250
|
-
finding from either judge is `NEEDS_WORK`; a
|
|
251
|
-
`verdict_binding: true` to become `NEEDS_WORK
|
|
284
|
+
finding from either judge is `NEEDS_WORK`; a MEDIUM must set literal
|
|
285
|
+
`verdict_binding: true` to become `NEEDS_WORK`, regardless of confidence.
|
|
252
286
|
|
|
253
287
|
Do not create, edit, truncate, or summarize `.devlyn/verify-merged.findings.jsonl`
|
|
254
288
|
or `.devlyn/verify-merge.summary.json` by hand. Those files have exactly one
|
|
@@ -259,13 +293,15 @@ VERIFY to `BLOCKED`; do not synthesize merge artifacts in prose.
|
|
|
259
293
|
|
|
260
294
|
<output>
|
|
261
295
|
- `.devlyn/verify-mechanical.findings.jsonl` — MECHANICAL findings.
|
|
296
|
+
- `.devlyn/spec-verify.results.json` plus its sealed VERIFY process-evidence manifest/raw streams — immutable JUDGE inputs.
|
|
262
297
|
- `.devlyn/verify.findings.jsonl` — JUDGE findings.
|
|
263
298
|
- `.devlyn/verify-merged.findings.jsonl` and `.devlyn/verify-merge.summary.json` — deterministic merge artifacts.
|
|
264
|
-
- `phases.verify.{verdict, sub_verdicts, merged}` are written by `verify-merge-findings.py --write-state`, never by you (VERIFY agents have no code-mutation tools). `completed_at`/`duration_ms`/`artifacts` are recorded by the orchestrator via `state-phase-write.py` after this phase returns. `PASS` requires zero CRITICAL/HIGH findings, zero verdict-binding MEDIUM
|
|
299
|
+
- `phases.verify.{verdict, sub_verdicts, merged}` are written by `verify-merge-findings.py --write-state`, never by you (VERIFY agents have no code-mutation tools). `completed_at`/`duration_ms`/`artifacts` are recorded by the orchestrator via `state-phase-write.py` after this phase returns. `PASS` requires zero CRITICAL/HIGH findings, zero verdict-binding MEDIUM findings, and coverage met.
|
|
265
300
|
</output>
|
|
266
301
|
|
|
267
302
|
<quality_bar>
|
|
268
303
|
- Independence is structural (fresh context) and behavioral (no code mutation). Both must hold.
|
|
304
|
+
- MECHANICAL is the sole verification executor; JUDGE performs sealed-evidence and clause/code-order review only.
|
|
269
305
|
- Quote, do not paraphrase. Findings without quoted file:line evidence are excluded.
|
|
270
306
|
- Coverage > confidence. Missing-evidence findings outrank a confident "looks fine."
|
|
271
307
|
</quality_bar>
|
|
@@ -15,6 +15,7 @@ Single authoritative verdict source for `/devlyn:resolve`. The orchestrator bran
|
|
|
15
15
|
"complexity": null,
|
|
16
16
|
"risk_profile": { "high_risk": false, "reasons": [], "risk_probes_enabled": false, "risk_probes_explicit": false, "pair_default_enabled": true },
|
|
17
17
|
"risk_probes_digest": null,
|
|
18
|
+
"process_evidence": null,
|
|
18
19
|
"base_ref": { "branch": "main", "sha": "abc123..." },
|
|
19
20
|
"rounds": { "max_rounds": 4, "global": 0 },
|
|
20
21
|
"bypasses": [],
|
|
@@ -53,8 +54,11 @@ Single authoritative verdict source for `/devlyn:resolve`. The orchestrator bran
|
|
|
53
54
|
- **engine** — any engine name with a shipped adapter (`_shared/adapters/<name>.md`); `"claude"` and `"codex"` today. A required unavailable engine stops the run with `BLOCKED:<engine>-unavailable`.
|
|
54
55
|
- **engine_source** — `"flag" | "engines.json" | "default"` — provenance of the resolved executor engine (`_shared/engine-preflight.md#role-resolution`). Optional on archived pre-iter-0038 runs; absent means `default`.
|
|
55
56
|
- **source** — downstream contract provenance. Spec/verify-only set `type: "spec"`, `spec_path`, and `spec_sha256`. Free-form sets `type: "generated"`, `goal_path: ".devlyn/goal.raw.txt"` + exact-byte `goal_sha256`, and `criteria_path: ".devlyn/criteria.generated.md"` + raw-byte `criteria_sha256`. VERIFY re-checks its contract hash.
|
|
57
|
+
- **`.devlyn/external-diff.patch` artifact** — verify-only input; the verifier consumes it only when `mode` is `"verify-only"` and fails closed if it exists in any other mode.
|
|
56
58
|
- **risk_profile** — PHASE 0 routing state. `high_risk` controls automatic risk probes; `risk_probes_enabled` / `risk_probes_explicit` preserve their current semantics; `pair_default_enabled` is false only for explicit `--no-pair`. `risk_profile` must remain an object with boolean `high_risk`, `risk_probes_enabled`, `risk_probes_explicit`, and `pair_default_enabled` fields when present, plus `reasons` as a string list. Malformed state blocks VERIFY because it can hide required routes.
|
|
57
59
|
- **risk_probes_digest** — top-level sha256 written by PHASE 1.5 after probe validation; BUILD_GATE/VERIFY replay semantics live in `phases/build-gate.md` step 4.
|
|
60
|
+
- **process_evidence** — absent or JSON null for legacy runs and new runs with no captured obligation; otherwise an append-only array of `{phase, round, manifest: {path, sha256}, streams: [{id, stdout: {path, sha256}, stderr: {path, sha256}}]}` carriers. Paths are worktree-relative and live under `.devlyn/process-evidence/<run_id>/<phase>/round-<round>/`. A successful IMPLEMENT completion/transition validates every sibling `spec.expected.json.process_evidence[]` item declared for `phase: "implement"`, rehashes both raw streams (including zero-byte files), and appends the carrier before accepting the checkpoint. Missing, altered, escaping, duplicate-id, or expectation-mismatched evidence blocks without a state mutation. Later phases reuse the same carrier shape; no schema-major bump is required because readers must accept absent/null legacy state.
|
|
61
|
+
- **base_ref.branch** — checked-out branch name, or `null` for a detached HEAD; `base_ref.sha` remains the exact immutable base in both states.
|
|
58
62
|
- **rounds.global** — incremented every fix-loop pass (BUILD_GATE → fix-loop, VERIFY → fix-loop, OR a phase-gate fix respawn inside phase-gated IMPLEMENT).
|
|
59
63
|
- **phases.implement.exec** — only on phase-gated large runs (plan.md has `## Execution phases` with >1 phase): `{ "total": N, "current": k, "statuses": ["PASS"|"FAIL"|null, ...], "commits": ["<sha>", ...] }`. Phase definitions live in plan.md (immutable contract); progress lives here (routing truth — never route on plan.md checkboxes). Absent on single-phase runs. `cumulative.patch` for such runs is `git diff <base_ref.sha>...HEAD`.
|
|
60
64
|
- **phases.probe_derive** — optional PHASE 1.5 entry when `--risk-probes` is enabled. Artifacts include `.devlyn/risk-probes.jsonl`. Probe failures later surface through BUILD_GATE/VERIFY as `correctness.risk-probe-failed`, or as `correctness.verification-timeout` when the probe exceeds its verification budget.
|
|
@@ -88,10 +92,12 @@ Each entry under `phases.<name>` (for `plan`, `probe_derive`, `implement`, `surf
|
|
|
88
92
|
}
|
|
89
93
|
```
|
|
90
94
|
|
|
91
|
-
`history` is absent until a phase is re-entered. Immediately before re-entry overwrites the live
|
|
95
|
+
`history` is absent until a phase is re-entered. Immediately before re-entry overwrites the live lifecycle, `state-phase-write.py` appends those prior values as one object to `history[]`, creating the append-only array on first re-entry. Codex worker history also retains its state-bound `invocation_receipt`; PLAN history retains the complete prompt/output receipt.
|
|
92
96
|
|
|
93
97
|
- `verdict` — `"PASS" | "PASS_WITH_ISSUES" | "FAIL" | "NEEDS_WORK" | "BLOCKED"`. PHASE 6 (FINAL_REPORT) writes its own verdict per the terminal-verdict precedence.
|
|
94
|
-
- `model_requested` / `model_effective` — `model_requested` is supplied by the orchestrator through `--model`;
|
|
98
|
+
- `model_requested` / `model_effective` — `model_requested` is supplied by the orchestrator through `--model`; schema-v3 completion cannot replace the spawn engine or requested model. Schema-v3 Codex PLAN/mutation/BUILD_GATE phases validate requested dispatch using the canonical `.devlyn/<phase>.invocation.<round>.json`; state rehashes the receipt, round-scoped prompt and worker session and cross-checks model, sandbox, prompt, run, phase, round and exit. A structured native model-reroute event or malformed JSONL blocks completion, even at exit0. The receipt proves argv delivery, so `model_effective` remains null; absence of a reroute is not positive model attestation. Arbitrary paths and plaintext `model:` headers are not evidence. Claude JSON output accepts a singleton `modelUsage` key or, for multiple entries, the sole key whose token counters exactly match top-level usage. Missing/malformed evidence and requested/effective mismatch fail closed: the phase verdict becomes `BLOCKED` and completion fails.
|
|
99
|
+
- `output_sha256` — on schema-v3 PLAN completion, SHA-256 of exact `.devlyn/plan.md` bytes. Only caller-verdict BLOCKED with a lexically absent, never-bound plan records null; receipt/session attestation still applies. This terminal null permits only FINAL_REPORT lifecycle operations while the path remains lexically absent. Bound bytes are rehashed on later spawn/completion/transition, including BLOCKED; legal PLAN re-entry replaces the digest only at the new PLAN completion. FINAL_REPORT completion binds exact `.devlyn/final-report.md` bytes here and its canonical path in `artifacts.log_file`.
|
|
100
|
+
- `invocation_receipt` — state-bound `{path, sha256, sandbox, sandbox_network_access, argv_sha256, exit_code}` for schema-v3 Codex PLAN/IMPLEMENT/BUILD_GATE/CLEANUP. Receipt schema 2.0 makes `sandbox` exactly `workspace-write`; `sandbox_network_access` is exactly `true` for BUILD_GATE and `false` for the other mutation phases. A missing CI capability or wider phase authority blocks before launch. Pre-2.0 receipts are intentionally rejected rather than migrated because they did not attest this capability; do not rewrite in-flight evidence. Continue through the owning session or start in a distinct worktree; restarting via explicit manual archive requires the operator to establish that prior writers stopped, never automatic archival to defeat admission refusal. A completed final report does not prove all interactive writers exited. The receipt is retained in history and rehashed before archive.
|
|
95
101
|
- `triggered_by` — null on first run; one of `"build_gate" | "verify"` when the phase is a fix-loop respawn.
|
|
96
102
|
- `pre_sha` — captured before SURFACE_CLOSE or CLEANUP. It bounds that phase's post-spawn diff and rollback.
|
|
97
103
|
- `post_sha` — captured when SURFACE_CLOSE or CLEANUP completes; finish-gate subtracts CLEANUP's `pre_sha..post_sha` window.
|
|
@@ -105,10 +111,10 @@ Each entry under `phases.<name>` (for `plan`, `probe_derive`, `implement`, `surf
|
|
|
105
111
|
|
|
106
112
|
Phase lifecycle (`started_at`/`completed_at`/`duration_ms`/`round`/`triggered_by`/`verdict`) is written by a deterministic script, never hand-edited JSON — a prior hand-edited fix-loop respawn left `started_at` stale, corrupting cross-phase ordering because `completed_at`/`round`/`triggered_by` advanced to the new round while `started_at` didn't. Phase workers report their verdict and artifact paths in their reply; they never edit `pipeline.state.json` themselves.
|
|
107
113
|
|
|
108
|
-
1. **Spawn**: `state-phase-write.py --devlyn-dir .devlyn --phase <name> spawn --round <N> [--triggered-by build_gate|verify] [--pre-sha <sha>] [--engine <e>] [--model <requested-model>]`. Re-entry preserves lifecycle in `history[]`, resets owned fields, and leaves fields such as `phases.implement.exec`; VERIFY also clears prior-round files.
|
|
109
|
-
2. **Transition** (normal direct handoff): `state-phase-write.py ... --phase <current> transition` takes the same completion/attestation arguments plus caller-specified `--next-phase`, `--next-round`, `--next-triggered-by`, `--next-engine`, `--next-model`, and next-phase metadata. It validates a fixed legal edge, applies complete plus spawn to a detached candidate, and replaces state once; either both lifecycle writes land or neither. The verb returns JSON paths/digests/state facts only and never chooses a phase, engine, branch, prompt, or agent.
|
|
110
|
-
3. **Complete** (after the orchestrator reads the phase's reply and determines the verdict, when no next phase opens): `python3 "$DEVLYN_SHARED_DIR/state-phase-write.py" --devlyn-dir .devlyn --phase <name> complete --verdict <V> [--post-sha <sha>] [--findings-file <path>] [--log-file <path>] [--engine-session-log <path>]`. Pass the
|
|
111
|
-
4. **Before archive** (PHASE 6 step 5): `
|
|
114
|
+
1. **Spawn**: `state-phase-write.py --devlyn-dir .devlyn --phase <name> spawn --round <N> [--triggered-by build_gate|verify] [--pre-sha <sha>] [--engine <e>] [--model <requested-model>] [--prompt-sha256 <hex>]`. Re-entry preserves lifecycle/receipts in `history[]`, resets owned fields, and leaves fields such as `phases.implement.exec`; VERIFY also clears prior-round files. Schema-v3 Codex IMPLEMENT/BUILD_GATE/CLEANUP requires the requested model and prompt digest; an inherited top-level Codex engine is persisted into the phase rather than becoming an unattested null route. SURFACE_CLOSE inherits native Claude model settings when `--model` is omitted and rejects explicit blank models; it additionally requires `--input-patch-sha256 <hex> --prompt-sha256 <hex> --untracked-before-json <array>`; use `surface-check --authorized-surface-json <array>`, `surface-adjudication-recover --authorized-surface-json <array>` only after its adjudication-malformed result, `surface-rollback` for other failures, or `surface-skip` only for automatic Claude unavailability.
|
|
115
|
+
2. **Transition** (normal direct handoff): `state-phase-write.py ... --phase <current> transition` takes the same completion/attestation arguments plus caller-specified `--next-phase`, `--next-round`, `--next-triggered-by`, `--next-engine`, `--next-model`, and next-phase metadata. It validates a fixed legal edge, applies complete plus spawn to a detached candidate, and replaces state once; either both lifecycle writes land or neither. A passing IMPLEMENT edge first validates and binds all declared process evidence through `_shared/process-evidence.py`. The verb returns JSON paths/digests/state facts only and never chooses a phase, engine, branch, prompt, or agent.
|
|
116
|
+
3. **Complete** (after the orchestrator reads the phase's reply and determines the verdict, when no next phase opens): `python3 "$DEVLYN_SHARED_DIR/state-phase-write.py" --devlyn-dir .devlyn --phase <name> complete --verdict <V> [--post-sha <sha>] [--findings-file <path>] [--log-file <path>] [--engine-session-log <path>]`. Pass only the phase/round canonical worker session. SURFACE_CLOSE requires `.devlyn/surface-close.output.json` even for inherited model selection (`model_requested=null`); native JSON supplies `model_effective`, and missing/malformed evidence or an explicit model mismatch blocks completion; schema-v3 Codex PLAN/IMPLEMENT/BUILD_GATE/CLEANUP passes the wrapper-captured `.devlyn/<phase>.worker-session.<round>.jsonl` and requires its sibling invocation receipt plus `.devlyn/<phase>.prompt.<round>`. Never scan an engine-global session directory or synthesize a model header. `duration_ms` is derived from the phase's own `started_at`. `--verdict` is required except VERIFY, whose verdict remains owned by `verify-merge-findings.py --write-state`.
|
|
117
|
+
4. **Before archive** (PHASE 6 step 5): write `.devlyn/final-report.md` with exactly one first-line `<!-- devlyn:final-report run_id=<current state.run_id> -->` and a non-whitespace body, then complete FINAL_REPORT with `--log-file .devlyn/final-report.md`. Missing, unsafe, empty or mismatched report input fails before completion; the marker proves run identity, not semantic completeness. This also applies to BLOCKED missing-PLAN-output closure. Archive rehashes a present FINAL_REPORT `output_sha256` binding before any move; missing/altered files or malformed/null bindings fail without moving artifacts. Historical states with that field absent retain existing archive behavior. Archive prune skips runs whose final_report verdict is null (treated as in-flight).
|
|
112
118
|
|
|
113
119
|
## Terminal verdict (PHASE 6)
|
|
114
120
|
|
|
@@ -116,7 +122,7 @@ Precedence:
|
|
|
116
122
|
|
|
117
123
|
1. finish-gate exit 1 or 2 → `BLOCKED:finish-gate-unclean`; `phases.<any>.verdict == "BLOCKED"` → terminal `BLOCKED:<reason>`.
|
|
118
124
|
2. `phases.verify.verdict == "NEEDS_WORK"` after fix-loop exhaustion → terminal `NEEDS_WORK`.
|
|
119
|
-
3. `phases.verify.verdict == "PASS_WITH_ISSUES"`
|
|
125
|
+
3. `phases.verify.verdict == "PASS_WITH_ISSUES"` → terminal `PASS_WITH_ISSUES`; any finish-gate offender → `BLOCKED:finish-gate-unclean` under precedence 1.
|
|
120
126
|
4. `phases.verify.verdict == "PASS"` → terminal `PASS`.
|
|
121
127
|
5. Verify-only mode: terminal = `phases.verify.verdict` directly (PHASE 1-4 are skipped).
|
|
122
128
|
|
|
@@ -132,4 +138,12 @@ Follow-up notes: when `phases.surface_close.continued_after_block` is true, exac
|
|
|
132
138
|
|
|
133
139
|
## Archive contract
|
|
134
140
|
|
|
135
|
-
PHASE 6 step 4 moves `PER_RUN_PATTERNS` into `.devlyn/runs/<run_id>/`,
|
|
141
|
+
PHASE 6 step 4 moves `PER_RUN_PATTERNS` plus every state-bound process manifest/raw stream and invocation receipt into `.devlyn/runs/<run_id>/`, preserving relative layout and rehashing each bound digest first. It includes every phase/round prompt and worker session, including BUILD_GATE, and revalidates the receipt's internal prompt/session digests; it never scans engine-global sessions. Unsafe paths, symlinks, missing files, digest mismatches, and destination collisions block archive visibly without changing the already-derived product verdict. Machine config stays; the last 10 completed runs remain.
|
|
142
|
+
|
|
143
|
+
## Explicit role configuration
|
|
144
|
+
|
|
145
|
+
PHASE0 `state-phase-write.py --freeze-roles --default-engine <orchestrator-default>` freezes `role_resolution` (legacy engine/source, three role entries, input paths/digests, snapshot digest). `role_config_input` holds only the validated per-run roles object and provenance; `role_no_pair` records that flag. No config reread after freezing. Missing legacy snapshots are compatible; present null/malformed snapshots fail. The helper does not launch models.
|
|
146
|
+
|
|
147
|
+
`state.engine` remains the legacy executor for unchanged PLAN/BUILD/probe routing. IMPLEMENT/CLEANUP spawn enforce frozen worker engine/model. VERIFY spawn explicitly records the primary in `phases.verify.engine`; merge, timeout and OTHER exclusion use it, falling back only when absent. Requested settings are distinct from native-observed fields. Explicit Codex worker effort binds `role_argv` against the existing invocation argv digest; effort remains dispatch evidence, not provider-internal attestation.
|
|
148
|
+
|
|
149
|
+
Successful explicit judges retain `<engine>-judge.r<round>.*` raw/prompt/argv/derived/evidence files; `verify.role_evidence` binds their hashes, observed model/nullable effort and named evidence basis. VERIFY history preserves those bindings. Merge alone binds successful evidence; archive rehashes every bound artifact. Canonical `<engine>-judge.stdout` is the current collector input and is never substituted for the immutable round raw. Failed raw remains diagnostic and cannot become a successful binding.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "devlyn-cli",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.1",
|
|
4
4
|
"description": "Loop & harness engineering for AI coding agents (Claude Code, Codex, and any AGENTS.md-driven agent) — take intent to shipped, engineer-quality software hands-free with phase-gated pipelines and agent orchestration",
|
|
5
5
|
"homepage": "https://github.com/fysoul17/devlyn-cli#readme",
|
|
6
6
|
"bin": {
|
|
@@ -33,10 +33,14 @@
|
|
|
33
33
|
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json",
|
|
34
34
|
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md",
|
|
35
35
|
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json",
|
|
36
|
+
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json",
|
|
37
|
+
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json",
|
|
38
|
+
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json",
|
|
36
39
|
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md",
|
|
37
40
|
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json",
|
|
38
41
|
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md",
|
|
39
42
|
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json",
|
|
43
|
+
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json",
|
|
40
44
|
"benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md",
|
|
41
45
|
"benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json",
|
|
42
46
|
"benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md",
|