devlyn-cli 2.10.0 → 2.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CLAUDE.md +1 -1
- package/README.md +2 -2
- package/benchmark/auto-resolve/scripts/run-fixture.sh +11 -1
- package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +83 -1
- package/bin/devlyn.js +53 -8
- package/config/skills/_shared/adapters/grok.md +34 -12
- package/config/skills/_shared/collect-codex-findings.py +228 -52
- package/config/skills/_shared/expected.schema.json +7 -0
- package/config/skills/_shared/grok-anchor-guard.py +61 -0
- package/config/skills/_shared/phase-prompt-render.py +181 -0
- package/config/skills/_shared/spec-verify-check.py +308 -31
- package/config/skills/_shared/state-phase-write.py +720 -32
- package/config/skills/_shared/verify-merge-findings.py +170 -0
- package/config/skills/devlyn:engines/SKILL.md +1 -1
- package/config/skills/devlyn:ideate/SKILL.md +1 -1
- package/config/skills/devlyn:resolve/SKILL.md +7 -6
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +2 -2
- package/config/skills/devlyn:resolve/references/phases/implement.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/plan.md +1 -5
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +4 -2
- package/config/skills/devlyn:resolve/references/phases/verify.md +5 -1
- package/config/skills/devlyn:resolve/references/state-schema.md +2 -2
- package/package.json +1 -1
- package/scripts/lint-fixtures.sh +4 -1
- package/scripts/lint-skills.sh +138 -10
|
@@ -169,6 +169,44 @@ def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str,
|
|
|
169
169
|
source_verdicts[source], RANK_VERDICT[finding_rank(item)]
|
|
170
170
|
)
|
|
171
171
|
findings.extend(detect_pair_stdout_contract_violations(devlyn, source_verdicts))
|
|
172
|
+
pair_summary_path = devlyn / "pair-judge.summary.json"
|
|
173
|
+
pair_carrier_exists = any(
|
|
174
|
+
(devlyn / name).is_file()
|
|
175
|
+
for name in ("verify.pair.findings.jsonl", "verify.pair-judge.findings.jsonl")
|
|
176
|
+
)
|
|
177
|
+
if pair_carrier_exists and pair_summary_path.is_file():
|
|
178
|
+
try:
|
|
179
|
+
pair_summary = loads_strict_json(pair_summary_path.read_text(encoding="utf-8"))
|
|
180
|
+
except ValueError as exc:
|
|
181
|
+
summary_error = f"pair-judge.summary.json is malformed JSON: {exc}"
|
|
182
|
+
else:
|
|
183
|
+
pair_verdict = pair_summary.get("verdict") if isinstance(pair_summary, dict) else None
|
|
184
|
+
summary_error = None
|
|
185
|
+
if not isinstance(pair_summary, dict):
|
|
186
|
+
summary_error = "pair-judge.summary.json must be a JSON object."
|
|
187
|
+
elif not isinstance(pair_verdict, str) or pair_verdict not in {
|
|
188
|
+
"PASS", "PASS_WITH_ISSUES", "FAIL", "NEEDS_WORK", "BLOCKED"
|
|
189
|
+
}:
|
|
190
|
+
summary_error = (
|
|
191
|
+
"pair-judge.summary.json verdict must be PASS, PASS_WITH_ISSUES, "
|
|
192
|
+
"FAIL, NEEDS_WORK, or BLOCKED."
|
|
193
|
+
)
|
|
194
|
+
if summary_error is not None:
|
|
195
|
+
findings.append(
|
|
196
|
+
pair_blocker(
|
|
197
|
+
"verify-pair-summary-invalid",
|
|
198
|
+
summary_error,
|
|
199
|
+
pair_summary_path.name,
|
|
200
|
+
)
|
|
201
|
+
)
|
|
202
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
203
|
+
elif (
|
|
204
|
+
source_verdicts["pair_judge"] != "TIMEOUT"
|
|
205
|
+
or rank(pair_verdict) > rank("TIMEOUT")
|
|
206
|
+
):
|
|
207
|
+
source_verdicts["pair_judge"] = worse(
|
|
208
|
+
source_verdicts["pair_judge"], pair_verdict
|
|
209
|
+
)
|
|
172
210
|
return findings, source_verdicts
|
|
173
211
|
|
|
174
212
|
|
|
@@ -2331,6 +2369,138 @@ def self_test() -> int:
|
|
|
2331
2369
|
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
2332
2370
|
assert summary["verdict"] == "PASS", summary
|
|
2333
2371
|
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "PASS", state
|
|
2372
|
+
|
|
2373
|
+
# iter-0083: canonical summary verdict conservation.
|
|
2374
|
+
iter_0083_paths = (
|
|
2375
|
+
"pipeline.state.json",
|
|
2376
|
+
"verify-mechanical.findings.jsonl",
|
|
2377
|
+
"verify.findings.jsonl",
|
|
2378
|
+
"verify.pair.findings.jsonl",
|
|
2379
|
+
"verify.pair-judge.findings.jsonl",
|
|
2380
|
+
"pair-judge.summary.json",
|
|
2381
|
+
"codex-primary-judge.summary.json",
|
|
2382
|
+
"grok-judge.summary.json",
|
|
2383
|
+
"verify.pair.timeout.json",
|
|
2384
|
+
"codex-judge.stdout",
|
|
2385
|
+
"claude-judge.stdout",
|
|
2386
|
+
)
|
|
2387
|
+
|
|
2388
|
+
def iter_0083_reset() -> None:
|
|
2389
|
+
for name in iter_0083_paths:
|
|
2390
|
+
(devlyn / name).unlink(missing_ok=True)
|
|
2391
|
+
|
|
2392
|
+
def iter_0083_case(
|
|
2393
|
+
summary_payload: object | None,
|
|
2394
|
+
severity: str | None,
|
|
2395
|
+
*,
|
|
2396
|
+
carrier: str | None = "verify.pair.findings.jsonl",
|
|
2397
|
+
other_summary: str | None = None,
|
|
2398
|
+
timeout: bool = False,
|
|
2399
|
+
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
|
2400
|
+
iter_0083_reset()
|
|
2401
|
+
if carrier is not None:
|
|
2402
|
+
content = (
|
|
2403
|
+
json.dumps({"id": "iter-0083", "severity": severity}) + "\n"
|
|
2404
|
+
if severity
|
|
2405
|
+
else ""
|
|
2406
|
+
)
|
|
2407
|
+
(devlyn / carrier).write_text(content, encoding="utf-8")
|
|
2408
|
+
if summary_payload is not None:
|
|
2409
|
+
content = (
|
|
2410
|
+
summary_payload
|
|
2411
|
+
if isinstance(summary_payload, str)
|
|
2412
|
+
else json.dumps(summary_payload)
|
|
2413
|
+
)
|
|
2414
|
+
(devlyn / "pair-judge.summary.json").write_text(content, encoding="utf-8")
|
|
2415
|
+
if other_summary is not None:
|
|
2416
|
+
(devlyn / other_summary).write_text(
|
|
2417
|
+
json.dumps({"verdict": "BLOCKED"}), encoding="utf-8"
|
|
2418
|
+
)
|
|
2419
|
+
if timeout:
|
|
2420
|
+
(devlyn / "verify.pair.timeout.json").write_text(
|
|
2421
|
+
json.dumps({"engine": "claude", "budget_seconds": 600}),
|
|
2422
|
+
encoding="utf-8",
|
|
2423
|
+
)
|
|
2424
|
+
case_findings, case_source_verdicts = read_findings(devlyn)
|
|
2425
|
+
return case_findings, write_outputs(devlyn, case_findings, case_source_verdicts)
|
|
2426
|
+
|
|
2427
|
+
for case_id, verdict, severity, expected in (
|
|
2428
|
+
("P1", "NEEDS_WORK", "INFO", "NEEDS_WORK"),
|
|
2429
|
+
("P2", "NEEDS_WORK", "LOW", "NEEDS_WORK"),
|
|
2430
|
+
("P3", "NEEDS_WORK", "MEDIUM", "NEEDS_WORK"),
|
|
2431
|
+
("P4", "PASS", "HIGH", "NEEDS_WORK"),
|
|
2432
|
+
("P5", "BLOCKED", "INFO", "BLOCKED"),
|
|
2433
|
+
("P6", "FAIL", "INFO", "NEEDS_WORK"),
|
|
2434
|
+
("P7", "PASS", None, "PASS"),
|
|
2435
|
+
):
|
|
2436
|
+
_, summary = iter_0083_case({"verdict": verdict}, severity)
|
|
2437
|
+
assert summary["source_verdicts"]["pair_judge"] == expected, case_id
|
|
2438
|
+
assert summary["verdict"] == expected, case_id
|
|
2439
|
+
|
|
2440
|
+
for case_id, severity, expected in (
|
|
2441
|
+
("P8-INFO", "INFO", "PASS"),
|
|
2442
|
+
("P8-LOW", "LOW", "PASS_WITH_ISSUES"),
|
|
2443
|
+
):
|
|
2444
|
+
_, summary = iter_0083_case(None, severity)
|
|
2445
|
+
assert summary["source_verdicts"]["pair_judge"] == expected, case_id
|
|
2446
|
+
assert summary["verdict"] == expected, case_id
|
|
2447
|
+
|
|
2448
|
+
def assert_iter_0083_blocked(
|
|
2449
|
+
case_id: str,
|
|
2450
|
+
case_findings: list[dict[str, Any]],
|
|
2451
|
+
summary: dict[str, Any],
|
|
2452
|
+
) -> None:
|
|
2453
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", case_id
|
|
2454
|
+
assert summary["verdict"] == "BLOCKED", case_id
|
|
2455
|
+
assert any(
|
|
2456
|
+
finding.get("source") == "pair_judge"
|
|
2457
|
+
and finding.get("severity") == "CRITICAL"
|
|
2458
|
+
and finding.get("file") == "pair-judge.summary.json"
|
|
2459
|
+
for finding in case_findings
|
|
2460
|
+
), case_id
|
|
2461
|
+
|
|
2462
|
+
for case_id, payload in (
|
|
2463
|
+
("N1-malformed", "{"),
|
|
2464
|
+
("N1-non-object", []),
|
|
2465
|
+
("N1-unknown", {"verdict": "UNKNOWN"}),
|
|
2466
|
+
("N2", {}),
|
|
2467
|
+
("N7", {"verdict": "TIMEOUT"}),
|
|
2468
|
+
):
|
|
2469
|
+
case_findings, summary = iter_0083_case(payload, "INFO")
|
|
2470
|
+
assert_iter_0083_blocked(case_id, case_findings, summary)
|
|
2471
|
+
|
|
2472
|
+
_, summary = iter_0083_case({"verdict": "BLOCKED"}, None, carrier=None)
|
|
2473
|
+
assert summary["source_verdicts"]["pair_judge"] is None, summary
|
|
2474
|
+
assert summary["verdict"] == "PASS", summary
|
|
2475
|
+
|
|
2476
|
+
_, summary = iter_0083_case(
|
|
2477
|
+
{"verdict": "PASS"},
|
|
2478
|
+
"INFO",
|
|
2479
|
+
other_summary="codex-primary-judge.summary.json",
|
|
2480
|
+
)
|
|
2481
|
+
assert summary["source_verdicts"]["pair_judge"] == "PASS", "N4"
|
|
2482
|
+
assert summary["verdict"] == "PASS", "N4"
|
|
2483
|
+
|
|
2484
|
+
_, summary = iter_0083_case(
|
|
2485
|
+
None,
|
|
2486
|
+
"INFO",
|
|
2487
|
+
other_summary="grok-judge.summary.json",
|
|
2488
|
+
)
|
|
2489
|
+
assert summary["source_verdicts"]["pair_judge"] == "PASS", "N5"
|
|
2490
|
+
assert summary["verdict"] == "PASS", "N5"
|
|
2491
|
+
|
|
2492
|
+
_, summary = iter_0083_case(
|
|
2493
|
+
{"verdict": "NEEDS_WORK"},
|
|
2494
|
+
"INFO",
|
|
2495
|
+
carrier="verify.pair-judge.findings.jsonl",
|
|
2496
|
+
)
|
|
2497
|
+
assert summary["source_verdicts"]["pair_judge"] == "NEEDS_WORK", "N6"
|
|
2498
|
+
assert summary["verdict"] == "NEEDS_WORK", "N6"
|
|
2499
|
+
|
|
2500
|
+
case_findings, summary = iter_0083_case({"verdict": "BLOCKED"}, None, timeout=True)
|
|
2501
|
+
assert case_findings == [], "VERIFY-JUDGE-001"
|
|
2502
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", "VERIFY-JUDGE-001"
|
|
2503
|
+
assert summary["verdict"] == "BLOCKED", "VERIFY-JUDGE-001"
|
|
2334
2504
|
return 0
|
|
2335
2505
|
|
|
2336
2506
|
|
|
@@ -40,7 +40,7 @@ pair judge codex binary rule (pin order with: pair <name>,...)
|
|
|
40
40
|
|
|
41
41
|
## Subcommands
|
|
42
42
|
|
|
43
|
-
- `executor <name>` — pin the executor (
|
|
43
|
+
- `executor <name>` — pin the executor (IMPLEMENT/CLEANUP + primary VERIFY judge; PLAN is orchestrator-fixed and never follows the pin). Refuse names with no `_shared/adapters/<name>.md`, or whose adapter's `## Role eligibility` section declares `executor: no` (judge-only backends), listing the valid names either way. If the engine's CLI is not currently available, still write the pin but warn: pins are promises — the run will stop with `BLOCKED:<name>-unavailable` until it is installed.
|
|
44
44
|
- `pair <name>[,<name>...]` — set `pair_judge_priority` (ordered; first adapter-valid, pair-judge-eligible, non-primary, available entry wins at VERIFY/risk-probe time). Refuse names with no `_shared/adapters/<name>.md`, or whose adapter declares `pair_judge: no`.
|
|
45
45
|
- `clear` — remove both pins; delete `.devlyn/engines.json` when nothing else remains in it.
|
|
46
46
|
|
|
@@ -24,7 +24,7 @@ The user does not know context engineering. They will under-specify and over-ass
|
|
|
24
24
|
</elicitation_contract>
|
|
25
25
|
|
|
26
26
|
<harness_principles>
|
|
27
|
-
Read `_shared/runtime-principles.md` (Subtractive-first / Goal-locked /
|
|
27
|
+
Read `_shared/runtime-principles.md` (Subtractive-first / Goal-locked / Evidence-over-claim). The principles bind the spec content as well as your conversation. A spec that says "for future flexibility" is a Subtractive-first violation. A spec that asks for `try { ... } catch { return null }` is a No-workaround violation. AI flags these in elicitation, not after `/devlyn:resolve` has built them.
|
|
28
28
|
</harness_principles>
|
|
29
29
|
|
|
30
30
|
<runtime_paths>
|
|
@@ -51,10 +51,10 @@ fi
|
|
|
51
51
|
Each phase routes to an engine and prepends the per-engine adapter header from `_shared/adapters/<engine>.md` (e.g. `claude.md`, `codex.md`) to the canonical phase body. Adapter is the per-model delta (Anthropic's prompt-engineering guide for Claude, OpenAI's prompt guidance for Codex). Canonical body is engine-agnostic.
|
|
52
52
|
|
|
53
53
|
- Phase spawning is mandatory for every orchestrator. Same-context PLAN / IMPLEMENT / BUILD_GATE / CLEANUP / VERIFY is a contract violation. If the current CLI cannot spawn a fresh worker, write the current phase verdict as `"BLOCKED"` and report `BLOCKED:fresh-context-unavailable` with the failed spawn command; do not continue with ad-hoc same-context execution.
|
|
54
|
-
- Claude Code phases: spawn `Agent` (`mode: "bypassPermissions"`); prompt = adapter-header + canonical-body + task-context.
|
|
54
|
+
- Claude Code phases other than PLAN: spawn `Agent` (`mode: "bypassPermissions"`); prompt = adapter-header + canonical-body + task-context. PLAN uses the native foreground call shape in PHASE 1.
|
|
55
55
|
- Codex CLI phases: shell out via `bash "$CODEX_MONITORED_PATH"` with the same compounded prompt. Each `codex exec` child is a new session/fresh context. The wrapper closes stdin and emits a heartbeat. No MCP. The wrapper call is foreground-blocking — never a background shell (`run_in_background`, `&`, `nohup`), never end your message while it runs: headless print-mode wind-down kills backgrounded children (0-byte delivery); the heartbeat is the observability channel.
|
|
56
56
|
- oh-my-pi phases: spawn the native `task` tool with a fresh `context` containing adapter-header + canonical-body + task-context. Capture the task result into `.devlyn/<phase>.stdout` and any tool error into `.devlyn/<phase>.stderr` before updating state. If the `task` tool is unavailable for an omp-routed phase, write the current phase verdict as `"BLOCKED"` and report `BLOCKED:fresh-context-unavailable`; do not fall back to same-context execution or a nested `omp -p` subprocess.
|
|
57
|
-
- Default engine: Claude for
|
|
57
|
+
- Default engine: Claude for IMPLEMENT / BUILD_GATE / CLEANUP when the orchestrator has Claude Code's native `Agent` primitive; otherwise the default route is the current CLI's fresh worker (`codex` child process for Codex CLI, native `task` for oh-my-pi). PLAN is orchestrator-fixed and never inherits `--engine`, an executor pin, or `state.engine`. Role resolution (`_shared/engine-preflight.md#role-resolution`): explicit `--engine` flag > machine-local `.devlyn/engines.json` `executor` pin > orchestrator-supported default above. Explicit routes and pins still fail closed when their required engine/spawn channel is unavailable. VERIFY/JUDGE selects the OTHER engine by default when available — OTHER = first adapter-valid, non-primary, available entry of `engines.json` `pair_judge_priority`, or the binary claude↔codex complement when unset.
|
|
58
58
|
- The `--engine` flag does not disable default pairing: the second judge uses the OTHER engine by default when available.
|
|
59
59
|
- Multi-LLM evolution: when a new model adapter ships in `_shared/adapters/`, that engine becomes selectable for the roles its adapter declares eligible without further skill changes (NORTH-STAR.md "Multi-LLM evolution direction").
|
|
60
60
|
</engine_routing>
|
|
@@ -101,15 +101,15 @@ For every direct complete→spawn handoff, call `state-phase-write.py ... --phas
|
|
|
101
101
|
|
|
102
102
|
Skip in verify-only mode. The heaviest phase by design — spec/criteria define non-negotiable invariants; plan formalizes how the implementation hits them.
|
|
103
103
|
|
|
104
|
-
Engine: Claude (PLAN-pair is **research-only** — iter-0033d/f/g; unblock conditions in PHASE 1 / iter-0033g §H; iter-0020 falsified Codex-BUILD/IMPLEMENT, NOT PLAN-pair).
|
|
104
|
+
Engine: PLAN is orchestrator-fixed and never inherits `--engine`, an executor pin, or `state.engine`; Claude Code uses a native Claude `Agent` worker, while Codex CLI / oh-my-pi use their own fresh worker per `_shared/engine-preflight.md` (PLAN-pair is **research-only** — iter-0033d/f/g; unblock conditions in PHASE 1 / iter-0033g §H; iter-0020 falsified Codex-BUILD/IMPLEMENT, NOT PLAN-pair). Before rendering, assign the current orchestrator's literal adapter name (`PLAN_ENGINE=claude`, `PLAN_ENGINE=codex`, or `PLAN_ENGINE=omp`) and assign `PLAN_MODEL=<exact model id passed to that fresh worker>`; replace the metavalue and fail closed if either assignment is empty. Write the data-only task context to `.devlyn/plan.task-context` with `PLAN_WORKING_DIR="$(pwd -P)"`: its first two lines are exactly `Working directory: $PLAN_WORKING_DIR` and `Plan output: $PLAN_WORKING_DIR/.devlyn/plan.md`, followed by the task data. Then render adapter + `references/phases/plan.md` + task context without byte normalization: `PLAN_PROMPT_SHA256="$(python3 "$DEVLYN_SHARED_DIR/phase-prompt-render.py" --adapter "$DEVLYN_SHARED_DIR/adapters/$PLAN_ENGINE.md" --canonical-body "$DEVLYN_SKILL_DIR/references/phases/plan.md" --task-context .devlyn/plan.task-context --output .devlyn/plan.prompt)"`. The renderer fails closed before writing `.devlyn/plan.prompt` if either fixed header field is missing, non-absolute, or mismatches the active repository paths. Open round 0 with `state-phase-write.py ... --phase plan spawn --round 0 --engine "$PLAN_ENGINE" --model "$PLAN_MODEL" --prompt-sha256 "$PLAN_PROMPT_SHA256"`. In Claude Code, invoke the native `Agent` with its `prompt` field set to the exact `.devlyn/plan.prompt` bytes, explicitly set `run_in_background: false`, and omit `mode`; this is a data field, so a path, `$(cat ...)`, or any variable-reference substitution is not the rendered prompt. If native input validation rejects this call, surface the failure without retrying a different `mode` or background shape and fail closed through the existing `BLOCKED:fresh-context-unavailable` path. The PLAN worker inherits the parent's permission context: the hands-free parent already uses native `bypassPermissions`, while a non-bypass parent's permission prompt remains the user's native boundary and is never silently escalated. Codex CLI and oh-my-pi deliver the same prompt through their unchanged routes. Product code records rendered intent; it never parses a transcript or intercepts, rewrites, wraps, or forces native delivery.
|
|
105
105
|
|
|
106
106
|
Subagent output (writes `.devlyn/plan.md`): file list to touch, risk list (out-of-scope expansions, ambiguous spec sections), acceptance restatement (what `## Verification` actually requires verbatim). For large work only, PLAN may add a fourth section `## Execution phases` under the conditions in `references/phases/plan.md`; a phase block missing a runnable `gate:` line disqualifies the whole section — treat the run as single-phase and log the disqualification.
|
|
107
107
|
|
|
108
|
-
State write: `
|
|
108
|
+
State write: each PLAN receipt carries spawn-known `round`, `started_at`, `triggered_by`, `engine`, `model_requested`, and `prompt_sha256`; completion enriches it with `completed_at`, `duration_ms`, `verdict`, and nullable `model_effective`.
|
|
109
109
|
|
|
110
110
|
After return:
|
|
111
111
|
1. If `.devlyn/plan.md` lists zero files → halt with verdict `BLOCKED:plan-empty`.
|
|
112
|
-
2. If risk list flags an out-of-scope expansion the user did not authorize → re-spawn
|
|
112
|
+
2. If risk list flags an out-of-scope expansion the user did not authorize → the sole legal corrective re-spawn is round 1: update the data-only task context, repeat render + digest, open PLAN with `--round 1 --triggered-by plan` and the actual engine/model/digest, then deliver the new `.devlyn/plan.prompt` using the same native call shape as round 0. A second failed plan halts; no further PLAN dispatch is authorized.
|
|
113
113
|
3. After any re-spawn above, if `state.risk_profile.risk_probes_enabled == true` and `state.risk_profile.risk_probes_explicit == false`, parse the `authorized_surface` array from the JSON block under `<!-- devlyn:authorized-surface -->`. For a well-formed string array, compute `probe_scale_small := len(authorized_surface) <= 2 AND no entry ends in "/**"`. If true, set `risk_probes_enabled = false`, leave `high_risk` unchanged, and append `auto-risk-probes demoted: plan surface small (<n> paths)` to `reasons` using the actual length. A missing or malformed block leaves probe state unchanged; BUILD_GATE owns its malformed-block failure.
|
|
114
114
|
|
|
115
115
|
## PHASE 1.5: RISK_PROBES
|
|
@@ -156,7 +156,8 @@ When visible text names exact keys, fields, row shapes, JSON objects, response
|
|
|
156
156
|
bodies, stdout/stderr objects, or exact error bodies, `shape_contract` must
|
|
157
157
|
include `uses_visible_input_key_names`, `asserts_visible_output_key_names`, and
|
|
158
158
|
`asserts_no_unexpected_output_keys`; exact JSON error objects/bodies must also
|
|
159
|
-
include `
|
|
159
|
+
include `visible_text_names_exact_json_error_object` and
|
|
160
|
+
`asserts_exact_error_object`. Cart/pricing success probes should use
|
|
160
161
|
`shape_contract` unless they satisfy the `ordering_inversion` markers. The probe
|
|
161
162
|
command must not reference external network URLs; use only worktree-local or
|
|
162
163
|
localhost resources.
|
|
@@ -20,7 +20,7 @@ Detect the project shape from files in `state.base_ref.sha`:
|
|
|
20
20
|
Run in this order; each emits findings into `.devlyn/build_gate.findings.jsonl`:
|
|
21
21
|
|
|
22
22
|
1. **Type check** (TypeScript / mypy / etc.). Each error → one finding, severity `HIGH`, rule `correctness.type-check`.
|
|
23
|
-
2. **Lint
|
|
23
|
+
2. **Lint**. Run a recognized configured language linter (for example, ESLint with a config, `ruff`, or `clippy`) and/or the exact `package.json` `scripts.lint` command. A declared specialized `lint:*` script (for example, `lint:json`) is not the general language lint gate on the strength of its declaration alone. If the spec's verification commands explicitly name such a specialized command, step 4 executes it and preserves its real failure severity. If no linter is applicable, log an explicit SKIP in `.devlyn/build_gate.log.md` and emit zero lint findings. Each error → finding, severity `MEDIUM`, rule `quality.lint`. Warnings stay LOW unless the spec elevates them.
|
|
24
24
|
3. **Test suite** (npm test / pytest / go test / cargo test). Each failing test → finding, severity `HIGH`, rule `correctness.test-failure`. Include the failing test's file:line and the assertion.
|
|
25
25
|
4. **Spec literal verification + risk probes**: `python3 "$DEVLYN_SHARED_DIR/spec-verify-check.py" --include-risk-probes`. The script self-stages from sibling `spec.expected.json` next to `state.source.spec_path`, or the legacy inline carrier when the sibling is absent; benchmark-prestaged `.devlyn/spec-verify.json` still wins. It appends `.devlyn/risk-probes.jsonl` when present, and requires that file when `state.risk_profile.risk_probes_enabled == true`. Malformed `state.risk_profile` is also CRITICAL because it can hide enabled risk probes. When probes are enabled or `.devlyn/risk-probes.jsonl` exists, missing or mismatched top-level `state.risk_probes_digest` emits `correctness.risk-probe-integrity` CRITICAL; the digest binds `.devlyn/risk-probes.jsonl` plus referenced `.devlyn/probes/` script bytes. Command or risk-probe mismatch → CRITICAL finding. Missing required risk probes, missing/malformed generated carrier, or malformed sibling expected file → `correctness.spec-verify-malformed` CRITICAL. The same invocation also enforces PLAN's declared `authorized_surface` (the json block under `.devlyn/plan.md`'s `<!-- devlyn:authorized-surface -->` section, located by that sentinel regardless of the heading's language) against this run's diff and untracked delta: any changed path or created-during-run untracked path outside the declared surface → `scope.out-of-scope-file` CRITICAL (`.devlyn/` exempt) — the only sanctioned fix is removing the file, never widening the declared surface, since that would let the same worker that leaked scope self-authorize the leak; missing sentinel, malformed `plan.md`/surface block, or missing `.devlyn/untracked.baseline` → `scope.authorized-surface-malformed` CRITICAL. This scope check runs only here (BUILD_GATE), not when VERIFY MECHANICAL reuses this script post-CLEANUP, because CLEANUP's own allowlist legitimately touches paths PLAN never declared.
|
|
26
26
|
5. **Browser** (only when diff touches `*.tsx`, `*.jsx`, `*.vue`, `*.svelte`, `page.*`, `layout.*`, `route.*`, `*.css`, `*.html`): start the dev server and run the repo's existing browser checks, or a minimal curl/HTML check when no browser test harness exists. Each failed check → finding, severity `HIGH`, rule `correctness.browser-flow-failed`.
|
|
@@ -35,7 +35,7 @@ Append all findings; do not stop on the first failure.
|
|
|
35
35
|
</output>
|
|
36
36
|
|
|
37
37
|
<quality_bar>
|
|
38
|
-
- Same commands every time. Configuration drift between this gate and CI is a defect; raise as a finding rather than soften this gate.
|
|
38
|
+
- Same commands every time. Configuration drift between this gate and CI is a defect; raise as a finding rather than soften this gate. A package-script declaration alone is not evidence that CI executes it.
|
|
39
39
|
- Forbidden-pattern check (regex against `git diff`) for `spec.expected.json.forbidden_patterns` runs as part of step 4. Disqualifier-severity matches → CRITICAL findings.
|
|
40
40
|
- Reporter artifacts the gate generates (Playwright traces, coverage HTML) belong in gitignored paths. If they leak into `git diff --stat`, flag as `scope.tooling-artifact-leak` MEDIUM and let the fix loop / cleanup handle removal.
|
|
41
41
|
</quality_bar>
|
|
@@ -33,7 +33,7 @@ Read `_shared/runtime-principles.md`. Codex-routed phases receive the inlined ex
|
|
|
33
33
|
|
|
34
34
|
- Subtractive-first: every accretion-shaped change is visible in the commit message or a flagged finding. Net-deletion is the default; pure-addition needs a citation.
|
|
35
35
|
- Goal-locked: implement only the listed Requirements. Adjacent code that "looks fixable" is drift unless the spec or plan listed it.
|
|
36
|
-
- No-workaround: no `any`, no `@ts-ignore`, no silent `catch`, no hardcoded
|
|
36
|
+
- No-workaround: no `any`, no `@ts-ignore`, no silent `catch`, no hardcoded fallback that hides a broken contract, no helper scripts that bypass root cause. Required unavailable engines stop with `BLOCKED:<engine>-unavailable`; they do not downgrade.
|
|
37
37
|
- Evidence: every claim cites file:line you opened. Hallucinated APIs are excluded.
|
|
38
38
|
</runtime_principles>
|
|
39
39
|
|
|
@@ -32,12 +32,8 @@ Report your verdict in this reply: `PASS` if plan is shippable; `BLOCKED` if spe
|
|
|
32
32
|
</quality_bar>
|
|
33
33
|
|
|
34
34
|
<runtime_principles>
|
|
35
|
-
Read `_shared/runtime-principles.md` (Subtractive-first / Goal-locked / No-workaround / Evidence). Codex-routed phases receive the contract excerpt inlined:
|
|
36
|
-
|
|
37
35
|
- Subtractive-first: prefer trimming an existing helper to introducing a new one. Pure-addition needs a cited prior failure mode or an explicit spec/user requirement.
|
|
38
36
|
- Goal-locked: refuse "while I'm here" cleanups, speculative robustness, mid-flight re-scoping. Single test before any deviation: "did the user ask for this OR does the stated goal strictly require it?"
|
|
39
|
-
- No-workaround: no `any`, no `@ts-ignore`, no silent `catch`, no hardcoded
|
|
37
|
+
- No-workaround: no `any`, no `@ts-ignore`, no silent `catch`, no hardcoded fallback that hides a broken contract, no helper scripts that bypass root cause.
|
|
40
38
|
- Evidence: every claim cites file:line opened at planning time. Vague claims excluded.
|
|
41
39
|
</runtime_principles>
|
|
42
|
-
|
|
43
|
-
The task is: [orchestrator pastes the task description and spec context here]
|
|
@@ -170,8 +170,8 @@ Rules:
|
|
|
170
170
|
shapes, JSON objects, response bodies, stdout/stderr objects, or exact error
|
|
171
171
|
bodies: `uses_visible_input_key_names`,
|
|
172
172
|
`asserts_visible_output_key_names`, `asserts_no_unexpected_output_keys`.
|
|
173
|
-
If it names an exact JSON error object/body, also include
|
|
174
|
-
`asserts_exact_error_object`.
|
|
173
|
+
If it names an exact JSON error object/body, also include both
|
|
174
|
+
`visible_text_names_exact_json_error_object` and `asserts_exact_error_object`.
|
|
175
175
|
Tags not listed here may use an empty evidence list or be omitted from
|
|
176
176
|
`tag_evidence`.
|
|
177
177
|
- `cmd` must not reference `BENCH_FIXTURE_DIR`, `verifiers/`, benchmark fixture
|
|
@@ -182,6 +182,8 @@ Rules:
|
|
|
182
182
|
replayed probe and obeys the same hidden-path/external-URL rules as `cmd`.
|
|
183
183
|
Solo-headroom hypothesis probes keep the hypothesis's backticked observable
|
|
184
184
|
command inline in `cmd` for mechanical validation.
|
|
185
|
+
- A probe may declare `timeout_sec` from 1 through 600 when the exercised
|
|
186
|
+
verification legitimately needs more than the default 60 seconds.
|
|
185
187
|
- Match the spec's visible input and output key names literally; do not invent
|
|
186
188
|
aliases such as `stock` for `lots`, `order_id` for `id`, or `warehouse_id`
|
|
187
189
|
for `warehouse`.
|
|
@@ -177,7 +177,11 @@ When eligible and the orchestrator spawns a second VERIFY agent with the OTHER e
|
|
|
177
177
|
A targeted probe must compare the full externally visible result
|
|
178
178
|
(stdout/stderr/exit and full parsed output object, including accepted/scheduled
|
|
179
179
|
rows, rejected rows, and remaining state when present), not just a single
|
|
180
|
-
property.
|
|
180
|
+
property. The mandatory dominance-loss anchor must run as a single command
|
|
181
|
+
with no shell chain operators. Chains used only to re-capture exit status are
|
|
182
|
+
forbidden because the tool result already carries it; this does not ban
|
|
183
|
+
compound shell generally or multi-step probes issued as separate tool calls.
|
|
184
|
+
When the spec names exact keys, row shapes, JSON object shape, or an
|
|
181
185
|
exact error body, compare parsed key sets/deep equality so aliased keys,
|
|
182
186
|
missing keys, and extra keys are verdict-binding failures. Use the spec's
|
|
183
187
|
visible input key names literally when constructing the probe input. For
|
|
@@ -57,7 +57,7 @@ Single authoritative verdict source for `/devlyn:resolve`. The orchestrator bran
|
|
|
57
57
|
- **risk_probes_digest** — top-level sha256 written by PHASE 1.5 after probe validation; BUILD_GATE/VERIFY replay semantics live in `phases/build-gate.md` step 4.
|
|
58
58
|
- **rounds.global** — incremented every fix-loop pass (BUILD_GATE → fix-loop, VERIFY → fix-loop, OR a phase-gate fix respawn inside phase-gated IMPLEMENT).
|
|
59
59
|
- **phases.implement.exec** — only on phase-gated large runs (plan.md has `## Execution phases` with >1 phase): `{ "total": N, "current": k, "statuses": ["PASS"|"FAIL"|null, ...], "commits": ["<sha>", ...] }`. Phase definitions live in plan.md (immutable contract); progress lives here (routing truth — never route on plan.md checkboxes). Absent on single-phase runs. `cumulative.patch` for such runs is `git diff <base_ref.sha>...HEAD`.
|
|
60
|
-
- **phases.probe_derive** — optional PHASE 1.5 entry when `--risk-probes` is enabled. Artifacts include `.devlyn/risk-probes.jsonl`. Probe failures later surface through BUILD_GATE/VERIFY as `correctness.risk-probe-failed
|
|
60
|
+
- **phases.probe_derive** — optional PHASE 1.5 entry when `--risk-probes` is enabled. Artifacts include `.devlyn/risk-probes.jsonl`. Probe failures later surface through BUILD_GATE/VERIFY as `correctness.risk-probe-failed`, or as `correctness.verification-timeout` when the probe exceeds its verification budget.
|
|
61
61
|
- **phases.surface_close** — generated trivial/medium one-shot: `pre_sha`, `input_patch_sha256`, assembled-byte `prompt_sha256`, `untracked_before`; spawn requires Claude plus a requested model, and `surface-check` gates PASS. `.devlyn/surface-close.output.json` retains the Claude JSON wrapper used for completion attestation while the round JSONL remains the execution-audit backstop. Automatic Claude unavailability alone records null verdict plus `skipped_reason: "auto_surface_close_claude_unavailable"`. After an adjudication-malformed reply only, successful rollback plus transcript write-audit records the two nonterminal fields `skipped_reason: "surface_close_rolled_back_adjudication_malformed"` and `continued_after_block: true`; neither creates a terminal floor. Other failures store bare `"BLOCKED"`. `durability[]` is append-only: `{round, origin_phase, receipt_path, receipt_sha256, restore_commit_sha}` per `.devlyn/closure-durability.round-<n>.json`.
|
|
62
62
|
- **bypasses** — array of phase names from `--bypass`. Valid: `"build-gate" | "cleanup"`. PLAN, IMPLEMENT, VERIFY are non-bypassable (orchestrator rejects at parse time).
|
|
63
63
|
- **implement_passed_sha** — captured at end of PHASE 2; null until then. Activates the post-implement invariant for SURFACE_CLOSE, CLEANUP, and VERIFY.
|
|
@@ -91,7 +91,7 @@ Each entry under `phases.<name>` (for `plan`, `probe_derive`, `implement`, `surf
|
|
|
91
91
|
`history` is absent until a phase is re-entered. Immediately before re-entry overwrites the live `started_at`, `verdict`, `completed_at`, and `duration_ms`, `state-phase-write.py` appends those prior values as one object to `history[]`, creating the append-only array on first re-entry.
|
|
92
92
|
|
|
93
93
|
- `verdict` — `"PASS" | "PASS_WITH_ISSUES" | "FAIL" | "NEEDS_WORK" | "BLOCKED"`. PHASE 6 (FINAL_REPORT) writes its own verdict per the terminal-verdict precedence.
|
|
94
|
-
- `model_requested` / `model_effective` — `model_requested` is supplied by the orchestrator through `--model`; `model_effective` is parsed only from completion-time engine session evidence. Codex evidence accepts the `model: <id>` exec header or, for `.jsonl`, rollout `turn_context.payload.model`; Claude JSON output accepts
|
|
94
|
+
- `model_requested` / `model_effective` — `model_requested` is supplied by the orchestrator through `--model`; `model_effective` is parsed only from completion-time engine session evidence. Codex evidence accepts the `model: <id>` exec header or, for `.jsonl`, rollout `turn_context.payload.model`; Claude JSON output accepts a singleton `modelUsage` key or, for multiple entries, the sole key whose `inputTokens`, `outputTokens`, `cacheReadInputTokens`, and `cacheCreationInputTokens` exactly match top-level `usage.input_tokens`, `usage.output_tokens`, `usage.cache_read_input_tokens`, and `usage.cache_creation_input_tokens`. Missing or malformed counters, zero or multiple matches, and requested/effective mismatch fail closed: the phase verdict becomes `BLOCKED` and the completion command fails. For mutation phases, omission of `--engine-session-log` also fails closed when `.devlyn/<artifact-phase>.worker-session.<round>.jsonl` exists; the error names that retained file. Otherwise omission records `model_effective: null` and attests that the engine class exposed no session log for that invocation.
|
|
95
95
|
- `triggered_by` — null on first run; one of `"build_gate" | "verify"` when the phase is a fix-loop respawn.
|
|
96
96
|
- `pre_sha` — captured before SURFACE_CLOSE or CLEANUP. It bounds that phase's post-spawn diff and rollback.
|
|
97
97
|
- `post_sha` — captured when SURFACE_CLOSE or CLEANUP completes; finish-gate subtracts CLEANUP's `pre_sha..post_sha` window.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "devlyn-cli",
|
|
3
|
-
"version": "2.10.
|
|
3
|
+
"version": "2.10.1",
|
|
4
4
|
"description": "Loop & harness engineering for AI coding agents (Claude Code, Codex, and any AGENTS.md-driven agent) — take intent to shipped, engineer-quality software hands-free with phase-gated pipelines and agent orchestration",
|
|
5
5
|
"homepage": "https://github.com/fysoul17/devlyn-cli#readme",
|
|
6
6
|
"bin": {
|
package/scripts/lint-fixtures.sh
CHANGED
|
@@ -161,7 +161,7 @@ def fallback_validate():
|
|
|
161
161
|
if not isinstance(command, dict):
|
|
162
162
|
errors.append(f"verification_commands[{idx}] must be an object")
|
|
163
163
|
continue
|
|
164
|
-
unknown_command = sorted(set(command) - {"cmd", "exit_code", "stdout_contains", "stdout_not_contains", "contract_refs"})
|
|
164
|
+
unknown_command = sorted(set(command) - {"cmd", "exit_code", "timeout_sec", "stdout_contains", "stdout_not_contains", "contract_refs"})
|
|
165
165
|
if unknown_command:
|
|
166
166
|
errors.append(f"verification_commands[{idx}] has unknown key(s): {', '.join(unknown_command)}")
|
|
167
167
|
if not isinstance(command.get("cmd"), str) or not command.get("cmd"):
|
|
@@ -169,6 +169,9 @@ def fallback_validate():
|
|
|
169
169
|
exit_code = command.get("exit_code", 0)
|
|
170
170
|
if isinstance(exit_code, bool) or not isinstance(exit_code, int):
|
|
171
171
|
errors.append(f"verification_commands[{idx}].exit_code must be an integer")
|
|
172
|
+
timeout_sec = command.get("timeout_sec", 60)
|
|
173
|
+
if isinstance(timeout_sec, bool) or not isinstance(timeout_sec, int) or not 1 <= timeout_sec <= 600:
|
|
174
|
+
errors.append(f"verification_commands[{idx}].timeout_sec must be an integer from 1 to 600")
|
|
172
175
|
for key in ("stdout_contains", "stdout_not_contains", "contract_refs"):
|
|
173
176
|
if key in command and not is_string_list(command[key]):
|
|
174
177
|
errors.append(f"verification_commands[{idx}].{key} must be an array of non-empty strings")
|