devlyn-cli 2.6.1 → 2.8.0

package/scripts/lint-skills.sh

@@ -59,6 +59,7 @@ critical_path_files=$(cat <<'EOF'
 _shared/spec-verify-check.py
 _shared/collect-codex-findings.py
 _shared/verify-merge-findings.py
+_shared/state-phase-write.py
 devlyn:ideate/SKILL.md
 devlyn:ideate/references/spec-template.md
 devlyn:ideate/references/elicitation.md
@@ -77,6 +78,7 @@ _shared/expected.schema.json
 _shared/adapters/README.md
 _shared/adapters/claude.md
 _shared/adapters/codex.md
+_shared/adapters/omp.md
 _shared/codex-config.md
 _shared/codex-monitored.sh
 _shared/engine-preflight.md
@@ -196,10 +198,11 @@ else
 fi
 
 # ---------------------------------------------------------------------------
-# 4. No stale Opus 4.6 benchmark references (should be 4.7 after P1).
+# 4. No model-pinned Claude references, any generation. Engine-scope docs say
+#    "Claude"; exact model ids belong in run metadata only.
 # ---------------------------------------------------------------------------
-section "Check 4: No stale 'Claude Opus 4.6' in routing table"
-offenders=$(grep -RIln 'Claude Opus 4\.6' \
+section "Check 4: No model-pinned Claude references"
+offenders=$(grep -RIlnE 'Claude (Opus|Sonnet|Haiku|Fable)|claude-(opus|sonnet|haiku|fable)-[0-9]' \
   config/skills 2>/dev/null \
   | grep -v 'config/skills/roadmap-archival-workspace/' \
   | grep -v 'config/skills/devlyn:auto-resolve-workspace/' \
@@ -207,7 +210,7 @@ offenders=$(grep -RIln 'Claude Opus 4\.6' \
   | grep -v 'config/skills/preflight-workspace/' \
   || true)
 if [ -z "$offenders" ]; then
-  ok "routing table on Opus 4.7"
+  ok "no model-pinned Claude references"
 else
   while IFS= read -r f; do bad "$f"; done <<< "$offenders"
 fi
@@ -242,7 +245,7 @@ if [ ! -e "optional-skills/devlyn:design-ui" ]; then
 else
   bad "devlyn:design-ui must not be installed as an optional addon"
 fi
-if grep -Fq "const DEVLYN_CORE_SKILLS = ['devlyn:resolve', 'devlyn:ideate', 'devlyn:design-ui', '_shared'];" bin/devlyn.js \
+if grep -Fq "const DEVLYN_CORE_SKILLS = ['devlyn:resolve', 'devlyn:ideate', 'devlyn:design-ui', 'devlyn:engines', 'devlyn:queue', '_shared'];" bin/devlyn.js \
    && grep -Fq "skillsToInstall: DEVLYN_CORE_SKILLS" bin/devlyn.js; then
   ok "Codex install includes devlyn:design-ui (via shared DEVLYN_CORE_SKILLS bundle)"
 else
@@ -393,7 +396,7 @@ if ! grep -Fq 'SAFE_RUN_ID_RE' config/skills/_shared/archive_run.py \
   || ! grep -Fq 'invalid JSON numeric constant: NaN' config/skills/_shared/archive_run.py \
   || ! grep -Fq '"verify.pair.findings.jsonl"' config/skills/_shared/archive_run.py \
   || ! grep -Fq '"verify-merge.summary.json"' config/skills/_shared/archive_run.py \
-  || ! grep -Fq '"codex-judge.*"' config/skills/_shared/archive_run.py; then
+  || ! grep -Fq '"*-judge.*"' config/skills/_shared/archive_run.py; then
   bad "archive_run.py must safely archive pair/risk-probe evidence and reject unsafe run ids"
 fi
 
@@ -600,9 +603,16 @@ then
 else
   bad "solo-headroom command detection constants must stay in parity"
 fi
-if ! grep -Fq 'required.add("rollback_state")' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'rollback verification text did not require rollback_state probe tag' config/skills/_shared/spec-verify-check.py; then
-  bad "spec-verify-check.py must require rollback_state risk probes for rollback/all-or-nothing verification text"
+if ! grep -Fq 'def resolve_required_risk_probe_requirements' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'def validate_required_risk_probe_requirement' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'missing required probe(s)' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'risk-probes.jsonl missing a declared required_risk_probe_requirements' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'risk-probes.jsonl covering a declared required_risk_probe_requirements' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'required_risk_probe_requirements with an unknown tag was accepted' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'required_risk_probe_requirements.derived_from not present in the' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'required_risk_probe_requirements' config/skills/_shared/expected.schema.json \
+  || ! grep -Fq 'required_risk_probe_requirements' config/skills/devlyn:ideate/references/spec-template.md; then
+  bad "spec-verify-check.py must enforce declared required_risk_probe_requirements (iter-0049 language-neutral F3 replacement)"
 fi
 if ! grep -Fq 'spec.expected.json top-level array produced a traceback' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'invalid spec.expected.json produced a traceback' config/skills/_shared/spec-verify-check.py \
@@ -615,10 +625,6 @@ if ! grep -Fq 'spec.expected.json top-level array produced a traceback' config/s
   || ! grep -Fq 'has invalid JSON' config/skills/_shared/spec-verify-check.py; then
   bad "spec-verify-check.py self-test must fail malformed spec.expected.json cleanly without traceback"
 fi
-if ! grep -Fq 'required.add("error_contract")' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'error_contract without exit-code evidence was accepted' config/skills/_shared/spec-verify-check.py; then
-  bad "spec-verify-check.py must require error_contract risk probes for invalid/stderr/JSON-error/exit-2 verification text"
-fi
 if ! grep -Fq '"asserts_named_stream_output"' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq '"asserts_error_payload_or_stderr"' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq '"asserts_nonzero_or_exit_2"' config/skills/_shared/spec-verify-check.py \
@@ -631,7 +637,6 @@ if ! grep -Fq '"asserts_named_stream_output"' config/skills/_shared/spec-verify-
 fi
 if ! grep -Fq '"http_error_contract"' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'asserts_http_error_status' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'http error text did not require http_error_contract tag' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'http_error_contract without payload evidence was accepted' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'exact error body shape_contract without exact object evidence was accepted' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'exact error body shape_contract with exact object evidence was rejected' config/skills/_shared/spec-verify-check.py \
@@ -642,55 +647,31 @@ fi
 if ! grep -Fq '"uses_visible_input_key_names"' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq '"asserts_visible_output_key_names"' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq '"asserts_no_unexpected_output_keys"' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'JSON error object text did not require shape_contract tag' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'shape_contract co-occurring with error_contract without' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'JSON error object shape_contract with exact object evidence was rejected' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'INLINE_JSON_OBJECT_RE' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'inline JSON object text did not require shape_contract tag' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'inline JSON object shape_contract with key evidence was rejected' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'inline JSON error text did not require shape_contract tag' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'inline JSON error shape_contract with exact object evidence was rejected' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'shape_contract without exact key evidence was accepted' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'shape_contract without any evidence was accepted' config/skills/_shared/spec-verify-check.py \
+  || ! grep -Fq 'shape_contract with exact key evidence was rejected' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq '`shape_contract` when the visible text names exact keys' config/skills/devlyn:resolve/references/phases/probe-derive.md \
   || ! grep -Fq '`shape_contract` must' config/skills/devlyn:resolve/SKILL.md; then
-  bad "risk-probe shape contracts must require exact visible input/output key evidence when visible text names shape"
-fi
-if ! grep -Fq "forbidden[ -]+window" config/skills/_shared/spec-verify-check.py \
-  || grep -Fq "r'blocked|overlap|forbidden|window'" config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'generic forbidden-pattern verification text incorrectly required boundary_overlap' config/skills/_shared/spec-verify-check.py; then
-  bad "risk-probe boundary_overlap must trigger for forbidden windows / blocked overlap, not generic forbidden pattern text"
-fi
-if ! grep -Fq '(?:stock|inventory|balance|availability).{0,80}(?:later|remaining|after failures)' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'stock validation error text incorrectly required prior_consumption' config/skills/_shared/spec-verify-check.py; then
-  bad "risk-probe prior_consumption must trigger on later/remaining state consumption, not plain stock validation errors"
+  bad "risk-probe shape contracts must require exact visible input/output key evidence unconditionally, and asserts_exact_error_object when co-occurring with error_contract/http_error_contract"
 fi
 if ! grep -Fq '"auth_signature_contract"' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq '"idempotency_replay"' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'asserts_signature_over_exact_bytes' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'duplicate_id_rejected_regardless_of_body' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'webhook signature/replay text did not require auth/idempotency probe tags' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq '`auth_signature_contract`: `asserts_signature_over_exact_bytes`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
   || ! grep -Fq '`idempotency_replay`: `first_delivery_then_duplicate`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
   || ! grep -Fq '`auth_signature_contract` must include `asserts_signature_over_exact_bytes`' config/skills/devlyn:resolve/SKILL.md; then
   bad "risk-probe webhook/signature/replay contracts must require concrete auth_signature_contract and idempotency_replay tags"
 fi
-if ! grep -Fq 'signing|signed' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'same.{0,40}`?id`?' config/skills/_shared/spec-verify-check.py; then
-  bad "risk-probe webhook signature/replay trigger must catch signing/signed and same accepted id wording"
-fi
-if ! grep -Fq 'duplicate[ -]+(?:delivery|event|id)' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'duplicate SKU verification text incorrectly required idempotency_replay' config/skills/_shared/spec-verify-check.py; then
-  bad "risk-probe idempotency_replay must trigger on duplicate delivery/event/id, not duplicate SKU aggregation"
-fi
 if ! grep -Fq '"concurrent_state_consistency"' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'overlapping_mutations_exercised' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'concurrent state text did not require concurrent_state_consistency tag' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq '`concurrent_state_consistency`: `overlapping_mutations_exercised`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
   || ! grep -Fq '`concurrent_state_consistency` must' config/skills/devlyn:resolve/SKILL.md; then
   bad "risk-probe concurrent state contracts must require concrete concurrent_state_consistency markers"
 fi
 if ! grep -Fq '"atomic_batch_state"' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'mixed_valid_invalid_batch' config/skills/_shared/spec-verify-check.py \
-  || ! grep -Fq 'atomic batch text did not require atomic_batch_state tag' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq 'atomic_batch_state without success-order evidence was accepted' config/skills/_shared/spec-verify-check.py \
   || ! grep -Fq '`atomic_batch_state`: `mixed_valid_invalid_batch`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
   || ! grep -Fq '`atomic_batch_state` must include `mixed_valid_invalid_batch`' config/skills/devlyn:resolve/SKILL.md; then
@@ -1097,6 +1078,20 @@ if ! grep -Fq 'If the spec includes a solo-headroom hypothesis, one of the two t
 else
   ok "resolve pair-JUDGE prompts prioritize solo-headroom hypothesis"
 fi
+if ! grep -Fq 'names the primary judge; if pair-mode triggers, the first available OTHER' config/skills/devlyn:resolve/references/phases/verify.md \
+  || ! grep -Fq 'names the primary judge; if pair-mode triggers, the first available OTHER' .claude/skills/devlyn:resolve/references/phases/verify.md \
+  || ! grep -Fq 'names the primary judge; if pair-mode triggers, the first available OTHER' .agents/skills/devlyn:resolve/references/phases/verify.md \
+  || ! grep -Fq 'do not run the OTHER engine as the' config/skills/devlyn:resolve/SKILL.md \
+  || ! grep -Fq 'do not run the OTHER engine as the' .claude/skills/devlyn:resolve/SKILL.md \
+  || ! grep -Fq 'do not run the OTHER engine as the' .agents/skills/devlyn:resolve/SKILL.md \
+  || ! grep -Fq '_shared/adapters/claude.md` `## Invocation' config/skills/devlyn:resolve/references/phases/verify.md \
+  || ! grep -Fq '_shared/adapters/claude.md` `## Invocation' config/skills/devlyn:resolve/SKILL.md \
+  || ! grep -Fq '## Invocation' config/skills/_shared/adapters/claude.md \
+  || ! grep -Fq '.devlyn/claude-judge.stdout' config/skills/_shared/adapters/claude.md; then
+  bad "engine-neutral pair-judge dual declaration (iter-0060) out of sync"
+else
+  ok "engine-neutral pair-judge dual declaration (iter-0060) in sync"
+fi
 if ! grep -Fq 'current free-form `state.complexity` is `"large"`' config/skills/devlyn:resolve/SKILL.md \
   || ! grep -Fq 'current free-form `state.complexity` is `"large"`' .claude/skills/devlyn:resolve/SKILL.md \
   || ! grep -Fq 'current free-form `state.complexity` is `"large"`' .agents/skills/devlyn:resolve/SKILL.md \
@@ -1351,7 +1346,7 @@ for file in \
   .agents/skills/devlyn:resolve/references/state-schema.md
 do
   if ! grep -Fq 'MECHANICAL and the primary JUDGE have no verdict-binding blockers' "$file" \
-    || ! grep -Fq 'may set only `user_no_pair`, `mechanical_blocker`, `primary_judge_blocker`, or null' "$file" \
+    || ! grep -Fq 'may set only `user_no_pair`, `mechanical_blocker`, `primary_judge_blocker`, `auto_pair_other_engine_unavailable`, or null' "$file" \
     || ! grep -Fq '`risk_profile` must remain an object with boolean' "$file" \
     || ! grep -Fq 'state implies a pair decision is required but `pair_trigger` is missing' "$file"; then
     bad "$file — state schema must document pair_trigger blocker and missing-trigger enforcement"
@@ -1457,7 +1452,7 @@ isolation_missing=0
 for needle in \
   'CODEX_MONITORED_ISOLATED=1 bash "$CODEX_MONITORED_PATH"' \
   'CODEX_MONITORED_ISOLATED=1` and `-c model_reasoning_effort=medium' \
-  'CODEX_MONITORED_ISOLATED=1 bash .claude/skills/_shared/codex-monitored.sh'
+  'CODEX_MONITORED_PATH="$DEVLYN_SHARED_DIR/codex-monitored.sh"'
 do
   if ! grep -RInF "$needle" config/skills >/dev/null 2>&1; then
     bad "missing isolated Codex invocation contract: $needle"
@@ -1543,6 +1538,19 @@ if [ $isolation_missing -eq 0 ]; then
   ok "bounded Codex probe/judge calls require isolated wrapper mode"
 fi
 
+section "Check 10a1: Shared helper paths resolve from the invoked skill"
+shared_path_offenders=$(grep -RInF '.claude/skills/_shared' config/skills 2>/dev/null \
+  | grep -v 'roadmap-archival-workspace/' \
+  | grep -v 'devlyn:auto-resolve-workspace/' \
+  | grep -v 'devlyn:ideate-workspace/' \
+  | grep -v 'preflight-workspace/' \
+  || true)
+if [ -z "$shared_path_offenders" ]; then
+  ok "no project-relative .claude/skills/_shared paths in managed skills"
+else
+  while IFS= read -r f; do bad "$f"; done <<< "$shared_path_offenders"
+fi
+
 # ---------------------------------------------------------------------------
 # 10b. Shared routing docs must describe the current 2-skill surface.
 #      A stale auto-resolve/preflight/ideate-CHALLENGE reference can misroute
@@ -1566,7 +1574,8 @@ fi
 section "Check 10c: User-facing current docs avoid retired skill surfaces"
 offenders=$(
   {
-    grep -nE '/devlyn:auto-resolve|ideate CHALLENGE|--with-codex|Quick Start pointing to ideate → auto-resolve → preflight|auto-resolve'\''s build agent|Core pipeline skills \(`ideate`, `auto-resolve`, `preflight`\)' README.md 2>/dev/null || true
+    sed '/<!-- legacy-surface-map:begin/,/<!-- legacy-surface-map:end/s/.*//' README.md 2>/dev/null \
+      | grep -nE '/devlyn:auto-resolve|ideate CHALLENGE|--with-codex|Quick Start pointing to ideate → auto-resolve → preflight|auto-resolve'\''s build agent|Core pipeline skills \(`ideate`, `auto-resolve`, `preflight`\)' || true
     grep -nE '"description": .*auto-resolve|"description": .*preflight' package.json 2>/dev/null || true
     grep -nE 'so auto-resolve doesn'\''t prompt' bin/devlyn.js 2>/dev/null || true
     grep -nE 'devlyn:auto-resolve|phase-1-build\.md|phase-2-evaluate\.md|phase-3-critic\.md' scripts/static-ab.sh 2>/dev/null || true
@@ -3638,7 +3647,7 @@ if ! grep -Fq 'high-risk fixture must include a resolve risk-trigger term' scrip
   || ! grep -Fq 'DEVLYN_FIXTURES_DIR="$FIXTURES_DIR" bash "$ROOT/scripts/lint-fixtures.sh"' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
   || ! grep -Fq 'DEVLYN_LINT_FIXTURES_NO_JSONSCHEMA=1' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
 	  || ! grep -Fq 'spec-verify-check --check-expected failed' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
-	  || ! grep -Fq 'unless sibling spec.md declares all Requirements are pure-design' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
+	  || ! grep -Fq 'unless spec.expected.json declares "pure_design": true' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
 	  || ! grep -Fq 'verification_commands must be an array' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
 	  || ! grep -Fq 'expected.json must be an object' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
 	  || ! grep -Fq 'hidden oracle missing contract_refs' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \

package/agents-config/evaluator.md

@@ -1,64 +0,0 @@
-# Evaluator Agent
-
-You are a code quality evaluator. Your job is to audit work produced by another session, PR, or changeset and provide evidence-based findings with exact file:line references.
-
-## Before You Start
-
-1. **Check for done criteria**: Read `.devlyn/runs/<run_id>/spec/expected.json` if a structured done-criteria source exists for the current run. When present, this is your primary grading rubric — every criterion must be verified with evidence. When absent, fall back to the checklists below.
-
-## Calibration
-
-You will be too lenient by default. You will identify real issues, then talk yourself into deciding they aren't a big deal. Fight this tendency.
-
-**Rule**: When in doubt, score DOWN. A false negative ships broken code. A false positive costs minutes of review. The cost is asymmetric.
-
-- A catch block that logs but doesn't surface error to user = HIGH (not MEDIUM). Logging is not error handling.
-- A `let` that could be `const` = LOW note only. Linters catch this.
-- "The error handling is generally quite good" = WRONG. Count the instances. Name the files.
-
-## Evaluation Process
-
-1. **Discover scope**: Read the changeset (git diff, PR diff, or specified files)
-2. **Assess correctness**: Find bugs, logic errors, silent failures, missing error handling
-3. **Check architecture**: Verify patterns match existing codebase, no type duplication, proper wiring
-4. **Verify spec compliance**: If a spec exists (HANDOFF.md, RFC, issue, done-criteria.md), compare requirements vs implementation
-5. **Check error handling**: Every async operation needs loading, error, and empty states in UI. No silent catches.
-6. **Review API contracts**: New endpoints must follow existing conventions for naming, validation, error envelopes
-7. **Assess test coverage**: New modules need tests. Run the test suite and report results.
-8. **Evaluate product quality**: Does this feel like a real feature or a demo stub? Are workflows complete end-to-end? Is the UI coherent?
-
-## Rules
-
-- Every finding must have a file:line reference. No guesswork.
-- Classify by severity: CRITICAL (must fix), HIGH (should fix), MEDIUM (fix or justify), LOW (note)
-- Call out what's done well, not just problems
-- Look for cross-cutting patterns (e.g., same mistake repeated in multiple files)
-
-## Output
-
-Write findings to `.devlyn/runs/<run_id>/eval-findings.md` for downstream consumption:
-
-```markdown
-# Evaluation Findings
-
-## Verdict: [PASS / PASS WITH ISSUES / NEEDS WORK / BLOCKED]
-
-## Done Criteria Results (if a done-criteria source existed)
-- [x] [criterion] — VERIFIED: [evidence]
-- [ ] [criterion] — FAILED: [what's wrong, file:line]
-
-## Findings Requiring Action
-### CRITICAL
-- `file:line` — [description] — Fix: [suggested approach]
-
-### HIGH
-- `file:line` — [description] — Fix: [suggested approach]
-
-## Cross-Cutting Patterns
-- [pattern description]
-
-## What's Good
-- [positive observations]
-```
-
-Do NOT delete the run-directory artifacts (`spec/expected.json`, `eval-findings.md`) — the orchestrator or user is responsible for cleanup.