devlyn-cli 2.10.1 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/AGENTS.md +4 -9
  2. package/CLAUDE.md +11 -15
  3. package/README.md +31 -7
  4. package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +25 -0
  5. package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +10 -0
  6. package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +18 -0
  7. package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +9 -0
  8. package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +18 -0
  9. package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +9 -0
  10. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +28 -0
  11. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +28 -0
  12. package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +30 -0
  13. package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +28 -0
  14. package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +15 -0
  15. package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +28 -0
  16. package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +15 -0
  17. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +32 -0
  18. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +44 -0
  19. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +16 -0
  20. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +18 -0
  21. package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +9 -0
  22. package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +28 -0
  23. package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +15 -0
  24. package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +28 -0
  25. package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +15 -0
  26. package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +28 -0
  27. package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +15 -0
  28. package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +28 -0
  29. package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +15 -0
  30. package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +28 -0
  31. package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +15 -0
  32. package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +28 -0
  33. package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +15 -0
  34. package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +28 -0
  35. package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +15 -0
  36. package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +28 -0
  37. package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +15 -0
  38. package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +28 -0
  39. package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +15 -0
  40. package/bin/devlyn.js +6 -1
  41. package/config/skills/_shared/.ruff.toml +2 -0
  42. package/config/skills/_shared/adapters/claude.md +9 -1
  43. package/config/skills/_shared/adapters/codex.md +5 -0
  44. package/config/skills/_shared/adapters/grok.md +4 -2
  45. package/config/skills/_shared/archive_run.py +738 -56
  46. package/config/skills/_shared/codex-config.md +19 -4
  47. package/config/skills/_shared/codex-monitored.sh +81 -12
  48. package/config/skills/_shared/collect-codex-findings.py +311 -128
  49. package/config/skills/_shared/engine-preflight.md +8 -7
  50. package/config/skills/_shared/expected.schema.json +61 -0
  51. package/config/skills/_shared/finish-gate.py +12 -11
  52. package/config/skills/_shared/grok-anchor-guard.py +18 -1
  53. package/config/skills/_shared/invocation-receipt.py +1041 -0
  54. package/config/skills/_shared/judge-output-parser.py +200 -0
  55. package/config/skills/_shared/judge-role-evidence.py +273 -0
  56. package/config/skills/_shared/process-evidence.py +909 -0
  57. package/config/skills/_shared/resolve-bootstrap.py +776 -84
  58. package/config/skills/_shared/resolve-stop-hook.py +33 -1
  59. package/config/skills/_shared/role-config.py +552 -0
  60. package/config/skills/_shared/run-bounded.py +1 -1
  61. package/config/skills/_shared/spec-verify-check.py +749 -181
  62. package/config/skills/_shared/state-phase-write.py +1838 -58
  63. package/config/skills/_shared/terminal-claim-check.py +15 -0
  64. package/config/skills/_shared/verify-merge-findings.py +958 -51
  65. package/config/skills/devlyn:engines/SKILL.md +8 -16
  66. package/config/skills/devlyn:ideate/references/spec-template.md +1 -0
  67. package/config/skills/devlyn:queue/SKILL.md +7 -4
  68. package/config/skills/devlyn:resolve/SKILL.md +91 -33
  69. package/config/skills/devlyn:resolve/references/outer-loop.md +15 -0
  70. package/config/skills/devlyn:resolve/references/phases/build-gate.md +31 -2
  71. package/config/skills/devlyn:resolve/references/phases/implement.md +3 -2
  72. package/config/skills/devlyn:resolve/references/phases/verify.md +120 -84
  73. package/config/skills/devlyn:resolve/references/state-schema.md +22 -8
  74. package/package.json +5 -1
  75. package/scripts/lint-skills.sh +314 -285
@@ -31,10 +31,11 @@ Default mode (BUILD_GATE invocation, no args):
31
31
  mis-config). Real-user mode silent no-op + drops any stale
32
32
  pre-staged file (preserves iter-0019.6 backward compat for
33
33
  handwritten specs without the carrier).
34
- - For each verification_commands entry, runs the command in the work-dir,
35
- captures combined stdout+stderr, and asserts exit_code matches +
36
- stdout_contains all required literals + stdout_not_contains none of the
37
- forbidden literals. Mirrors run-fixture.sh's post-run verifier semantics.
34
+ - For each verification_commands entry and included risk probe, routes the
35
+ command through the sibling `process-evidence.py` runner. Raw stdout/stderr,
36
+ execution outcome, classification, and byte digests are written under the
37
+ authenticated run/phase/round manifest; expectations retain
38
+ run-fixture.sh's combined-stream matching semantics.
38
39
 
39
40
  Check mode (`--check <markdown_path>`):
40
41
  - Used by /devlyn:ideate after writing each item spec to validate that the
@@ -60,6 +61,8 @@ Output routing:
60
61
  `SPEC_VERIFY_FINDINGS_FILE=verify-mechanical.findings.jsonl`, and
61
62
  `SPEC_VERIFY_FINDING_PREFIX=VERIFY-MECH` so `verify-merge-findings.py` consumes
62
63
  deterministic blockers directly.
64
+ - `.devlyn/spec-verify.results.json` points each result at its sealed raw
65
+ streams and includes the validated process-evidence carrier.
63
66
 
64
67
  Why: iter-0018.5's prompt-only contract enforcement was empirically dead
65
68
  (F9 verify=0.4 across all engines in iter-0019). Same lesson as iter-0008
@@ -80,11 +83,13 @@ Exit codes:
80
83
  `SPEC_VERIFY_FINDINGS_FILE` (for example, VERIFY uses
81
84
  `.devlyn/verify-mechanical.findings.jsonl`).
82
85
  - 2: invocation error (unreadable spec-verify.json, missing markdown in
83
- --check mode, etc.)
86
+ --check mode, etc.) or explicit evidence-backed
87
+ `BLOCKED:build-env-underprovisioned` capability denial.
84
88
  """
85
89
 
86
90
  from __future__ import annotations
87
91
 
92
+ import importlib.util
88
93
  import json
89
94
  import hashlib
90
95
  import os
@@ -99,8 +104,21 @@ def reject_json_constant(token: str) -> None:
99
104
  raise ValueError(f"invalid JSON numeric constant: {token}")
100
105
 
101
106
 
107
+ def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict:
108
+ result = {}
109
+ for key, value in pairs:
110
+ if key in result:
111
+ raise ValueError(f"duplicate JSON key: {key}")
112
+ result[key] = value
113
+ return result
114
+
115
+
102
116
  def loads_strict_json(text: str):
103
- return json.loads(text, parse_constant=reject_json_constant)
117
+ return json.loads(
118
+ text,
119
+ parse_constant=reject_json_constant,
120
+ object_pairs_hook=reject_duplicate_keys,
121
+ )
104
122
 
105
123
 
106
124
  def output_phase() -> str:
@@ -115,6 +133,95 @@ def output_finding_prefix() -> str:
115
133
  return os.environ.get("SPEC_VERIFY_FINDING_PREFIX", "BGATE")
116
134
 
117
135
 
136
+ _PROCESS_EVIDENCE_MODULE = None
137
+ RUNNER_ENV_KEYS = (
138
+ "SPEC_VERIFY_PHASE",
139
+ "SPEC_VERIFY_FINDINGS_FILE",
140
+ "SPEC_VERIFY_FINDING_PREFIX",
141
+ )
142
+
143
+
144
+ def process_evidence_module():
145
+ global _PROCESS_EVIDENCE_MODULE
146
+ if _PROCESS_EVIDENCE_MODULE is None:
147
+ module_path = Path(__file__).with_name("process-evidence.py")
148
+ spec = importlib.util.spec_from_file_location("devlyn_process_evidence", module_path)
149
+ if spec is None or spec.loader is None:
150
+ raise RuntimeError(f"cannot load process evidence runner: {module_path}")
151
+ module = importlib.util.module_from_spec(spec)
152
+ previous_bytecode_setting = sys.dont_write_bytecode
153
+ sys.dont_write_bytecode = True
154
+ try:
155
+ spec.loader.exec_module(module)
156
+ finally:
157
+ sys.dont_write_bytecode = previous_bytecode_setting
158
+ _PROCESS_EVIDENCE_MODULE = module
159
+ return _PROCESS_EVIDENCE_MODULE
160
+
161
+
162
+ def mechanical_evidence_identity(state: dict, runner) -> tuple[str, str, int, str]:
163
+ phase = "verify" if output_phase() == "verify_mechanical" else "build_gate"
164
+ run_id = state.get("run_id")
165
+ phase_state = (state.get("phases") or {}).get(phase)
166
+ round_ = phase_state.get("round") if isinstance(phase_state, dict) else None
167
+ invalid_identity = (
168
+ not isinstance(run_id, str)
169
+ or runner.ID_RE.fullmatch(run_id) is None
170
+ or isinstance(round_, bool)
171
+ or not isinstance(round_, int)
172
+ or round_ < 0
173
+ )
174
+ if invalid_identity and state.get("version") == "3.0":
175
+ raise runner.EvidenceError(
176
+ f"schema-v3 MECHANICAL evidence identity is invalid for phase {phase}"
177
+ )
178
+ if invalid_identity:
179
+ # Standalone/benchmark invocations predate pipeline run identity. Keep
180
+ # them evidence-backed without letting one process reuse another's
181
+ # manifest. Real pipeline runs always take the authenticated branch.
182
+ run_id = f"standalone-{os.getpid()}"
183
+ round_ = 0
184
+ identity = {"run_id": run_id, "phases": {phase: {"round": round_}}}
185
+ return phase, run_id, round_, runner.manifest_relative_path(identity, phase)
186
+
187
+
188
+ def mechanical_obligation(vc: dict, idx: int, phase: str) -> dict:
189
+ evidence_id = (
190
+ f"risk-probe-{idx + 1:04d}"
191
+ if vc.get("_risk_probe")
192
+ else f"verification-command-{idx + 1:04d}"
193
+ )
194
+ return {
195
+ "id": evidence_id,
196
+ "phase": phase,
197
+ "cmd": vc["cmd"],
198
+ "exit_code": vc.get("exit_code", 0),
199
+ "timeout_sec": verification_timeout_sec(vc),
200
+ "stdout_contains": vc.get("stdout_contains", []) or [],
201
+ "stdout_not_contains": vc.get("stdout_not_contains", []) or [],
202
+ }
203
+
204
+
205
+ def capture_mechanical_command(
206
+ runner, work: Path, manifest_path: Path, run_id: str, phase: str,
207
+ round_: int, obligation: dict,
208
+ ) -> dict:
209
+ saved = {key: os.environ.get(key) for key in (*RUNNER_ENV_KEYS, "BENCH_WORKDIR")}
210
+ for key in RUNNER_ENV_KEYS:
211
+ os.environ.pop(key, None)
212
+ os.environ["BENCH_WORKDIR"] = str(work)
213
+ try:
214
+ return runner.capture_process(
215
+ work, manifest_path, run_id, phase, round_, obligation,
216
+ )
217
+ finally:
218
+ for key, value in saved.items():
219
+ if value is None:
220
+ os.environ.pop(key, None)
221
+ else:
222
+ os.environ[key] = value
223
+
224
+
118
225
  VERIFICATION_SECTION_RE = re.compile(
119
226
  r'(?ms)^<!--[ \t]*devlyn:verification[ \t]*-->[ \t]*\n(#{1,6}[ \t]+[^\n]*\n.*?)(?=^#{1,6}[ \t]+|\Z)'
120
227
  )
@@ -127,8 +234,9 @@ FORBIDDEN_RISK_PROBE_CMD_RE = re.compile(
127
234
  )
128
235
  EXTERNAL_URL_RE = re.compile(r"https?://([^/\s\"']+)", re.IGNORECASE)
129
236
  RISK_PROBE_SCRIPT_RE = re.compile(
130
- r'(?<![\w./-])(\.devlyn/probes/[A-Za-z0-9][A-Za-z0-9._/-]*)'
237
+ r'(?<![\w./-])(?:\./)?(\.devlyn/probes/[A-Za-z0-9][A-Za-z0-9._/-]*)'
131
238
  )
239
+ RISK_PROBE_RAW_REF = ".devlyn/probes/"
132
240
  RISK_PROBE_INTEGRITY_FIX_HINT = (
133
241
  "Probe artifacts changed after PHASE 1.5. Only the orchestrator may regenerate probes: "
134
242
  "re-run probe validation and re-write state.risk_probes_digest; workers must never modify "
@@ -480,7 +588,28 @@ def referenced_risk_probe_scripts(cmd: str) -> list[str]:
480
588
  return scripts
481
589
 
482
590
 
591
+ def unrecognized_risk_probe_reference(cmd: str) -> str | None:
592
+ command = cmd or ""
593
+ matches = list(RISK_PROBE_SCRIPT_RE.finditer(command))
594
+ offset = command.find(RISK_PROBE_RAW_REF)
595
+ while offset >= 0:
596
+ if not any(
597
+ match.start() <= offset and offset + len(RISK_PROBE_RAW_REF) <= match.end()
598
+ for match in matches
599
+ ):
600
+ tail = command[offset:].split(maxsplit=1)[0][:80]
601
+ return (
602
+ f"unrecognized probe script reference {tail!r}; write it as "
603
+ ".devlyn/probes/<file> (a ./ prefix is the only alias)"
604
+ )
605
+ offset = command.find(RISK_PROBE_RAW_REF, offset + len(RISK_PROBE_RAW_REF))
606
+ return None
607
+
608
+
483
609
  def validate_risk_probe_scripts(cmd: str, index: int, work: Path) -> str | None:
610
+ unrecognized = unrecognized_risk_probe_reference(cmd)
611
+ if unrecognized:
612
+ return f"risk-probes[{index}].cmd has {unrecognized}"
484
613
  for rel_path in referenced_risk_probe_scripts(cmd):
485
614
  path = Path(rel_path)
486
615
  if ".." in path.parts:
@@ -540,6 +669,9 @@ def risk_probes_digest(devlyn_dir: Path) -> tuple[str | None, str | None]:
540
669
  except ValueError as e:
541
670
  return (None, f"risk-probes[{index}] invalid JSON: {e}")
542
671
  cmd = probe.get("cmd") if isinstance(probe, dict) else ""
672
+ unrecognized = unrecognized_risk_probe_reference(cmd)
673
+ if unrecognized:
674
+ return (None, f"risk-probes[{index}].cmd has {unrecognized}")
543
675
  scripts.update(referenced_risk_probe_scripts(cmd))
544
676
 
545
677
  digest = hashlib.sha256()
@@ -1115,7 +1247,13 @@ def stage_from_expected(
1115
1247
  return (True, True, None, expected_path, data)
1116
1248
 
1117
1249
 
1118
- def write_malformed_finding(devlyn_dir: Path, error: str, source_path: Path | None) -> None:
1250
+ def write_malformed_finding(
1251
+ devlyn_dir: Path,
1252
+ error: str,
1253
+ source_path: Path | None,
1254
+ *,
1255
+ fix_hint: str | None = None,
1256
+ ) -> None:
1119
1257
  """Emit a single CRITICAL finding for a malformed verification carrier."""
1120
1258
  devlyn_dir.mkdir(parents=True, exist_ok=True)
1121
1259
  findings_path = devlyn_dir / output_findings_name()
@@ -1131,7 +1269,7 @@ def write_malformed_finding(devlyn_dir: Path, error: str, source_path: Path | No
1131
1269
  "line": 1,
1132
1270
  "phase": output_phase(),
1133
1271
  "criterion_ref": "spec-verify://carrier",
1134
- "fix_hint": (
1272
+ "fix_hint": fix_hint if fix_hint is not None else (
1135
1273
  "Fix the sibling `spec.expected.json` file or the `## Verification` "
1136
1274
  "```json``` block: a JSON object with a non-empty `verification_commands` array of "
1137
1275
  "{cmd, exit_code?, stdout_contains?, stdout_not_contains?} "
@@ -1360,6 +1498,24 @@ def expected_contract_findings(
1360
1498
  return (findings, seq)
1361
1499
 
1362
1500
 
1501
+ def validate_surface_brace_glob(entry: str) -> str | None:
1502
+ if "{" not in entry and "}" not in entry:
1503
+ return None
1504
+ brace = re.search(r"\{([^{}]*)\}", entry)
1505
+ alternatives = brace.group(1).split(",") if brace else []
1506
+ if (
1507
+ entry.count("{") != 1
1508
+ or entry.count("}") != 1
1509
+ or len(alternatives) < 2
1510
+ or any(not value or any(char in value for char in "{},/*") for value in alternatives)
1511
+ ):
1512
+ return (
1513
+ f"unsupported brace glob {entry!r}; supported form is {{alt1,alt2,...}} "
1514
+ "with at least two non-empty plain alternatives"
1515
+ )
1516
+ return None
1517
+
1518
+
1363
1519
  def validate_authorized_surface_shape(data: object) -> str | None:
1364
1520
  if not isinstance(data, dict):
1365
1521
  return "top-level must be a JSON object"
@@ -1384,17 +1540,34 @@ def validate_authorized_surface_shape(data: object) -> str | None:
1384
1540
  return f"authorized_surface directory grant needs a non-empty prefix before '/**': {entry!r}"
1385
1541
  if ".." in stem.split("/"):
1386
1542
  return f"authorized_surface entry must not contain '..': {entry!r}"
1543
+ brace_error = validate_surface_brace_glob(entry)
1544
+ if brace_error is not None:
1545
+ return brace_error
1387
1546
  return None
1388
1547
 
1389
1548
 
1390
1549
  def path_matches_surface(path: str, surface: list[str]) -> bool:
1391
1550
  for entry in surface:
1392
- if entry.endswith("/**"):
1393
- prefix = entry[:-3]
1394
- if path == prefix or path.startswith(prefix + "/"):
1551
+ brace_error = validate_surface_brace_glob(entry)
1552
+ if brace_error is not None:
1553
+ raise ValueError(brace_error)
1554
+ if "{" in entry:
1555
+ brace = re.search(r"\{([^{}]*)\}", entry)
1556
+ assert brace is not None
1557
+ alternatives = brace.group(1).split(",")
1558
+ entries = tuple(
1559
+ entry[:brace.start()] + value + entry[brace.end():]
1560
+ for value in alternatives
1561
+ )
1562
+ else:
1563
+ entries = (entry,)
1564
+ for expanded in entries:
1565
+ if expanded.endswith("/**"):
1566
+ prefix = expanded[:-3].rstrip("/")
1567
+ if path == prefix or path.startswith(f"{prefix}/"):
1568
+ return True
1569
+ elif path == expanded:
1395
1570
  return True
1396
- elif path == entry:
1397
- return True
1398
1571
  return False
1399
1572
 
1400
1573
 
@@ -1753,6 +1926,12 @@ def run_check_expected_mode(expected_path: Path) -> int:
1753
1926
 
1754
1927
 
1755
1928
  def run_self_test() -> int:
1929
+ try:
1930
+ loads_strict_json('{"verification_commands":[],"verification_commands":[{}]}')
1931
+ except ValueError as exc:
1932
+ assert "duplicate JSON key" in str(exc)
1933
+ else:
1934
+ raise AssertionError("duplicate expected-contract key was accepted")
1756
1935
  # Hermeticity: a pipeline replay exports BENCH_WORKDIR at the live repo;
1757
1936
  # inherited into scenario children it wins over their tmp cwd (the
1758
1937
  # default-mode work resolution below) and re-executes the LIVE
@@ -1761,10 +1940,53 @@ def run_self_test() -> int:
1761
1940
  # (observed 2026-08-03, iter-0089 canary).
1762
1941
  os.environ.pop("BENCH_WORKDIR", None)
1763
1942
  script_path = str(Path(__file__).resolve())
1943
+ with tempfile.TemporaryDirectory() as help_tmp:
1944
+ for help_flag in ("--help", "-h"):
1945
+ help_result = subprocess.run(
1946
+ [sys.executable, script_path, help_flag],
1947
+ cwd=help_tmp,
1948
+ capture_output=True,
1949
+ text=True,
1950
+ )
1951
+ if help_result.returncode != 0 or "usage:" not in help_result.stdout:
1952
+ print(f"{help_flag} did not return usage successfully", file=sys.stderr)
1953
+ return 1
1954
+ if (Path(help_tmp) / ".devlyn").exists():
1955
+ print(f"{help_flag} mutated .devlyn", file=sys.stderr)
1956
+ return 1
1957
+ unknown_result = subprocess.run(
1958
+ [sys.executable, script_path, "--not-a-real-option"],
1959
+ cwd=help_tmp,
1960
+ capture_output=True,
1961
+ text=True,
1962
+ )
1963
+ if unknown_result.returncode != 2 or "unknown argument" not in unknown_result.stderr:
1964
+ print("unknown option did not fail closed", file=sys.stderr)
1965
+ return 1
1966
+ if (Path(help_tmp) / ".devlyn").exists():
1967
+ print("unknown option mutated .devlyn", file=sys.stderr)
1968
+ return 1
1969
+ runner = process_evidence_module()
1970
+ try:
1971
+ mechanical_evidence_identity({"version": "3.0", "phases": {}}, runner)
1972
+ except runner.EvidenceError as exc:
1973
+ assert "schema-v3 MECHANICAL evidence identity is invalid" in str(exc)
1974
+ else:
1975
+ raise AssertionError("schema-v3 MECHANICAL evidence accepted missing run identity")
1764
1976
  with tempfile.TemporaryDirectory() as td:
1765
1977
  work = Path(td)
1766
1978
  devlyn = work / ".devlyn"
1767
1979
  devlyn.mkdir()
1980
+ assert path_matches_surface("src/a/example.py", ["src/{a,b}/**"])
1981
+ assert path_matches_surface("src/b/example.py", ["src/{a,b}/**"])
1982
+ assert not path_matches_surface("src/c/example.py", ["src/{a,b}/**"])
1983
+ for entry in ("src/{a,b", "src/{a,}/**", "src/{a,{b,c}}", "src/{a,**}"):
1984
+ try:
1985
+ path_matches_surface("src/a/example.py", [entry])
1986
+ except ValueError as exc:
1987
+ assert entry in str(exc) and "supported form" in str(exc)
1988
+ else:
1989
+ raise AssertionError(f"malformed brace glob accepted: {entry}")
1768
1990
  spec_md = work / "spec.md"
1769
1991
  spec_md.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- probe must pass visible marker.\n")
1770
1992
  (devlyn / "pipeline.state.json").write_text(json.dumps({
@@ -1915,13 +2137,18 @@ def run_self_test() -> int:
1915
2137
  "cmd": "python3 -c \"import time; time.sleep(1)\"",
1916
2138
  "timeout_sec": 5,
1917
2139
  },
2140
+ {
2141
+ "cmd": "printf 'Operation not permitted' >&2; exit 1",
2142
+ },
1918
2143
  ]
1919
2144
  })
1920
2145
  + "\n```\n",
1921
2146
  encoding="utf-8",
1922
2147
  )
1923
2148
  (timeout_run_devlyn / "pipeline.state.json").write_text(json.dumps({
1924
- "source": {"type": "spec", "spec_path": str(timeout_run_spec)}
2149
+ "run_id": "rs-timeout-run",
2150
+ "source": {"type": "spec", "spec_path": str(timeout_run_spec)},
2151
+ "phases": {"build_gate": {"round": 2}},
1925
2152
  }))
1926
2153
  timeout_run = subprocess.run(
1927
2154
  [sys.executable, script_path],
@@ -1932,9 +2159,10 @@ def run_self_test() -> int:
1932
2159
  if timeout_run.returncode == 0:
1933
2160
  print("declared one-second timeout did not fail", file=sys.stderr)
1934
2161
  return 1
1935
- timeout_results = loads_strict_json(
2162
+ timeout_document = loads_strict_json(
1936
2163
  (timeout_run_devlyn / "spec-verify.results.json").read_text()
1937
- )["commands"]
2164
+ )
2165
+ timeout_results = timeout_document["commands"]
1938
2166
  timeout_findings = [
1939
2167
  loads_strict_json(line)
1940
2168
  for line in (timeout_run_devlyn / output_findings_name()).read_text().splitlines()
@@ -1944,10 +2172,37 @@ def run_self_test() -> int:
1944
2172
  timeout_results[0].get("reason") != "timeout"
1945
2173
  or timeout_results[0].get("timeout_sec") != 1
1946
2174
  or not timeout_results[1].get("pass")
2175
+ or timeout_results[2].get("reason") != "exit"
2176
+ or timeout_results[2].get("classification") != {
2177
+ "kind": "product_result", "operation": None,
2178
+ }
1947
2179
  ):
1948
2180
  print("declared timeout budgets were not honored", file=sys.stderr)
1949
2181
  print(timeout_results, file=sys.stderr)
1950
2182
  return 1
2183
+ timeout_carrier = timeout_document.get("process_evidence") or {}
2184
+ timeout_manifest_path = (
2185
+ ".devlyn/process-evidence/rs-timeout-run/build_gate/round-2/manifest.json"
2186
+ )
2187
+ timeout_manifest = loads_strict_json(
2188
+ (timeout_run_root / timeout_manifest_path).read_text()
2189
+ )
2190
+ if (
2191
+ timeout_carrier.get("manifest", {}).get("path") != timeout_manifest_path
2192
+ or [entry.get("id") for entry in timeout_manifest.get("entries", [])] != [
2193
+ "verification-command-0001",
2194
+ "verification-command-0002",
2195
+ "verification-command-0003",
2196
+ ]
2197
+ or any(
2198
+ not (timeout_run_root / entry[stream]["path"]).is_file()
2199
+ for entry in timeout_manifest.get("entries", [])
2200
+ for stream in ("stdout", "stderr")
2201
+ )
2202
+ ):
2203
+ print("literal commands did not emit sealed BUILD_GATE evidence", file=sys.stderr)
2204
+ print(timeout_document, file=sys.stderr)
2205
+ return 1
1951
2206
  timeout_finding = timeout_findings[0] if timeout_findings else {}
1952
2207
  if (
1953
2208
  timeout_finding.get("rule_id") != "correctness.verification-timeout"
@@ -1958,6 +2213,17 @@ def run_self_test() -> int:
1958
2213
  print("timeout finding did not carry the distinct budget contract", file=sys.stderr)
1959
2214
  print(timeout_finding, file=sys.stderr)
1960
2215
  return 1
2216
+ if (
2217
+ not any(
2218
+ finding.get("rule_id") == "correctness.spec-literal-mismatch"
2219
+ for finding in timeout_findings
2220
+ )
2221
+ or "BLOCKED:build-env-underprovisioned" in timeout_run.stderr
2222
+ ):
2223
+ print("ordinary restricted-looking stderr did not remain a product finding", file=sys.stderr)
2224
+ print(timeout_findings, file=sys.stderr)
2225
+ print(timeout_run.stderr, file=sys.stderr)
2226
+ return 1
1961
2227
 
1962
2228
  runner_env_root = work / "runner-env-isolation"
1963
2229
  runner_env_root.mkdir()
@@ -1980,7 +2246,9 @@ def run_self_test() -> int:
1980
2246
  encoding="utf-8",
1981
2247
  )
1982
2248
  (runner_env_devlyn / "pipeline.state.json").write_text(json.dumps({
1983
- "source": {"type": "spec", "spec_path": str(runner_env_spec)}
2249
+ "run_id": "rs-runner-env",
2250
+ "source": {"type": "spec", "spec_path": str(runner_env_spec)},
2251
+ "phases": {"verify": {"round": 3}},
1984
2252
  }))
1985
2253
  runner_env = os.environ.copy()
1986
2254
  runner_env.update({
@@ -1995,13 +2263,20 @@ def run_self_test() -> int:
1995
2263
  capture_output=True,
1996
2264
  text=True,
1997
2265
  )
1998
- runner_env_results = loads_strict_json(
2266
+ runner_env_document = loads_strict_json(
1999
2267
  (runner_env_devlyn / "spec-verify.results.json").read_text()
2000
- )["commands"]
2268
+ )
2269
+ runner_env_results = runner_env_document["commands"]
2001
2270
  if runner_env_run.returncode != 0 or not runner_env_results[0].get("pass"):
2002
2271
  print("runner-directed environment leaked into a verification command", file=sys.stderr)
2003
2272
  print(runner_env_run.stderr, file=sys.stderr)
2004
2273
  return 1
2274
+ if runner_env_document.get("process_evidence", {}).get("manifest", {}).get("path") != (
2275
+ ".devlyn/process-evidence/rs-runner-env/verify/round-3/manifest.json"
2276
+ ):
2277
+ print("VERIFY MECHANICAL evidence used the wrong phase identity", file=sys.stderr)
2278
+ print(runner_env_document, file=sys.stderr)
2279
+ return 1
2005
2280
  env = os.environ.copy()
2006
2281
  env["BENCH_WORKDIR"] = str(work)
2007
2282
  validate_without_digest = subprocess.run(
@@ -2032,9 +2307,11 @@ def run_self_test() -> int:
2032
2307
  print(repr(digest_run.stdout), file=sys.stderr)
2033
2308
  return 1
2034
2309
  (devlyn / "pipeline.state.json").write_text(json.dumps({
2310
+ "run_id": "rs-risk-probes",
2035
2311
  "source": {"type": "spec", "spec_path": str(spec_md)},
2036
2312
  "risk_profile": {"risk_probes_enabled": True},
2037
2313
  "risk_probes_digest": risk_digest,
2314
+ "phases": {"build_gate": {"round": 4}},
2038
2315
  }))
2039
2316
  good = subprocess.run(
2040
2317
  [sys.executable, script_path, "--include-risk-probes"],
@@ -2046,6 +2323,24 @@ def run_self_test() -> int:
2046
2323
  if good.returncode != 0:
2047
2324
  print(good.stderr, file=sys.stderr)
2048
2325
  return 1
2326
+ good_document = loads_strict_json(
2327
+ (devlyn / "spec-verify.results.json").read_text()
2328
+ )
2329
+ good_manifest_path = good_document.get("process_evidence", {}).get("manifest", {}).get("path")
2330
+ good_manifest = (
2331
+ loads_strict_json((work / good_manifest_path).read_text())
2332
+ if isinstance(good_manifest_path, str)
2333
+ else {}
2334
+ )
2335
+ if (
2336
+ good_manifest_path != ".devlyn/process-evidence/rs-risk-probes/build_gate/round-4/manifest.json"
2337
+ or [entry.get("id") for entry in good_manifest.get("entries", [])] != [
2338
+ "verification-command-0001", "risk-probe-0002",
2339
+ ]
2340
+ ):
2341
+ print("literal command and risk probe did not share the sealed manifest", file=sys.stderr)
2342
+ print(good_document, file=sys.stderr)
2343
+ return 1
2049
2344
 
2050
2345
  probe_script.write_text("print('mutated-probe')\n", encoding="utf-8")
2051
2346
  mutated_script = subprocess.run(
@@ -2424,22 +2719,13 @@ def run_self_test() -> int:
2424
2719
  probes_dir = devlyn / "probes"
2425
2720
  probes_dir.mkdir(exist_ok=True)
2426
2721
  (probes_dir / "Pscript.py").write_text("print('script-ok')\n", encoding="utf-8")
2427
- (devlyn / "risk-probes.jsonl").write_text(json.dumps({
2722
+ script_probe_payload = {
2723
+ **risk_probe_payload,
2428
2724
  "id": "Pscript",
2429
- "derived_from": "probe must pass visible marker.",
2430
2725
  "cmd": "python3 .devlyn/probes/Pscript.py",
2431
- "exit_code": 0,
2432
2726
  "stdout_contains": ["script-ok"],
2433
- "stdout_not_contains": [],
2434
- "tags": ["shape_contract"],
2435
- "tag_evidence": {
2436
- "shape_contract": [
2437
- "uses_visible_input_key_names",
2438
- "asserts_visible_output_key_names",
2439
- "asserts_no_unexpected_output_keys",
2440
- ],
2441
- },
2442
- }) + "\n")
2727
+ }
2728
+ (devlyn / "risk-probes.jsonl").write_text(json.dumps(script_probe_payload) + "\n")
2443
2729
  good_script_probe = subprocess.run(
2444
2730
  [sys.executable, script_path, "--validate-risk-probes"],
2445
2731
  cwd=work,
@@ -2452,10 +2738,47 @@ def run_self_test() -> int:
2452
2738
  print(good_script_probe.stderr, file=sys.stderr)
2453
2739
  return 1
2454
2740
 
2741
+ mixed_script_payload = {
2742
+ **script_probe_payload,
2743
+ "id": "Pmixed",
2744
+ "cmd": "python3 .devlyn/probes/Pscript.py && python3 ./.devlyn/probes/P1.py",
2745
+ }
2746
+ (devlyn / "risk-probes.jsonl").write_text(json.dumps(mixed_script_payload) + "\n")
2747
+ mixed_script_probe = subprocess.run(
2748
+ [sys.executable, script_path, "--validate-risk-probes"],
2749
+ cwd=work,
2750
+ env=env,
2751
+ capture_output=True,
2752
+ text=True,
2753
+ )
2754
+ if mixed_script_probe.returncode != 0:
2755
+ print("canonical and ./-alias probe script references were rejected", file=sys.stderr)
2756
+ print(mixed_script_probe.stderr, file=sys.stderr)
2757
+ return 1
2758
+ mixed_digest, mixed_digest_error = risk_probes_digest(devlyn)
2759
+ if mixed_digest_error:
2760
+ print(mixed_digest_error, file=sys.stderr)
2761
+ return 1
2762
+ (probes_dir / "Pscript.py").write_text("print('script-mutated')\n", encoding="utf-8")
2763
+ canonical_mutated_digest, canonical_mutated_error = risk_probes_digest(devlyn)
2764
+ (probes_dir / "Pscript.py").write_text("print('script-ok')\n", encoding="utf-8")
2765
+ if canonical_mutated_error:
2766
+ print(canonical_mutated_error, file=sys.stderr)
2767
+ return 1
2768
+ probe_script.write_text("print('probe-mutated')\n", encoding="utf-8")
2769
+ alias_mutated_digest, alias_mutated_error = risk_probes_digest(devlyn)
2770
+ probe_script.write_text("print('probe-ok')\n", encoding="utf-8")
2771
+ if alias_mutated_error:
2772
+ print(alias_mutated_error, file=sys.stderr)
2773
+ return 1
2774
+ if len({mixed_digest, canonical_mutated_digest, alias_mutated_digest}) != 3:
2775
+ print("probe digest did not change for both canonical and ./-alias scripts", file=sys.stderr)
2776
+ return 1
2777
+
2455
2778
  (devlyn / "risk-probes.jsonl").write_text(json.dumps({
2456
2779
  "id": "Pmissing",
2457
2780
  "derived_from": "probe must pass visible marker.",
2458
- "cmd": "python3 .devlyn/probes/missing.py",
2781
+ "cmd": "python3 ./.devlyn/probes/missing.py",
2459
2782
  "exit_code": 0,
2460
2783
  }) + "\n")
2461
2784
  missing_script_probe = subprocess.run(
@@ -2488,6 +2811,45 @@ def run_self_test() -> int:
2488
2811
  print(missing_script_digest.stderr, file=sys.stderr)
2489
2812
  return 1
2490
2813
 
2814
+ for bad_form in (
2815
+ "../.devlyn/probes/Pscript.py",
2816
+ "/tmp/.devlyn/probes/Pscript.py",
2817
+ "$PWD/.devlyn/probes/Pscript.py",
2818
+ "././.devlyn/probes/Pscript.py",
2819
+ ):
2820
+ (devlyn / "risk-probes.jsonl").write_text(json.dumps({
2821
+ "id": "Pbadref",
2822
+ "derived_from": "probe must pass visible marker.",
2823
+ "cmd": f"python3 {bad_form}",
2824
+ "exit_code": 0,
2825
+ }) + "\n")
2826
+ bad_ref_probe = subprocess.run(
2827
+ [sys.executable, script_path, "--validate-risk-probes"],
2828
+ cwd=work,
2829
+ env=env,
2830
+ capture_output=True,
2831
+ text=True,
2832
+ )
2833
+ bad_ref_digest = subprocess.run(
2834
+ [sys.executable, script_path, "--print-risk-probes-digest"],
2835
+ cwd=work,
2836
+ env=env,
2837
+ capture_output=True,
2838
+ text=True,
2839
+ )
2840
+ unrecognized = unrecognized_risk_probe_reference(f"python3 {bad_form}")
2841
+ expected_error = f"risk-probes[0].cmd has {unrecognized}"
2842
+ if (
2843
+ bad_ref_probe.returncode == 0
2844
+ or expected_error not in bad_ref_probe.stderr
2845
+ or bad_ref_digest.returncode == 0
2846
+ or expected_error not in bad_ref_digest.stderr
2847
+ ):
2848
+ print(f"bad probe script reference was not rejected: {bad_form}", file=sys.stderr)
2849
+ print(bad_ref_probe.stderr, file=sys.stderr)
2850
+ print(bad_ref_digest.stderr, file=sys.stderr)
2851
+ return 1
2852
+
2491
2853
  (probes_dir / "Phidden.py").write_text("print('benchmark/auto-resolve/fixtures')\n", encoding="utf-8")
2492
2854
  (devlyn / "risk-probes.jsonl").write_text(json.dumps({
2493
2855
  "id": "Phidden",
@@ -3038,6 +3400,103 @@ def run_self_test() -> int:
3038
3400
  print(expected_nan_json.stderr, file=sys.stderr)
3039
3401
  return 1
3040
3402
 
3403
+ external_diff_root = work / "external-diff-mode-authority"
3404
+ external_diff_root.mkdir()
3405
+ external_diff_devlyn = external_diff_root / ".devlyn"
3406
+ external_diff_devlyn.mkdir()
3407
+ external_diff_spec = external_diff_root / "spec.md"
3408
+ external_diff_spec.write_text(
3409
+ "# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
3410
+ "- external diff remains verify-only.\n",
3411
+ encoding="utf-8",
3412
+ )
3413
+ (external_diff_root / "spec.expected.json").write_text(json.dumps({
3414
+ "verification_commands": [
3415
+ {"cmd": "printf external-diff-ok", "stdout_contains": ["external-diff-ok"]}
3416
+ ]
3417
+ }) + "\n", encoding="utf-8")
3418
+ (external_diff_root / "external-only.txt").write_text("base\n", encoding="utf-8")
3419
+ (external_diff_root / "outside.txt").write_text("base\n", encoding="utf-8")
3420
+ subprocess.run(["git", "init", "-q"], cwd=external_diff_root, check=True)
3421
+ subprocess.run(["git", "add", "-A"], cwd=external_diff_root, check=True)
3422
+ subprocess.run(
3423
+ ["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "base"],
3424
+ cwd=external_diff_root,
3425
+ check=True,
3426
+ )
3427
+ external_diff_base_sha = subprocess.check_output(
3428
+ ["git", "rev-parse", "HEAD"], cwd=external_diff_root, text=True,
3429
+ ).strip()
3430
+ (external_diff_root / "outside.txt").write_text("worktree-only\n", encoding="utf-8")
3431
+ (external_diff_devlyn / "external-diff.patch").write_text(
3432
+ "diff --git a/external-only.txt b/external-only.txt\n"
3433
+ "--- a/external-only.txt\n"
3434
+ "+++ b/external-only.txt\n"
3435
+ "@@ -1 +1 @@\n"
3436
+ "-base\n"
3437
+ "+external\n",
3438
+ encoding="utf-8",
3439
+ )
3440
+ (external_diff_devlyn / "plan.md").write_text(
3441
+ "<!-- devlyn:authorized-surface -->\n## Files to touch\n\n"
3442
+ "```json\n{\"authorized_surface\": [\"external-only.txt\"]}\n```\n",
3443
+ encoding="utf-8",
3444
+ )
3445
+ (external_diff_devlyn / "untracked.baseline").write_text("", encoding="utf-8")
3446
+ external_diff_state = {
3447
+ "mode": "free-form",
3448
+ "source": {"type": "spec", "spec_path": str(external_diff_spec)},
3449
+ "base_ref": {"sha": external_diff_base_sha},
3450
+ }
3451
+ external_diff_state_path = external_diff_devlyn / "pipeline.state.json"
3452
+ external_diff_state_path.write_text(json.dumps(external_diff_state) + "\n")
3453
+ external_diff_free_form = subprocess.run(
3454
+ [sys.executable, script_path],
3455
+ cwd=external_diff_root,
3456
+ capture_output=True,
3457
+ text=True,
3458
+ )
3459
+ external_diff_findings_path = external_diff_devlyn / output_findings_name()
3460
+ external_diff_free_form_findings = (
3461
+ external_diff_findings_path.read_text()
3462
+ if external_diff_findings_path.is_file()
3463
+ else ""
3464
+ )
3465
+ external_diff_state["mode"] = "verify-only"
3466
+ external_diff_state_path.write_text(json.dumps(external_diff_state) + "\n")
3467
+ external_diff_verify_only = subprocess.run(
3468
+ [sys.executable, script_path],
3469
+ cwd=external_diff_root,
3470
+ capture_output=True,
3471
+ text=True,
3472
+ )
3473
+ if external_diff_free_form.returncode != 1:
3474
+ print("non-verify-only mode accepted .devlyn/external-diff.patch", file=sys.stderr)
3475
+ print(external_diff_free_form.stderr, file=sys.stderr)
3476
+ return 1
3477
+ if (
3478
+ '"rule_id": "correctness.spec-verify-malformed"'
3479
+ not in external_diff_free_form_findings
3480
+ or '"severity": "CRITICAL"' not in external_diff_free_form_findings
3481
+ or ".devlyn/external-diff.patch" not in external_diff_free_form_findings
3482
+ or "free-form" not in external_diff_free_form_findings
3483
+ or "verify-only" not in external_diff_free_form_findings
3484
+ or "Remove `.devlyn/external-diff.patch` for ordinary runs"
3485
+ not in external_diff_free_form_findings
3486
+ or "only when intentionally verifying an external patch"
3487
+ not in external_diff_free_form_findings
3488
+ ):
3489
+ print(
3490
+ "non-verify-only external diff did not emit the named CRITICAL finding and remediation",
3491
+ file=sys.stderr,
3492
+ )
3493
+ print(external_diff_free_form_findings, file=sys.stderr)
3494
+ return 1
3495
+ if external_diff_verify_only.returncode != 0:
3496
+ print("verify-only external diff was rejected or not consumed", file=sys.stderr)
3497
+ print(external_diff_verify_only.stderr, file=sys.stderr)
3498
+ return 1
3499
+
3041
3500
  spec_integrity = work / "spec-integrity"
3042
3501
  spec_integrity.mkdir()
3043
3502
  spec_integrity_devlyn = spec_integrity / ".devlyn"
@@ -3975,6 +4434,32 @@ def run_self_test() -> int:
3975
4434
  print("--print-authorized-surface accepted malformed plan.md", file=sys.stderr)
3976
4435
  return 1
3977
4436
 
4437
+ for entry in ("src/{a,b", "src/{a,}/**", "src/{a,{b,c}}", "src/{a,**}"):
4438
+ (scope_devlyn / "plan.md").write_text(
4439
+ "# PLAN\n\n<!-- devlyn:authorized-surface -->\n## 1. Files to touch\n\n"
4440
+ "```json\n"
4441
+ + json.dumps({"authorized_surface": [entry]}) + "\n```\n"
4442
+ )
4443
+ brace_build_gate = subprocess.run(
4444
+ [sys.executable, script_path], cwd=scope_root, env=scope_build_gate_env,
4445
+ capture_output=True, text=True,
4446
+ )
4447
+ brace_print_surface = subprocess.run(
4448
+ [sys.executable, script_path, "--print-authorized-surface"], cwd=scope_root,
4449
+ capture_output=True, text=True,
4450
+ )
4451
+ findings_text = scope_findings_path.read_text(encoding="utf-8")
4452
+ if (
4453
+ brace_build_gate.returncode == 0
4454
+ or brace_print_surface.returncode == 0
4455
+ or "scope.authorized-surface-malformed" not in findings_text
4456
+ or "supported form" not in findings_text
4457
+ or "supported form" not in brace_print_surface.stderr
4458
+ or "Traceback" in brace_build_gate.stderr + brace_print_surface.stderr
4459
+ ):
4460
+ print(f"malformed brace glob escaped the authorized-surface carrier: {entry}", file=sys.stderr)
4461
+ return 1
4462
+
3978
4463
  # Test 3: valid surface, in-scope-only diff -> no scope findings, exit 0.
3979
4464
  (scope_devlyn / "plan.md").write_text(
3980
4465
  "# PLAN\n\n<!-- devlyn:authorized-surface -->\n## 1. Files to touch\n\n"
@@ -4107,12 +4592,16 @@ def run_self_test() -> int:
4107
4592
  print(verify_mech_findings, file=sys.stderr)
4108
4593
  return 1
4109
4594
 
4110
- # Test 7: shape validation rejects absolute paths, `..`, and duplicates.
4595
+ # Test 7: shape validation rejects absolute paths, `..`, duplicates, and malformed braces.
4111
4596
  for bad_surface in (
4112
4597
  {"authorized_surface": ["/etc/passwd"]},
4113
4598
  {"authorized_surface": ["bin/../etc/passwd"]},
4114
4599
  {"authorized_surface": ["bin/cli.js", "bin/cli.js"]},
4115
4600
  {"authorized_surface": []},
4601
+ {"authorized_surface": ["src/{a,b"]},
4602
+ {"authorized_surface": ["src/{a,}/**"]},
4603
+ {"authorized_surface": ["src/{a,{b,c}}"]},
4604
+ {"authorized_surface": ["src/{a,**}"]},
4116
4605
  ):
4117
4606
  err = validate_authorized_surface_shape(bad_surface)
4118
4607
  if err is None:
@@ -4166,6 +4655,14 @@ def run_self_test() -> int:
4166
4655
 
4167
4656
 
4168
4657
  def main() -> int:
4658
+ if "--help" in sys.argv[1:] or "-h" in sys.argv[1:]:
4659
+ print(
4660
+ "usage: spec-verify-check.py [-h | --help | --include-risk-probes | "
4661
+ "--validate-risk-probes | --print-risk-probes-digest | "
4662
+ "--print-authorized-surface | --write-untracked-baseline | "
4663
+ "--check <markdown-path> | --check-expected <json-path> | --self-test]"
4664
+ )
4665
+ return 0
4169
4666
  include_risk_probes = False
4170
4667
  validate_risk_probes_only = False
4171
4668
  print_risk_probes_digest = False
@@ -4202,6 +4699,10 @@ def main() -> int:
4202
4699
  return 2
4203
4700
  return run_check_expected_mode(Path(sys.argv[2]))
4204
4701
 
4702
+ if len(sys.argv) != 1:
4703
+ print(f"unknown argument(s): {' '.join(sys.argv[1:])}", file=sys.stderr)
4704
+ return 2
4705
+
4205
4706
  bench_mode = "BENCH_WORKDIR" in os.environ
4206
4707
  work = Path(os.environ.get("BENCH_WORKDIR") or os.getcwd())
4207
4708
  devlyn_dir = work / ".devlyn"
@@ -4254,6 +4755,24 @@ def main() -> int:
4254
4755
  trust_bench_staged = bench_mode and pre_staged
4255
4756
  src_type, source_md = read_source(work, devlyn_dir)
4256
4757
  state = read_state(devlyn_dir)
4758
+ external_diff = devlyn_dir / "external-diff.patch"
4759
+ if external_diff.is_file() and state.get("mode") != "verify-only":
4760
+ error = (
4761
+ ".devlyn/external-diff.patch requires pipeline.state.json mode "
4762
+ f"'verify-only'; actual mode is {state.get('mode')!r}"
4763
+ )
4764
+ print(f"[spec-verify] carrier malformed: {error}", file=sys.stderr)
4765
+ write_malformed_finding(
4766
+ devlyn_dir,
4767
+ error,
4768
+ external_diff,
4769
+ fix_hint=(
4770
+ "Remove `.devlyn/external-diff.patch` for ordinary runs, or use "
4771
+ "`/devlyn:resolve --verify-only <diff-or-PR-ref> --spec <path>` only when "
4772
+ "intentionally verifying an external patch."
4773
+ ),
4774
+ )
4775
+ return 1
4257
4776
  integrity_error = source_integrity_error(src_type, state, source_md)
4258
4777
  if integrity_error:
4259
4778
  print(f"[spec-verify] carrier malformed: {integrity_error}", file=sys.stderr)
@@ -4372,18 +4891,17 @@ def main() -> int:
4372
4891
  results_path = devlyn_dir / "spec-verify.results.json"
4373
4892
  findings_path = devlyn_dir / output_findings_name()
4374
4893
 
4375
- verify_env = os.environ.copy()
4376
- for runner_key in (
4377
- "SPEC_VERIFY_PHASE",
4378
- "SPEC_VERIFY_FINDINGS_FILE",
4379
- "SPEC_VERIFY_FINDING_PREFIX",
4380
- ):
4381
- verify_env.pop(runner_key, None)
4382
- verify_env["BENCH_WORKDIR"] = str(work)
4383
-
4384
4894
  results: list[dict] = []
4385
4895
  findings: list[dict] = []
4386
4896
  finding_seq = 1
4897
+ runner = process_evidence_module()
4898
+ evidence_phase, evidence_run_id, evidence_round, manifest_relative = (
4899
+ mechanical_evidence_identity(state, runner)
4900
+ )
4901
+ manifest_path = work / manifest_relative
4902
+ obligations: list[dict] = []
4903
+ capability_denials: list[dict] = []
4904
+ evidence_error: str | None = None
4387
4905
 
4388
4906
  for idx, vc in enumerate(commands):
4389
4907
  cmd = vc.get("cmd")
@@ -4397,177 +4915,185 @@ def main() -> int:
4397
4915
  stdout_contains = vc.get("stdout_contains", []) or []
4398
4916
  stdout_not_contains = vc.get("stdout_not_contains", []) or []
4399
4917
  timeout_sec = verification_timeout_sec(vc)
4918
+ obligation = mechanical_obligation(vc, idx, evidence_phase)
4919
+ obligations.append(obligation)
4920
+ criterion_ref = (
4921
+ f"risk-probe:{vc.get('id')}"
4922
+ if is_risk_probe
4923
+ else f"spec-verify://verification_commands/{idx}"
4924
+ )
4925
+ file_ref = (
4926
+ ".devlyn/risk-probes.jsonl"
4927
+ if is_risk_probe
4928
+ else ".devlyn/spec-verify.json"
4929
+ )
4400
4930
 
4401
4931
  try:
4402
- proc = subprocess.run(
4403
- cmd,
4404
- cwd=str(work),
4405
- shell=True,
4406
- env=verify_env,
4407
- capture_output=True,
4408
- text=True,
4409
- timeout=timeout_sec,
4932
+ entry = capture_mechanical_command(
4933
+ runner, work, manifest_path, evidence_run_id, evidence_phase,
4934
+ evidence_round, obligation,
4410
4935
  )
4411
- # Mirror run-fixture.sh post-run verifier: combined stdout+stderr.
4412
- out = (proc.stdout or "") + (proc.stderr or "")
4413
- ok_exit = proc.returncode == expected_exit
4414
- ok_contains = all(s in out for s in stdout_contains)
4415
- ok_not = not any(s in out for s in stdout_not_contains)
4416
- passed = bool(ok_exit and ok_contains and ok_not)
4417
-
4418
- if passed:
4419
- reason = None
4420
- elif not ok_exit:
4421
- reason = "exit"
4422
- elif not ok_contains:
4423
- reason = "missing_contains"
4424
- else:
4425
- reason = "unexpected_text"
4426
-
4936
+ stdout = (work / entry["stdout"]["path"]).read_bytes()
4937
+ stderr = (work / entry["stderr"]["path"]).read_bytes()
4938
+ except (runner.EvidenceError, OSError, UnicodeError, ValueError) as exc:
4939
+ evidence_error = str(exc)
4427
4940
  results.append({
4428
4941
  "index": idx,
4429
4942
  "cmd": cmd,
4430
- "expected_exit": expected_exit,
4431
- "actual_exit": proc.returncode,
4432
- "stdout_contains": stdout_contains,
4433
- "stdout_not_contains": stdout_not_contains,
4434
- "pass": passed,
4435
- "reason": reason,
4436
- "stdout_tail": out[-500:],
4943
+ "pass": False,
4944
+ "reason": "process_evidence_invalid",
4437
4945
  })
4438
-
4439
- if not passed:
4440
- # Construct fine-grained message naming the specific failure.
4441
- if not ok_exit:
4442
- msg = (
4443
- f"Verification command #{idx + 1} failed: expected exit "
4444
- f"{expected_exit}, got {proc.returncode}."
4445
- )
4446
- elif not ok_contains:
4447
- missing = [s for s in stdout_contains if s not in out]
4448
- msg = (
4449
- f"Verification command #{idx + 1} failed: expected "
4450
- f"output to contain {missing!r}."
4451
- )
4452
- else:
4453
- forbidden = [s for s in stdout_not_contains if s in out]
4454
- msg = (
4455
- f"Verification command #{idx + 1} failed: output "
4456
- f"contained forbidden literal(s) {forbidden!r}."
4457
- )
4458
-
4459
- fix_hint = (
4460
- f"See .devlyn/spec-verify.results.json for the captured "
4461
- f"output. Update implementation so `{cmd}` matches the "
4462
- f"contract (exit_code={expected_exit}, "
4463
- f"contains={stdout_contains}, not_contains={stdout_not_contains})."
4464
- )
4465
-
4466
- rule_id = (
4467
- "correctness.risk-probe-failed"
4468
- if is_risk_probe
4469
- else "correctness.spec-literal-mismatch"
4470
- )
4471
- criterion_ref = (
4472
- f"risk-probe:{vc.get('id')}"
4473
- if is_risk_probe
4474
- else f"spec-verify://verification_commands/{idx}"
4475
- )
4476
- file_ref = (
4477
- ".devlyn/risk-probes.jsonl"
4478
- if is_risk_probe
4479
- else ".devlyn/spec-verify.json"
4480
- )
4481
- if is_risk_probe:
4482
- fix_hint = (
4483
- f"Risk probe `{vc.get('id')}` derived from "
4484
- f"{vc.get('derived_from')!r} failed. See "
4485
- ".devlyn/spec-verify.results.json for captured output "
4486
- "and update the implementation to satisfy the visible "
4487
- "verification bullet."
4488
- )
4489
-
4490
- findings.append({
4491
- "id": f"{output_finding_prefix()}-{finding_seq:04d}",
4492
- "rule_id": rule_id,
4493
- "level": "error",
4494
- "severity": "CRITICAL",
4495
- "confidence": 1.0,
4496
- "message": msg,
4497
- "file": file_ref,
4498
- "line": 1,
4499
- "phase": output_phase(),
4500
- "criterion_ref": criterion_ref,
4501
- "fix_hint": fix_hint,
4502
- "blocking": True,
4503
- "status": "open",
4504
- })
4505
- finding_seq += 1
4506
-
4507
- except subprocess.TimeoutExpired:
4508
- results.append({"index": idx, "cmd": cmd, "pass": False,
4509
- "reason": "timeout", "timeout_sec": timeout_sec})
4510
4946
  findings.append({
4511
4947
  "id": f"{output_finding_prefix()}-{finding_seq:04d}",
4512
- "rule_id": "correctness.verification-timeout",
4948
+ "rule_id": "invariant.process-evidence-invalid",
4513
4949
  "level": "error",
4514
4950
  "severity": "CRITICAL",
4515
4951
  "confidence": 1.0,
4516
- "message": (
4517
- f"Verification command #{idx + 1} timed out after {timeout_sec}s "
4518
- f"(timeout_sec={timeout_sec}, maximum 600)."
4519
- ),
4520
- "file": ".devlyn/risk-probes.jsonl" if vc.get("_risk_probe") else ".devlyn/spec-verify.json",
4952
+ "message": f"MECHANICAL process evidence is invalid: {exc}.",
4953
+ "file": manifest_relative,
4521
4954
  "line": 1,
4522
4955
  "phase": output_phase(),
4523
- "criterion_ref": (
4524
- f"risk-probe:{vc.get('id')}"
4525
- if vc.get("_risk_probe")
4526
- else f"spec-verify://verification_commands/{idx}"
4527
- ),
4956
+ "criterion_ref": "process-evidence://mechanical",
4528
4957
  "fix_hint": (
4529
- f"Command `{cmd}` exceeded its {timeout_sec}s timeout_sec budget. "
4530
- "Increase timeout_sec up to 600 when the verification legitimately "
4531
- "needs more time, or fix a hang in the implementation."
4958
+ "Preserve the existing manifest and raw streams, then fix the "
4959
+ "runner identity or evidence mutation before rerunning this phase."
4532
4960
  ),
4533
4961
  "blocking": True,
4534
4962
  "status": "open",
4535
4963
  })
4536
4964
  finding_seq += 1
4537
- except Exception as e: # noqa: BLE001 — surface any harness error explicitly
4538
- results.append({"index": idx, "cmd": cmd, "pass": False,
4539
- "reason": f"error:{e.__class__.__name__}:{e}"})
4540
- rule_id = (
4541
- "correctness.risk-probe-failed"
4542
- if vc.get("_risk_probe")
4543
- else "correctness.spec-literal-mismatch"
4544
- )
4965
+ break
4966
+
4967
+ outcome = entry["outcome"]
4968
+ classification = entry["classification"]
4969
+ combined = stdout + stderr
4970
+ actual_exit = outcome["exit_code"] if outcome["kind"] == "exit" else None
4971
+ ok_exit = outcome["kind"] == "exit" and actual_exit == expected_exit
4972
+ ok_contains = all(s.encode("utf-8") in combined for s in stdout_contains)
4973
+ ok_not = not any(s.encode("utf-8") in combined for s in stdout_not_contains)
4974
+ passed = entry["expectation_met"]
4975
+ if classification["kind"] == "capability_denied":
4976
+ reason = "capability_denied"
4977
+ capability_denials.append({
4978
+ "cmd": cmd,
4979
+ "operation": classification["operation"],
4980
+ "evidence_id": entry["id"],
4981
+ })
4982
+ elif passed:
4983
+ reason = None
4984
+ elif outcome["kind"] == "timeout":
4985
+ reason = "timeout"
4986
+ elif not ok_exit:
4987
+ reason = "exit"
4988
+ elif not ok_contains:
4989
+ reason = "missing_contains"
4990
+ else:
4991
+ reason = "unexpected_text"
4992
+
4993
+ results.append({
4994
+ "index": idx,
4995
+ "cmd": cmd,
4996
+ "expected_exit": expected_exit,
4997
+ "actual_exit": actual_exit,
4998
+ "timeout_sec": timeout_sec,
4999
+ "stdout_contains": stdout_contains,
5000
+ "stdout_not_contains": stdout_not_contains,
5001
+ "pass": passed,
5002
+ "reason": reason,
5003
+ "evidence_id": entry["id"],
5004
+ "outcome": outcome,
5005
+ "classification": classification,
5006
+ "stdout": entry["stdout"],
5007
+ "stderr": entry["stderr"],
5008
+ })
5009
+
5010
+ if classification["kind"] == "capability_denied" or passed:
5011
+ continue
5012
+
5013
+ if outcome["kind"] == "timeout":
4545
5014
  findings.append({
4546
5015
  "id": f"{output_finding_prefix()}-{finding_seq:04d}",
4547
- "rule_id": rule_id,
5016
+ "rule_id": "correctness.verification-timeout",
4548
5017
  "level": "error",
4549
5018
  "severity": "CRITICAL",
4550
5019
  "confidence": 1.0,
4551
5020
  "message": (
4552
- f"Verification command #{idx + 1} raised "
4553
- f"{e.__class__.__name__}: {e}."
5021
+ f"Verification command #{idx + 1} timed out after {timeout_sec}s "
5022
+ f"(timeout_sec={timeout_sec}, maximum 600)."
4554
5023
  ),
4555
- "file": ".devlyn/risk-probes.jsonl" if vc.get("_risk_probe") else ".devlyn/spec-verify.json",
5024
+ "file": file_ref,
4556
5025
  "line": 1,
4557
5026
  "phase": output_phase(),
4558
- "criterion_ref": (
4559
- f"risk-probe:{vc.get('id')}"
4560
- if vc.get("_risk_probe")
4561
- else f"spec-verify://verification_commands/{idx}"
4562
- ),
5027
+ "criterion_ref": criterion_ref,
4563
5028
  "fix_hint": (
4564
- f"Command `{cmd}` could not be executed. Check the work-dir "
4565
- f"state and any environment setup the command requires."
5029
+ f"Command `{cmd}` exceeded its {timeout_sec}s timeout_sec budget. "
5030
+ "Increase timeout_sec up to 600 when the verification legitimately "
5031
+ "needs more time, or fix a hang in the implementation."
4566
5032
  ),
4567
5033
  "blocking": True,
4568
5034
  "status": "open",
4569
5035
  })
4570
5036
  finding_seq += 1
5037
+ continue
5038
+
5039
+ if not ok_exit:
5040
+ actual = (
5041
+ str(actual_exit)
5042
+ if outcome["kind"] == "exit"
5043
+ else f"{outcome['kind']}"
5044
+ + (f" signal {outcome['signal']}" if outcome["kind"] == "signal" else "")
5045
+ )
5046
+ msg = (
5047
+ f"Verification command #{idx + 1} failed: expected exit "
5048
+ f"{expected_exit}, got {actual}."
5049
+ )
5050
+ elif not ok_contains:
5051
+ missing = [s for s in stdout_contains if s.encode("utf-8") not in combined]
5052
+ msg = (
5053
+ f"Verification command #{idx + 1} failed: expected "
5054
+ f"output to contain {missing!r}."
5055
+ )
5056
+ else:
5057
+ forbidden = [s for s in stdout_not_contains if s.encode("utf-8") in combined]
5058
+ msg = (
5059
+ f"Verification command #{idx + 1} failed: output "
5060
+ f"contained forbidden literal(s) {forbidden!r}."
5061
+ )
5062
+
5063
+ rule_id = (
5064
+ "correctness.risk-probe-failed"
5065
+ if is_risk_probe
5066
+ else "correctness.spec-literal-mismatch"
5067
+ )
5068
+ fix_hint = (
5069
+ f"Inspect `{manifest_relative}` entry `{entry['id']}` and its sealed raw "
5070
+ f"streams. Update implementation so `{cmd}` matches the contract "
5071
+ f"(exit_code={expected_exit}, contains={stdout_contains}, "
5072
+ f"not_contains={stdout_not_contains})."
5073
+ )
5074
+ if is_risk_probe:
5075
+ fix_hint = (
5076
+ f"Risk probe `{vc.get('id')}` derived from {vc.get('derived_from')!r} "
5077
+ f"failed. Inspect `{manifest_relative}` entry `{entry['id']}` and "
5078
+ "update the implementation to satisfy the visible verification bullet."
5079
+ )
5080
+
5081
+ findings.append({
5082
+ "id": f"{output_finding_prefix()}-{finding_seq:04d}",
5083
+ "rule_id": rule_id,
5084
+ "level": "error",
5085
+ "severity": "CRITICAL",
5086
+ "confidence": 1.0,
5087
+ "message": msg,
5088
+ "file": file_ref,
5089
+ "line": 1,
5090
+ "phase": output_phase(),
5091
+ "criterion_ref": criterion_ref,
5092
+ "fix_hint": fix_hint,
5093
+ "blocking": True,
5094
+ "status": "open",
5095
+ })
5096
+ finding_seq += 1
4571
5097
 
4572
5098
  expected_findings, finding_seq = expected_contract_findings(
4573
5099
  expected_data,
@@ -4586,7 +5112,40 @@ def main() -> int:
4586
5112
  )
4587
5113
  findings.extend(surface_findings)
4588
5114
 
4589
- results_path.write_text(json.dumps({"commands": results}, indent=2) + "\n")
5115
+ evidence_carrier = None
5116
+ if evidence_error is None and obligations:
5117
+ try:
5118
+ evidence_carrier = runner.validate_manifest(
5119
+ work, manifest_relative, evidence_run_id, evidence_phase,
5120
+ evidence_round, obligations, require_expectations=False,
5121
+ )
5122
+ results = runner.bound_carrier_summary_commands(work, evidence_carrier)
5123
+ except (runner.EvidenceError, OSError, UnicodeError, ValueError) as exc:
5124
+ evidence_error = str(exc)
5125
+ findings.append({
5126
+ "id": f"{output_finding_prefix()}-{finding_seq:04d}",
5127
+ "rule_id": "invariant.process-evidence-invalid",
5128
+ "level": "error",
5129
+ "severity": "CRITICAL",
5130
+ "confidence": 1.0,
5131
+ "message": f"MECHANICAL process evidence validation failed: {exc}.",
5132
+ "file": manifest_relative,
5133
+ "line": 1,
5134
+ "phase": output_phase(),
5135
+ "criterion_ref": "process-evidence://mechanical",
5136
+ "fix_hint": (
5137
+ "Preserve the manifest and raw streams, then correct the "
5138
+ "missing, altered, or path-escaping evidence before rerunning."
5139
+ ),
5140
+ "blocking": True,
5141
+ "status": "open",
5142
+ })
5143
+ finding_seq += 1
5144
+
5145
+ results_path.write_text(json.dumps({
5146
+ "commands": results,
5147
+ "process_evidence": evidence_carrier,
5148
+ }, indent=2) + "\n")
4590
5149
 
4591
5150
  # Append findings (jsonl). BUILD_GATE merge step concatenates this onto
4592
5151
  # build_gate.findings.jsonl; never overwrite the orchestrator's own gate
@@ -4597,6 +5156,15 @@ def main() -> int:
4597
5156
 
4598
5157
  failed = [r for r in results if r.get("pass") is False]
4599
5158
  blocking_findings = [f for f in findings if f.get("severity") in {"CRITICAL", "HIGH"}]
5159
+ if capability_denials:
5160
+ denial = capability_denials[0]
5161
+ print(
5162
+ "BLOCKED:build-env-underprovisioned: "
5163
+ f"{denial['operation']} denied for `{denial['cmd']}`; "
5164
+ f"evidence {manifest_relative}#{denial['evidence_id']}",
5165
+ file=sys.stderr,
5166
+ )
5167
+ return 2
4600
5168
  if failed or blocking_findings:
4601
5169
  print(
4602
5170
  f"[spec-verify] {len(failed)}/{len(results)} command(s) failed; "