devlyn-cli 4.0.1 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/AGENTS.md +11 -95
  2. package/CLAUDE.md +7 -160
  3. package/LICENSE +21 -0
  4. package/README.md +83 -114
  5. package/bin/devlyn.js +330 -472
  6. package/bin/instruction-templates.json +120 -2
  7. package/bin/instructions.js +35 -8
  8. package/config/skills/_shared/adapters/claude.md +8 -85
  9. package/config/skills/_shared/adapters/codex.md +14 -34
  10. package/config/skills/_shared/adapters/omp.md +1 -11
  11. package/config/skills/_shared/codex-monitored.sh +4 -54
  12. package/config/skills/_shared/engine-doctor.sh +18 -58
  13. package/config/skills/_shared/engine-preflight.md +15 -47
  14. package/config/skills/_shared/expected-contract.py +242 -0
  15. package/config/skills/_shared/expected.schema.json +10 -95
  16. package/config/skills/_shared/invocation-receipt.py +26 -757
  17. package/config/skills/_shared/platform-support.py +37 -28
  18. package/config/skills/_shared/role-config.py +100 -421
  19. package/config/skills/_shared/runtime-principles.md +17 -93
  20. package/config/skills/_shared/task-complete.py +490 -285
  21. package/config/skills/_shared/task-completion.md +235 -0
  22. package/config/skills/devlyn-engines/SKILL.md +9 -11
  23. package/config/skills/devlyn-ideate/SKILL.md +57 -107
  24. package/config/skills/devlyn-ideate/references/elicitation.md +18 -99
  25. package/config/skills/devlyn-ideate/references/loop.md +110 -0
  26. package/config/skills/devlyn-ideate/references/package-format.md +128 -0
  27. package/config/skills/devlyn-ideate/scripts/acceptance.py +795 -0
  28. package/config/skills/devlyn-ideate/scripts/queue.py +1780 -0
  29. package/package.json +2 -3
  30. package/config/commit-conventions.md +0 -28
  31. package/config/skills/_shared/adapters/README.md +0 -74
  32. package/config/skills/_shared/adapters/grok.md +0 -100
  33. package/config/skills/_shared/archive_run.py +0 -969
  34. package/config/skills/_shared/codex-config.md +0 -81
  35. package/config/skills/_shared/collect-codex-findings.py +0 -505
  36. package/config/skills/_shared/finish-gate.py +0 -521
  37. package/config/skills/_shared/grok-anchor-guard.py +0 -78
  38. package/config/skills/_shared/judge-output-parser.py +0 -200
  39. package/config/skills/_shared/judge-role-evidence.py +0 -320
  40. package/config/skills/_shared/phase-prompt-render.py +0 -183
  41. package/config/skills/_shared/process-evidence.py +0 -926
  42. package/config/skills/_shared/resolve-bootstrap.py +0 -1410
  43. package/config/skills/_shared/resolve-stop-hook.py +0 -359
  44. package/config/skills/_shared/run-bounded.py +0 -58
  45. package/config/skills/_shared/spec-verify-check.py +0 -5083
  46. package/config/skills/_shared/state-phase-write.py +0 -5989
  47. package/config/skills/_shared/terminal-claim-check.py +0 -777
  48. package/config/skills/_shared/verify-merge-findings.py +0 -3528
  49. package/config/skills/devlyn-design-ui/SKILL.md +0 -650
  50. package/config/skills/devlyn-ideate/references/from-spec-mode.md +0 -61
  51. package/config/skills/devlyn-ideate/references/project-mode.md +0 -80
  52. package/config/skills/devlyn-ideate/references/spec-template.md +0 -121
  53. package/config/skills/devlyn-ideate/references/templates/decision.md +0 -48
  54. package/config/skills/devlyn-ideate/references/templates/roadmap.md +0 -50
  55. package/config/skills/devlyn-ideate/references/templates/vision.md +0 -44
  56. package/config/skills/devlyn-queue/SKILL.md +0 -66
  57. package/config/skills/devlyn-queue/scripts/append.py +0 -94
  58. package/config/skills/devlyn-resolve/SKILL.md +0 -379
  59. package/config/skills/devlyn-resolve/references/free-form-mode.md +0 -71
  60. package/config/skills/devlyn-resolve/references/outer-loop.md +0 -23
  61. package/config/skills/devlyn-resolve/references/phases/build-gate.md +0 -75
  62. package/config/skills/devlyn-resolve/references/phases/cleanup.md +0 -33
  63. package/config/skills/devlyn-resolve/references/phases/implement.md +0 -44
  64. package/config/skills/devlyn-resolve/references/phases/plan.md +0 -39
  65. package/config/skills/devlyn-resolve/references/phases/probe-derive.md +0 -261
  66. package/config/skills/devlyn-resolve/references/phases/surface-close.md +0 -34
  67. package/config/skills/devlyn-resolve/references/phases/verify.md +0 -310
  68. package/config/skills/devlyn-resolve/references/state-schema.md +0 -152
  69. package/config/skills/devlyn-resolve/references/task-completion.md +0 -160
  70. package/config/templates/prompt-templates.md +0 -71
  71. package/config/templates/template-feature.spec.md +0 -255
  72. package/config/templates/template-product-spec.md +0 -680
@@ -1,3528 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Merge VERIFY findings and derive a deterministic verdict.
3
-
4
- VERIFY judges are model-written, but routing on finding severity must be
5
- mechanical. This script reads the known VERIFY JSONL finding files, writes a
6
- merged JSONL artifact, computes source-level and overall verdicts, and can
7
- write the merged verdict back to `.devlyn/pipeline.state.json`.
8
- """
9
-
10
- from __future__ import annotations
11
-
12
- import argparse
13
- import json
14
- import pathlib
15
- import re
16
- import sys
17
- import tempfile
18
- import runpy
19
- from typing import Any
20
-
21
-
22
- JUDGE_OUTPUT_PARSER = runpy.run_path(pathlib.Path(__file__).with_name("judge-output-parser.py"))
23
- PROCESS_EVIDENCE = runpy.run_path(pathlib.Path(__file__).with_name("process-evidence.py"))
24
-
25
-
26
- SOURCE_FILES = (
27
- ("mechanical", "verify-mechanical.findings.jsonl"),
28
- ("judge", "verify.findings.jsonl"),
29
- ("pair_judge", "verify.pair.findings.jsonl"),
30
- ("pair_judge", "verify.pair-judge.findings.jsonl"),
31
- )
32
- REQUIRED_SOURCE_FILES = {
33
- "mechanical": "verify-mechanical.findings.jsonl",
34
- "judge": "verify.findings.jsonl",
35
- }
36
-
37
- VERDICT_RANK = {
38
- "PASS": 0,
39
- "TIMEOUT": 0,
40
- "PASS_WITH_ISSUES": 1,
41
- "FAIL": 2,
42
- "NEEDS_WORK": 2,
43
- "BLOCKED": 3,
44
- }
45
- RANK_VERDICT = {0: "PASS", 1: "PASS_WITH_ISSUES", 2: "NEEDS_WORK", 3: "BLOCKED"}
46
- ALLOWED_PAIR_SKIP_REASONS = {
47
- "user_no_pair",
48
- "mechanical_blocker",
49
- "primary_judge_blocker",
50
- "auto_pair_other_engine_unavailable",
51
- }
52
- KNOWN_PAIR_TRIGGER_REASONS = {
53
- "pair.default",
54
- "mode.verify-only",
55
- "mode.pair-verify",
56
- "complexity.high",
57
- "complexity.large",
58
- "spec.complexity.high",
59
- "spec.complexity.large",
60
- "spec.solo_headroom_hypothesis",
61
- "risk.high",
62
- "risk_probes.enabled",
63
- "risk_probes.present",
64
- "coverage.failed",
65
- "mechanical.warning",
66
- "judge.warning",
67
- }
68
- OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")
69
- BACKTICKED_TEXT_RE = re.compile(r"`[^`\n]+`")
70
- RESERVED_BACKTICK_TERMS = {"solo-headroom hypothesis", "solo_claude", "miss"}
71
- COMMAND_PREFIXES = {
72
- "bash",
73
- "bun",
74
- "cargo",
75
- "git",
76
- "go",
77
- "jest",
78
- "make",
79
- "node",
80
- "npm",
81
- "pnpm",
82
- "printf",
83
- "pytest",
84
- "python",
85
- "python3",
86
- "ruff",
87
- "sh",
88
- "uv",
89
- "vitest",
90
- "yarn",
91
- }
92
-
93
-
94
- def reject_json_constant(token: str) -> None:
95
- raise ValueError(f"invalid JSON numeric constant: {token}")
96
-
97
-
98
- def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict:
99
- result = {}
100
- for key, value in pairs:
101
- if key in result:
102
- raise ValueError(f"duplicate JSON key: {key}")
103
- result[key] = value
104
- return result
105
-
106
-
107
- def loads_strict_json(text: str) -> Any:
108
- return json.loads(
109
- text,
110
- parse_constant=reject_json_constant,
111
- object_pairs_hook=reject_duplicate_keys,
112
- )
113
-
114
-
115
- def rank(verdict: str | None) -> int:
116
- return VERDICT_RANK.get(verdict or "PASS", 0)
117
-
118
-
119
- def worse(a: str | None, b: str | None) -> str:
120
- return RANK_VERDICT[max(rank(a), rank(b))]
121
-
122
-
123
- def is_known_pair_trigger_reason(reason: str) -> bool:
124
- return reason in KNOWN_PAIR_TRIGGER_REASONS
125
-
126
-
127
- def has_known_pair_trigger_reason(reasons: list[str]) -> bool:
128
- return any(is_known_pair_trigger_reason(reason) for reason in reasons)
129
-
130
-
131
- def all_known_pair_trigger_reasons(reasons: list[str]) -> bool:
132
- return all(is_known_pair_trigger_reason(reason) for reason in reasons)
133
-
134
-
135
- def state_uses_default_pair_contract(state: dict[str, Any]) -> bool:
136
- return state.get("version") == "3.0"
137
-
138
-
139
- def finding_rank(finding: dict[str, Any]) -> int:
140
- severity = str(finding.get("severity") or "").upper()
141
- if severity in {"CRITICAL", "HIGH"}:
142
- return 2
143
- if severity == "MEDIUM" and finding.get("verdict_binding") is True:
144
- return 2
145
- if severity in {"LOW", "MEDIUM"}:
146
- return 1
147
- return 0
148
-
149
-
150
- def mechanical_evidence_required(devlyn: pathlib.Path, state: dict[str, Any]) -> bool:
151
- return PROCESS_EVIDENCE["mechanical_evidence_required"](devlyn.parent.resolve(), state)
152
-
153
-
154
- def mechanical_evidence_carrier(devlyn: pathlib.Path) -> dict[str, Any] | None:
155
- state_path = devlyn / "pipeline.state.json"
156
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
157
- if not isinstance(state, dict):
158
- raise ValueError("pipeline.state.json must contain a JSON object")
159
- results_path = devlyn / "spec-verify.results.json"
160
- if not results_path.is_file():
161
- if state.get("version") == "3.0" and mechanical_evidence_required(devlyn, state):
162
- raise ValueError("spec-verify.results.json is required for VERIFY MECHANICAL evidence")
163
- return None
164
- results = loads_strict_json(results_path.read_text(encoding="utf-8"))
165
- if not isinstance(results, dict):
166
- raise ValueError("spec-verify.results.json must contain a JSON object")
167
- commands = results.get("commands")
168
- if not isinstance(commands, list):
169
- raise ValueError("spec-verify.results.json commands must be an array")
170
- carrier = results.get("process_evidence")
171
- if carrier is None:
172
- if commands:
173
- raise ValueError("MECHANICAL commands exist without a process-evidence carrier")
174
- if mechanical_evidence_required(devlyn, state):
175
- raise ValueError("required VERIFY MECHANICAL evidence has no process-evidence carrier")
176
- run_id = state.get("run_id")
177
- phases = state.get("phases")
178
- verify_phase = phases.get("verify") if isinstance(phases, dict) else None
179
- round_ = verify_phase.get("round") if isinstance(verify_phase, dict) else None
180
- if isinstance(run_id, str) and isinstance(round_, int) and not isinstance(round_, bool):
181
- manifest_path = (
182
- devlyn / "process-evidence" / run_id / "verify"
183
- / f"round-{round_}" / "manifest.json"
184
- )
185
- if manifest_path.exists():
186
- raise ValueError("VERIFY manifest exists without a process-evidence carrier")
187
- return None
188
- run_id = state.get("run_id")
189
- phases = state.get("phases")
190
- verify_phase = phases.get("verify") if isinstance(phases, dict) else None
191
- round_ = verify_phase.get("round") if isinstance(verify_phase, dict) else None
192
- if not isinstance(run_id, str) or not run_id:
193
- raise ValueError("state.run_id is required for sealed MECHANICAL evidence")
194
- if isinstance(round_, bool) or not isinstance(round_, int) or round_ < 0:
195
- raise ValueError("phases.verify.round is required for sealed MECHANICAL evidence")
196
- if not isinstance(carrier, dict) or carrier.get("phase") != "verify" or carrier.get("round") != round_:
197
- raise ValueError("MECHANICAL process-evidence carrier does not match the VERIFY round")
198
- PROCESS_EVIDENCE["validate_summary_commands"](
199
- devlyn.parent.resolve(), commands, carrier,
200
- )
201
- expected_prefix = f".devlyn/process-evidence/{run_id}/verify/round-{round_}/"
202
- manifest = carrier.get("manifest")
203
- if not isinstance(manifest, dict) or not str(manifest.get("path", "")).startswith(expected_prefix):
204
- raise ValueError("MECHANICAL process-evidence carrier does not match state.run_id")
205
-
206
- bound = state.get("process_evidence")
207
- if bound is not None:
208
- if not isinstance(bound, list):
209
- raise ValueError("state.process_evidence must be null or an array")
210
- same_round = [
211
- item for item in bound
212
- if isinstance(item, dict)
213
- and item.get("phase") == "verify"
214
- and item.get("round") == round_
215
- ]
216
- if same_round and same_round != [carrier]:
217
- raise ValueError("state-bound VERIFY evidence disagrees with MECHANICAL results")
218
- return carrier
219
-
220
-
221
- def mechanical_evidence_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
222
- try:
223
- mechanical_evidence_carrier(devlyn)
224
- except (PROCESS_EVIDENCE["EvidenceError"], OSError, UnicodeError, ValueError) as exc:
225
- return {
226
- "id": "verify-mechanical-evidence-invalid",
227
- "rule_id": "invariant.process-evidence-invalid",
228
- "severity": "CRITICAL",
229
- "confidence": "high",
230
- "file": "spec-verify.results.json",
231
- "line": 1,
232
- "message": f"Sealed VERIFY MECHANICAL evidence failed rehash before merge: {exc}",
233
- "criterion_ref": "process-evidence://mechanical",
234
- "source": "mechanical",
235
- }
236
- return None
237
-
238
-
239
- def mechanical_evidence_outcome(devlyn: pathlib.Path) -> dict[str, Any] | None:
240
- carrier = mechanical_evidence_carrier(devlyn)
241
- if carrier is None:
242
- return None
243
- return PROCESS_EVIDENCE["bound_carrier_outcome"](devlyn.parent.resolve(), carrier)
244
-
245
-
246
- ROLE_CONFIG = runpy.run_path(pathlib.Path(__file__).with_name("role-config.py"))
247
- JUDGE_ROLE_EVIDENCE = runpy.run_path(pathlib.Path(__file__).with_name("judge-role-evidence.py"))
248
-
249
-
250
- def resolved_primary_engine(state):
251
- return ROLE_CONFIG["primary_engine"](state)
252
-
253
-
254
- def primary_timeout_blocker(id_: str, message: str) -> dict[str, Any]:
255
- return {
256
- "id": id_,
257
- "rule_id": "verify.primary.timeout-contract",
258
- "severity": "CRITICAL",
259
- "confidence": "high",
260
- "file": "verify.primary.timeout.json",
261
- "line": 1,
262
- "message": message,
263
- "criterion_ref": "verify.primary.timeout",
264
- "source": "judge",
265
- }
266
-
267
-
268
- def read_primary_timeout_marker(
269
- devlyn: pathlib.Path,
270
- ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
271
- path = devlyn / "verify.primary.timeout.json"
272
- if not path.is_file():
273
- return None, None
274
- try:
275
- marker = loads_strict_json(path.read_text(encoding="utf-8"))
276
- except (OSError, UnicodeError, ValueError) as exc:
277
- return None, primary_timeout_blocker(
278
- "verify-primary-timeout-marker-malformed",
279
- f"verify.primary.timeout.json is malformed: {exc}",
280
- )
281
- if not isinstance(marker, dict) or set(marker) != {"engine", "budget_seconds"}:
282
- return None, primary_timeout_blocker(
283
- "verify-primary-timeout-marker-malformed",
284
- "verify.primary.timeout.json must contain exactly engine and budget_seconds.",
285
- )
286
- engine = marker["engine"]
287
- budget_seconds = marker["budget_seconds"]
288
- if not isinstance(engine, str) or not engine:
289
- return None, primary_timeout_blocker(
290
- "verify-primary-timeout-marker-malformed",
291
- "verify.primary.timeout.json engine must be a non-empty string.",
292
- )
293
- if (
294
- not isinstance(budget_seconds, int)
295
- or isinstance(budget_seconds, bool)
296
- or budget_seconds != 600
297
- ):
298
- return None, primary_timeout_blocker(
299
- "verify-primary-timeout-budget-mismatch",
300
- "verify.primary.timeout.json budget_seconds must equal 600.",
301
- )
302
- try:
303
- state = loads_strict_json(
304
- (devlyn / "pipeline.state.json").read_text(encoding="utf-8")
305
- )
306
- except (OSError, UnicodeError, ValueError) as exc:
307
- return None, primary_timeout_blocker(
308
- "verify-primary-timeout-engine-mismatch",
309
- f"Cannot authenticate the primary timeout engine from pipeline.state.json: {exc}",
310
- )
311
- try:
312
- expected_engine = resolved_primary_engine(state) if isinstance(state, dict) else None
313
- except ValueError:
314
- expected_engine = None
315
- if (
316
- not isinstance(expected_engine, str)
317
- or not expected_engine
318
- or engine != expected_engine
319
- ):
320
- return None, primary_timeout_blocker(
321
- "verify-primary-timeout-engine-mismatch",
322
- "Primary timeout engine does not match the resolved primary judge.",
323
- )
324
- return {"engine": engine, "budget_seconds": budget_seconds}, None
325
-
326
-
327
- def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str, str | None]]:
328
- findings: list[dict[str, Any]] = []
329
- source_verdicts: dict[str, str | None] = {source: "PASS" for source, _ in SOURCE_FILES}
330
- primary_timeout_marker, primary_timeout_violation = read_primary_timeout_marker(devlyn)
331
- # A verdict must come from a judge that ran: pair_judge stays null until
332
- # spawn evidence exists (a pair findings file or pair stdout). verify.md's
333
- # pair contract records null when no second agent is spawned.
334
- source_verdicts["pair_judge"] = None
335
- for source, name in SOURCE_FILES:
336
- path = devlyn / name
337
- if not path.is_file():
338
- # A verdict-binding mechanical result contractually skips both
339
- # judges. Its absent primary carrier is therefore not evidence a
340
- # dispatched primary judge failed to produce.
341
- if source == "judge" and rank(source_verdicts.get("mechanical")) >= 2:
342
- source_verdicts[source] = None
343
- continue
344
- if source == "judge" and primary_timeout_marker is not None:
345
- continue
346
- if REQUIRED_SOURCE_FILES.get(source) == name:
347
- findings.append({
348
- "id": f"verify-merge-required-source-missing-{source}",
349
- "rule_id": "verify.findings.required-source-missing",
350
- "severity": "CRITICAL",
351
- "confidence": "high",
352
- "file": name,
353
- "line": 1,
354
- "message": f"Required VERIFY {source} findings file is missing: {name}",
355
- "criterion_ref": "verify-merge",
356
- "source": source,
357
- })
358
- source_verdicts[source] = "BLOCKED"
359
- continue
360
- if source_verdicts[source] is None:
361
- source_verdicts[source] = "PASS"
362
- with path.open(encoding="utf-8") as handle:
363
- for line_no, line in enumerate(handle, 1):
364
- raw = line.strip()
365
- if not raw:
366
- continue
367
- try:
368
- item = loads_strict_json(raw)
369
- except ValueError as exc:
370
- blocked = {
371
- "id": f"verify-merge-invalid-json-{name}-{line_no}",
372
- "rule_id": "verify.findings.invalid-json",
373
- "severity": "CRITICAL",
374
- "confidence": "high",
375
- "file": name,
376
- "line": line_no,
377
- "message": f"Invalid JSONL finding: {exc}",
378
- "criterion_ref": "verify-merge",
379
- "source": source,
380
- }
381
- findings.append(blocked)
382
- source_verdicts[source] = "BLOCKED"
383
- continue
384
- if not isinstance(item, dict):
385
- continue
386
- item = dict(item)
387
- item.setdefault("source", source)
388
- findings.append(item)
389
- source_verdicts[source] = worse(
390
- source_verdicts[source], RANK_VERDICT[finding_rank(item)]
391
- )
392
- evidence_violation = mechanical_evidence_violation(devlyn)
393
- if evidence_violation is not None:
394
- findings.append(evidence_violation)
395
- source_verdicts["mechanical"] = "BLOCKED"
396
- else:
397
- outcome = mechanical_evidence_outcome(devlyn)
398
- if outcome is not None and outcome["verdict"] != "PASS":
399
- blocked = outcome["verdict"] == "BLOCKED"
400
- ids = (
401
- [item["id"] for item in outcome["capability_denials"]]
402
- if blocked else outcome["failed_ids"]
403
- )
404
- findings.append({
405
- "id": (
406
- "verify-mechanical-capability-denied"
407
- if blocked else "verify-mechanical-expectation-mismatch"
408
- ),
409
- "rule_id": (
410
- "invariant.build-env-underprovisioned"
411
- if blocked else "invariant.mechanical-expectation-mismatch"
412
- ),
413
- "severity": "CRITICAL" if blocked else "HIGH",
414
- "confidence": "high",
415
- "file": "spec-verify.results.json",
416
- "line": 1,
417
- "message": (
418
- "Sealed VERIFY MECHANICAL evidence records a capability denial: "
419
- if blocked else
420
- "Sealed VERIFY MECHANICAL evidence records failed expectations: "
421
- ) + ",".join(ids),
422
- "criterion_ref": "process-evidence://mechanical",
423
- "source": "mechanical",
424
- "verdict_binding": True,
425
- })
426
- source_verdicts["mechanical"] = outcome["verdict"]
427
- if primary_timeout_violation is not None:
428
- findings.append(primary_timeout_violation)
429
- source_verdicts["judge"] = "BLOCKED"
430
- elif primary_timeout_marker is not None:
431
- findings.append(primary_timeout_blocker(
432
- "verify-primary-timeout",
433
- "Primary JUDGE exceeded its authenticated 600-second wall budget.",
434
- ))
435
- source_verdicts["judge"] = "BLOCKED"
436
- findings.extend(detect_pair_stdout_contract_violations(devlyn, source_verdicts))
437
- pair_summary_path = devlyn / "pair-judge.summary.json"
438
- pair_carrier_exists = any(
439
- (devlyn / name).is_file()
440
- for name in ("verify.pair.findings.jsonl", "verify.pair-judge.findings.jsonl")
441
- )
442
- if pair_carrier_exists and pair_summary_path.is_file():
443
- try:
444
- pair_summary = loads_strict_json(pair_summary_path.read_text(encoding="utf-8"))
445
- except ValueError as exc:
446
- summary_error = f"pair-judge.summary.json is malformed JSON: {exc}"
447
- else:
448
- pair_verdict = pair_summary.get("verdict") if isinstance(pair_summary, dict) else None
449
- summary_error = None
450
- if not isinstance(pair_summary, dict):
451
- summary_error = "pair-judge.summary.json must be a JSON object."
452
- elif not isinstance(pair_verdict, str) or pair_verdict not in {
453
- "PASS", "PASS_WITH_ISSUES", "FAIL", "NEEDS_WORK", "BLOCKED"
454
- }:
455
- summary_error = (
456
- "pair-judge.summary.json verdict must be PASS, PASS_WITH_ISSUES, "
457
- "FAIL, NEEDS_WORK, or BLOCKED."
458
- )
459
- if summary_error is not None:
460
- findings.append(
461
- pair_blocker(
462
- "verify-pair-summary-invalid",
463
- summary_error,
464
- pair_summary_path.name,
465
- )
466
- )
467
- source_verdicts["pair_judge"] = "BLOCKED"
468
- elif (
469
- source_verdicts["pair_judge"] != "TIMEOUT"
470
- or rank(pair_verdict) > rank("TIMEOUT")
471
- ):
472
- source_verdicts["pair_judge"] = worse(
473
- source_verdicts["pair_judge"], pair_verdict
474
- )
475
- if (devlyn / "pipeline.state.json").is_file():
476
- try:
477
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
478
- required = JUDGE_ROLE_EVIDENCE["required_roles"](state)
479
- if rank(source_verdicts.get("mechanical")) >= 2:
480
- required = []
481
- for role in required:
482
- source = "judge" if role == "primary_judge" else "pair_judge"
483
- if source_verdicts.get(source) in {"BLOCKED", "TIMEOUT"}:
484
- continue
485
- try:
486
- JUDGE_ROLE_EVIDENCE["authenticate"](devlyn, state, role)
487
- except (ValueError, OSError, TypeError, KeyError) as exc:
488
- source_verdicts[source] = "BLOCKED"
489
- findings.append({**primary_timeout_blocker("verify-role-evidence-invalid", str(exc)),
490
- "source": source, "rule_id": "verify.role-evidence"})
491
- except (ValueError, OSError, TypeError, KeyError) as exc:
492
- source_verdicts["judge"] = "BLOCKED"
493
- findings.append({**primary_timeout_blocker("verify-role-resolution-invalid", str(exc)),
494
- "rule_id": "verify.role-resolution"})
495
- return findings, source_verdicts
496
-
497
-
498
- def has_pair_findings(devlyn: pathlib.Path) -> bool:
499
- for name in ("verify.pair.findings.jsonl", "verify.pair-judge.findings.jsonl"):
500
- path = devlyn / name
501
- if path.is_file() and path.read_text(encoding="utf-8").strip():
502
- return True
503
- return False
504
-
505
-
506
- def canonical_other_judge_stdout(devlyn: pathlib.Path, primary_engine: str) -> list[pathlib.Path]:
507
- adapters = pathlib.Path(__file__).with_name("adapters")
508
- engine_names = {
509
- path.stem for path in adapters.glob("*.md")
510
- if path.stem != "README"
511
- }
512
- return sorted(
513
- devlyn / f"{engine}-judge.stdout"
514
- for engine in engine_names
515
- if engine != primary_engine and (devlyn / f"{engine}-judge.stdout").is_file()
516
- )
517
-
518
-
519
- def pair_capture_exists(devlyn: pathlib.Path, stdout_paths: list[pathlib.Path]) -> bool:
520
- return bool(stdout_paths) or has_pair_findings(devlyn) or any(
521
- (devlyn / name).is_file()
522
- for name in (
523
- "verify.pair.timeout.json",
524
- "pair-judge.summary.json",
525
- )
526
- )
527
-
528
-
529
- def pair_trigger_status(devlyn: pathlib.Path) -> tuple[bool, dict[str, Any] | None]:
530
- state_path = devlyn / "pipeline.state.json"
531
- if not state_path.is_file():
532
- return False, None
533
- try:
534
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
535
- except ValueError:
536
- return False, {
537
- "id": "verify-pair-trigger-state-malformed",
538
- "message": "pipeline.state.json is malformed; cannot verify pair_trigger contract.",
539
- "file": "pipeline.state.json",
540
- }
541
- phases = state.get("phases") if isinstance(state, dict) else {}
542
- verify_phase = phases.get("verify") if isinstance(phases, dict) else None
543
- trigger = None
544
- if isinstance(verify_phase, dict):
545
- trigger = verify_phase.get("pair_trigger")
546
- if trigger is None and isinstance(state, dict):
547
- verify_state = state.get("verify")
548
- if isinstance(verify_state, dict):
549
- trigger = verify_state.get("pair_trigger")
550
- if trigger is None:
551
- return False, None
552
- if not isinstance(trigger, dict):
553
- return False, {
554
- "id": "verify-pair-trigger-malformed",
555
- "message": "pair_trigger must be an object.",
556
- "file": "pipeline.state.json",
557
- }
558
- eligible = trigger.get("eligible")
559
- if not isinstance(eligible, bool):
560
- return False, {
561
- "id": "verify-pair-trigger-eligible-malformed",
562
- "message": "pair_trigger.eligible must be a boolean.",
563
- "file": "pipeline.state.json",
564
- }
565
- reasons = trigger.get("reasons")
566
- if not isinstance(reasons, list) or not all(isinstance(item, str) for item in reasons):
567
- return False, {
568
- "id": "verify-pair-trigger-reasons-malformed",
569
- "message": "pair_trigger.reasons must be a list of strings.",
570
- "file": "pipeline.state.json",
571
- }
572
- skipped_reason = trigger.get("skipped_reason")
573
- if skipped_reason is not None and not isinstance(skipped_reason, str):
574
- return False, {
575
- "id": "verify-pair-trigger-skipped-reason-malformed",
576
- "message": "pair_trigger.skipped_reason must be a string or null.",
577
- "file": "pipeline.state.json",
578
- }
579
- if eligible is True and not reasons:
580
- return False, {
581
- "id": "verify-pair-trigger-reasons-empty",
582
- "message": "pair_trigger.eligible cannot be true with an empty reasons list.",
583
- "file": "pipeline.state.json",
584
- }
585
- if eligible is True and not has_known_pair_trigger_reason(reasons):
586
- return False, {
587
- "id": "verify-pair-trigger-reasons-unknown",
588
- "message": "pair_trigger.reasons must include a known pair-trigger reason.",
589
- "file": "pipeline.state.json",
590
- }
591
- if eligible is True and not all_known_pair_trigger_reasons(reasons):
592
- return False, {
593
- "id": "verify-pair-trigger-reasons-unknown",
594
- "message": "pair_trigger.reasons must only include known pair-trigger reasons.",
595
- "file": "pipeline.state.json",
596
- }
597
- if eligible is True and skipped_reason is not None:
598
- return False, {
599
- "id": "verify-pair-trigger-skip-contradiction",
600
- "message": "pair_trigger.eligible cannot be true while skipped_reason is set.",
601
- "file": "pipeline.state.json",
602
- }
603
- if eligible is False and reasons:
604
- return False, {
605
- "id": "verify-pair-trigger-ineligible-reasons",
606
- "message": "pair_trigger.reasons must be empty when pair_trigger.eligible is false.",
607
- "file": "pipeline.state.json",
608
- }
609
- return eligible is True and len(reasons) > 0, None
610
-
611
-
612
- def pair_trigger_required(devlyn: pathlib.Path) -> bool:
613
- required, _malformed = pair_trigger_status(devlyn)
614
- return required
615
-
616
-
617
- def pair_trigger_present(devlyn: pathlib.Path) -> bool:
618
- state_path = devlyn / "pipeline.state.json"
619
- if not state_path.is_file():
620
- return False
621
- try:
622
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
623
- except ValueError:
624
- return False
625
- phases = state.get("phases") if isinstance(state, dict) else {}
626
- verify_phase = phases.get("verify") if isinstance(phases, dict) else None
627
- if isinstance(verify_phase, dict) and "pair_trigger" in verify_phase:
628
- return True
629
- if isinstance(state, dict):
630
- verify_state = state.get("verify")
631
- if isinstance(verify_state, dict) and "pair_trigger" in verify_state:
632
- return True
633
- return False
634
-
635
-
636
- def pair_flag_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
637
- state_path = devlyn / "pipeline.state.json"
638
- if not state_path.is_file():
639
- return None
640
- try:
641
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
642
- except ValueError:
643
- return None
644
- if not isinstance(state, dict) or state.get("pair_verify") is not True:
645
- return None
646
- risk_profile = state.get("risk_profile")
647
- if isinstance(risk_profile, dict) and risk_profile.get("pair_default_enabled") is False:
648
- return {
649
- "id": "verify-pair-trigger-conflicting-pair-flags",
650
- "message": "--pair-verify and --no-pair are mutually exclusive.",
651
- "file": "pipeline.state.json",
652
- }
653
- return None
654
-
655
-
656
- def risk_profile_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
657
- state_path = devlyn / "pipeline.state.json"
658
- if not state_path.is_file():
659
- return None
660
- try:
661
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
662
- except ValueError:
663
- return None
664
- if not isinstance(state, dict) or "risk_profile" not in state:
665
- return None
666
- risk_profile = state.get("risk_profile")
667
- if not isinstance(risk_profile, dict):
668
- return {
669
- "id": "verify-risk-profile-malformed",
670
- "message": "risk_profile must be an object.",
671
- "file": "pipeline.state.json",
672
- }
673
- for field in ("high_risk", "risk_probes_enabled", "pair_default_enabled"):
674
- if field in risk_profile and not isinstance(risk_profile.get(field), bool):
675
- return {
676
- "id": "verify-risk-profile-malformed",
677
- "message": f"risk_profile.{field} must be a boolean.",
678
- "file": "pipeline.state.json",
679
- }
680
- reasons = risk_profile.get("reasons")
681
- if "reasons" in risk_profile and (
682
- not isinstance(reasons, list) or not all(isinstance(item, str) for item in reasons)
683
- ):
684
- return {
685
- "id": "verify-risk-profile-malformed",
686
- "message": "risk_profile.reasons must be a list of strings.",
687
- "file": "pipeline.state.json",
688
- }
689
- return None
690
-
691
-
692
- def verify_state_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
693
- state_path = devlyn / "pipeline.state.json"
694
- if not state_path.is_file():
695
- return {
696
- "id": "verify-state-missing",
697
- "rule_id": "verify.state.missing",
698
- "message": "pipeline.state.json is required before VERIFY merge.",
699
- "file": "pipeline.state.json",
700
- }
701
- try:
702
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
703
- except ValueError:
704
- return {
705
- "id": "verify-pair-trigger-state-malformed",
706
- "rule_id": "verify.pair.emission-contract",
707
- "message": "pipeline.state.json is malformed; cannot verify pair_trigger contract.",
708
- "file": "pipeline.state.json",
709
- }
710
- if not isinstance(state, dict):
711
- return {
712
- "id": "verify-state-malformed",
713
- "rule_id": "verify.state.malformed",
714
- "message": "pipeline.state.json must be a JSON object before VERIFY merge.",
715
- "file": "pipeline.state.json",
716
- }
717
- try:
718
- engine = resolved_primary_engine(state)
719
- except ValueError:
720
- engine = None
721
- if not isinstance(engine, str) or not engine.strip():
722
- rule = "verify.state.engine-malformed"
723
- return {
724
- "id": rule,
725
- "rule_id": rule,
726
- "message": "pipeline.state.json requires a valid primary engine before VERIFY merge.",
727
- "file": "pipeline.state.json",
728
- }
729
- if not state_uses_default_pair_contract(state):
730
- return None
731
- source = state.get("source")
732
- if not isinstance(source, dict) or source.get("type") != "generated":
733
- return None
734
- goal_path = source.get("goal_path")
735
- goal_sha256 = source.get("goal_sha256")
736
- if (
737
- not isinstance(goal_path, str)
738
- or not goal_path
739
- or not isinstance(goal_sha256, str)
740
- or re.fullmatch(r"[0-9a-f]{64}", goal_sha256) is None
741
- ):
742
- rule = "verify.state.goal-persistence-missing"
743
- return {
744
- "id": rule,
745
- "rule_id": rule,
746
- "message": (
747
- "schema-v3 generated runs require source.goal_path as a non-empty string "
748
- "and source.goal_sha256 as 64 lowercase hexadecimal characters."
749
- ),
750
- "file": "pipeline.state.json",
751
- }
752
- if state.get("complexity") in {"trivial", "medium"}:
753
- phases = state.get("phases")
754
- surface_close = phases.get("surface_close") if isinstance(phases, dict) else None
755
- skipped = (
756
- isinstance(surface_close, dict)
757
- and surface_close.get("verdict") is None
758
- and surface_close.get("started_at") is None
759
- and surface_close.get("skipped_reason") == "auto_surface_close_claude_unavailable"
760
- )
761
- if (
762
- not isinstance(surface_close, dict)
763
- or (surface_close.get("verdict") is None and not skipped)
764
- ):
765
- rule = "verify.state.surface-close-skipped"
766
- return {
767
- "id": rule,
768
- "rule_id": rule,
769
- "message": (
770
- "schema-v3 generated trivial/medium runs require completed SURFACE_CLOSE "
771
- "or its canonical automatic Claude-unavailable skip before VERIFY merge."
772
- ),
773
- "file": "pipeline.state.json",
774
- }
775
- return None
776
-
777
-
778
- def source_spec_text(state: dict[str, Any]) -> str | None:
779
- source = state.get("source") if isinstance(state.get("source"), dict) else {}
780
- for key in ("spec_path", "criteria_path"):
781
- raw_path = source.get(key)
782
- if not isinstance(raw_path, str) or not raw_path:
783
- continue
784
- path = pathlib.Path(raw_path)
785
- if not path.is_absolute():
786
- path = pathlib.Path.cwd() / path
787
- try:
788
- return path.read_text(encoding="utf-8")
789
- except OSError:
790
- continue
791
- return None
792
-
793
-
794
- def spec_frontmatter_complexity(state: dict[str, Any]) -> str | None:
795
- text = source_spec_text(state)
796
- if text is None:
797
- return None
798
- if not text.startswith("---"):
799
- return None
800
- end = text.find("\n---", 3)
801
- if end == -1:
802
- return None
803
- for line in text[3:end].splitlines():
804
- match = re.match(r"\s*complexity\s*:\s*[\"']?([A-Za-z_-]+)", line)
805
- if match:
806
- return match.group(1).lower()
807
- return None
808
-
809
-
810
- def spec_has_solo_headroom_hypothesis(state: dict[str, Any]) -> bool:
811
- text = source_spec_text(state)
812
- if text is None:
813
- return False
814
- lower = text.lower()
815
- return (
816
- "solo-headroom hypothesis" in lower
817
- and "solo_claude" in lower
818
- and "miss" in lower
819
- and has_backticked_observable_command(text)
820
- )
821
-
822
-
823
- def has_backticked_observable_command(text: str) -> bool:
824
- for line in text.splitlines():
825
- lower = line.lower()
826
- if "miss" not in lower or not any(marker in lower for marker in OBSERVABLE_COMMAND_MARKERS):
827
- continue
828
- if any(is_command_like_backtick(match.group(0).strip("`")) for match in BACKTICKED_TEXT_RE.finditer(line)):
829
- return True
830
- return False
831
-
832
-
833
- def is_command_like_backtick(value: str) -> bool:
834
- stripped = value.strip()
835
- lower = stripped.lower()
836
- if not stripped or lower in RESERVED_BACKTICK_TERMS:
837
- return False
838
- first = lower.split(maxsplit=1)[0]
839
- return (
840
- first in COMMAND_PREFIXES
841
- or any(marker in stripped for marker in ("/", "$", "=", "|", "&&", ";"))
842
- or stripped.endswith((".js", ".py", ".sh"))
843
- )
844
-
845
-
846
- def state_pair_trigger_reasons(
847
- devlyn: pathlib.Path,
848
- source_verdicts: dict[str, str | None],
849
- ) -> list[str]:
850
- state_path = devlyn / "pipeline.state.json"
851
- if not state_path.is_file():
852
- return []
853
- try:
854
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
855
- except ValueError:
856
- return []
857
- if not isinstance(state, dict):
858
- return []
859
- phases = state.get("phases") if isinstance(state.get("phases"), dict) else {}
860
- verify_phase = phases.get("verify") if isinstance(phases, dict) else {}
861
- verify_state = state.get("verify") if isinstance(state.get("verify"), dict) else {}
862
- risk_profile = state.get("risk_profile") if isinstance(state.get("risk_profile"), dict) else {}
863
- reasons: list[str] = []
864
- if (
865
- state_uses_default_pair_contract(state)
866
- and risk_profile.get("pair_default_enabled") is not False
867
- ):
868
- reasons.append("pair.default")
869
- if state.get("mode") == "verify-only":
870
- reasons.append("mode.verify-only")
871
- if state.get("pair_verify") is True:
872
- reasons.append("mode.pair-verify")
873
- if state.get("complexity") in {"high", "large"}:
874
- reasons.append(f"complexity.{state.get('complexity')}")
875
- spec_complexity = spec_frontmatter_complexity(state)
876
- if spec_complexity in {"high", "large"}:
877
- reasons.append(f"spec.complexity.{spec_complexity}")
878
- if spec_has_solo_headroom_hypothesis(state):
879
- reasons.append("spec.solo_headroom_hypothesis")
880
- if risk_profile.get("high_risk") is True:
881
- reasons.append("risk.high")
882
- if risk_profile.get("risk_probes_enabled") is True:
883
- reasons.append("risk_probes.enabled")
884
- if (devlyn / "risk-probes.jsonl").is_file():
885
- reasons.append("risk_probes.present")
886
- coverage_failed = False
887
- if isinstance(verify_state, dict) and verify_state.get("coverage_failed") is True:
888
- coverage_failed = True
889
- if isinstance(verify_phase, dict) and verify_phase.get("coverage_failed") is True:
890
- coverage_failed = True
891
- if coverage_failed:
892
- reasons.append("coverage.failed")
893
- if rank(source_verdicts.get("mechanical")) == 1:
894
- reasons.append("mechanical.warning")
895
- if rank(source_verdicts.get("judge")) == 1:
896
- reasons.append("judge.warning")
897
- return reasons
898
-
899
-
900
- def outcome_independent_reasons(devlyn: pathlib.Path) -> list[str]:
901
- return [
902
- reason
903
- for reason in state_pair_trigger_reasons(devlyn, {})
904
- if reason not in ("coverage.failed", "mechanical.warning", "judge.warning")
905
- ]
906
-
907
-
908
- def pair_trigger_missing_contract_violation(
909
- devlyn: pathlib.Path,
910
- source_verdicts: dict[str, str | None],
911
- ) -> dict[str, Any] | None:
912
- if rank(source_verdicts.get("mechanical")) >= 2:
913
- return None
914
- reasons = state_pair_trigger_reasons(devlyn, source_verdicts)
915
- if not reasons:
916
- return None
917
- return {
918
- "id": "verify-pair-trigger-required-missing",
919
- "message": (
920
- "pair_trigger is missing even though VERIFY state requires a pair decision: "
921
- + ", ".join(reasons)
922
- ),
923
- "file": "pipeline.state.json",
924
- }
925
-
926
-
927
- def pair_trigger_skip_contract_violation(
928
- devlyn: pathlib.Path,
929
- source_verdicts: dict[str, str | None],
930
- ) -> dict[str, Any] | None:
931
- state_path = devlyn / "pipeline.state.json"
932
- if not state_path.is_file():
933
- return None
934
- try:
935
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
936
- except ValueError:
937
- return None
938
- phases = state.get("phases") if isinstance(state, dict) else {}
939
- verify_phase = phases.get("verify") if isinstance(phases, dict) else None
940
- trigger = None
941
- if isinstance(verify_phase, dict):
942
- trigger = verify_phase.get("pair_trigger")
943
- if trigger is None and isinstance(state, dict):
944
- verify_state = state.get("verify")
945
- if isinstance(verify_state, dict):
946
- trigger = verify_state.get("pair_trigger")
947
- if not isinstance(trigger, dict):
948
- return None
949
- skipped_reason = trigger.get("skipped_reason")
950
- if trigger.get("eligible") is False and skipped_reason is None:
951
- natural_reasons = state_pair_trigger_reasons(devlyn, source_verdicts)
952
- if natural_reasons:
953
- return {
954
- "id": "verify-pair-trigger-ineligible-unjustified",
955
- "message": (
956
- "pair_trigger is ineligible without a skip reason even though "
957
- "VERIFY state requires a pair decision: "
958
- + ", ".join(natural_reasons)
959
- ),
960
- "file": "pipeline.state.json",
961
- }
962
- if skipped_reason is None:
963
- return None
964
- if skipped_reason not in ALLOWED_PAIR_SKIP_REASONS:
965
- return {
966
- "id": "verify-pair-trigger-skipped-reason-unsupported",
967
- "message": (
968
- "pair_trigger.skipped_reason must be user_no_pair, "
969
- "mechanical_blocker, primary_judge_blocker, "
970
- "auto_pair_other_engine_unavailable, or null."
971
- ),
972
- "file": "pipeline.state.json",
973
- }
974
- if skipped_reason == "auto_pair_other_engine_unavailable":
975
- # alpha+ capability-gating: an AUTOMATIC pair trigger may skip to solo
976
- # VERIFY when the OTHER engine is unavailable (single-LLM users stay
977
- # first-class). An EXPLICIT --pair-verify route is a promise and must
978
- # BLOCK on an unavailable engine, never skip — enforce that here so the
979
- # auto-skip cannot launder an explicit request.
980
- pair_verify = state.get("pair_verify") if isinstance(state, dict) else None
981
- if pair_verify is True:
982
- return {
983
- "id": "verify-pair-trigger-auto-skip-explicit-conflict",
984
- "message": (
985
- "pair_trigger skipped_reason auto_pair_other_engine_unavailable is "
986
- "only valid for an automatic trigger; an explicit --pair-verify run "
987
- "must BLOCK on an unavailable OTHER engine, not skip."
988
- ),
989
- "file": "pipeline.state.json",
990
- }
991
- if skipped_reason == "user_no_pair":
992
- risk_profile = state.get("risk_profile") if isinstance(state, dict) else {}
993
- if not isinstance(risk_profile, dict) or risk_profile.get("pair_default_enabled") is not False:
994
- return {
995
- "id": "verify-pair-trigger-user-no-pair-unsupported",
996
- "message": (
997
- "pair_trigger skipped_reason user_no_pair requires "
998
- "risk_profile.pair_default_enabled false from an explicit --no-pair opt-out."
999
- ),
1000
- "file": "pipeline.state.json",
1001
- }
1002
- if skipped_reason == "mechanical_blocker" and rank(source_verdicts.get("mechanical")) < 2:
1003
- return {
1004
- "id": "verify-pair-trigger-mechanical-blocker-unsupported",
1005
- "message": (
1006
- "pair_trigger skipped_reason mechanical_blocker requires a "
1007
- "verdict-binding MECHANICAL finding."
1008
- ),
1009
- "file": "pipeline.state.json",
1010
- }
1011
- if skipped_reason == "primary_judge_blocker":
1012
- if state_uses_default_pair_contract(state):
1013
- return {
1014
- "id": "verify-pair-trigger-primary-judge-blocker-retired",
1015
- "message": (
1016
- "pair_trigger skipped_reason primary_judge_blocker is archived-v2.0 "
1017
- "state only; schema-v3 runs must dispatch the pair-JUDGE."
1018
- ),
1019
- "file": "pipeline.state.json",
1020
- }
1021
- if rank(source_verdicts.get("judge")) < 2:
1022
- return {
1023
- "id": "verify-pair-trigger-primary-judge-blocker-unsupported",
1024
- "message": (
1025
- "pair_trigger skipped_reason primary_judge_blocker requires a "
1026
- "verdict-binding primary JUDGE finding."
1027
- ),
1028
- "file": "pipeline.state.json",
1029
- }
1030
- preknown_reasons = outcome_independent_reasons(devlyn)
1031
- if preknown_reasons:
1032
- return {
1033
- "id": "verify-pair-trigger-primary-judge-blocker-preknown",
1034
- "message": (
1035
- "pair_trigger cannot skip the pair-JUDGE for a primary JUDGE blocker "
1036
- "when outcome-independent reasons applied at spawn: "
1037
- + ", ".join(preknown_reasons)
1038
- ),
1039
- "file": "pipeline.state.json",
1040
- }
1041
- return None
1042
-
1043
-
1044
- def pair_trigger_reason_completeness_violation(
1045
- devlyn: pathlib.Path,
1046
- source_verdicts: dict[str, str | None],
1047
- ) -> dict[str, Any] | None:
1048
- if rank(source_verdicts.get("mechanical")) >= 2:
1049
- return None
1050
- state_path = devlyn / "pipeline.state.json"
1051
- if not state_path.is_file():
1052
- return None
1053
- try:
1054
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
1055
- except ValueError:
1056
- return None
1057
- phases = state.get("phases") if isinstance(state, dict) else {}
1058
- verify_phase = phases.get("verify") if isinstance(phases, dict) else None
1059
- trigger = None
1060
- if isinstance(verify_phase, dict):
1061
- trigger = verify_phase.get("pair_trigger")
1062
- if trigger is None and isinstance(state, dict):
1063
- verify_state = state.get("verify")
1064
- if isinstance(verify_state, dict):
1065
- trigger = verify_state.get("pair_trigger")
1066
- if not isinstance(trigger, dict) or trigger.get("eligible") is not True:
1067
- return None
1068
- reasons = trigger.get("reasons")
1069
- if not isinstance(reasons, list) or not all(isinstance(item, str) for item in reasons):
1070
- return None
1071
- missing = [
1072
- reason
1073
- for reason in state_pair_trigger_reasons(devlyn, source_verdicts)
1074
- if reason not in reasons
1075
- ]
1076
- if not missing:
1077
- return None
1078
- return {
1079
- "id": "verify-pair-trigger-reasons-incomplete",
1080
- "message": (
1081
- "pair_trigger.reasons is missing applicable canonical reason(s): "
1082
- + ", ".join(missing)
1083
- ),
1084
- "file": "pipeline.state.json",
1085
- }
1086
-
1087
-
1088
- def pair_blocker(
1089
- id_: str,
1090
- message: str,
1091
- file_: str | None = None,
1092
- rule_id: str = "verify.pair.emission-contract",
1093
- ) -> dict[str, Any]:
1094
- return {
1095
- "id": id_,
1096
- "rule_id": rule_id,
1097
- "severity": "CRITICAL",
1098
- "confidence": "high",
1099
- "file": file_,
1100
- "line": 1 if file_ else None,
1101
- "message": message,
1102
- "criterion_ref": "verify.pair.findings",
1103
- "source": "pair_judge",
1104
- }
1105
-
1106
-
1107
- def read_pair_timeout_marker(devlyn: pathlib.Path) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
1108
- path = devlyn / "verify.pair.timeout.json"
1109
- if not path.is_file():
1110
- return None, None
1111
- try:
1112
- marker = loads_strict_json(path.read_text(encoding="utf-8"))
1113
- except ValueError as exc:
1114
- return None, {
1115
- "id": "verify-pair-timeout-marker-malformed",
1116
- "message": f"verify.pair.timeout.json is malformed JSON: {exc}",
1117
- "file": "verify.pair.timeout.json",
1118
- }
1119
- if not isinstance(marker, dict):
1120
- return None, {
1121
- "id": "verify-pair-timeout-marker-malformed",
1122
- "message": "verify.pair.timeout.json must be a JSON object.",
1123
- "file": "verify.pair.timeout.json",
1124
- }
1125
- engine = marker.get("engine")
1126
- budget_seconds = marker.get("budget_seconds")
1127
- if not isinstance(engine, str) or not engine:
1128
- return None, {
1129
- "id": "verify-pair-timeout-marker-malformed",
1130
- "message": "verify.pair.timeout.json engine must be a non-empty string.",
1131
- "file": "verify.pair.timeout.json",
1132
- }
1133
- if (
1134
- not isinstance(budget_seconds, int)
1135
- or isinstance(budget_seconds, bool)
1136
- or budget_seconds <= 0
1137
- ):
1138
- return None, {
1139
- "id": "verify-pair-timeout-marker-malformed",
1140
- "message": "verify.pair.timeout.json budget_seconds must be a positive integer.",
1141
- "file": "verify.pair.timeout.json",
1142
- }
1143
- return {"engine": engine, "budget_seconds": budget_seconds}, None
1144
-
1145
-
1146
- def is_result_less_message_stream(stdout_text: str) -> bool:
1147
- """True when a declared whole-message stream stops before its terminal result.
1148
-
1149
- The timeout marker attests the killed process. This classifier admits only a
1150
- valid stream prefix, optionally ending in one truncated JSON record.
1151
- """
1152
- lines = [raw for line in stdout_text.splitlines() if (raw := line.strip())]
1153
- if not lines:
1154
- return False
1155
- try:
1156
- first = JUDGE_OUTPUT_PARSER["loads_strict_json"](lines[0])
1157
- except ValueError:
1158
- return False
1159
- if (
1160
- not isinstance(first, dict)
1161
- or first.get("type") != "system"
1162
- or first.get("subtype") != "init"
1163
- or not isinstance(first.get("session_id"), str)
1164
- ):
1165
- return False
1166
- session_id = first["session_id"]
1167
- for index, raw in enumerate(lines[1:], 1):
1168
- try:
1169
- item = JUDGE_OUTPUT_PARSER["loads_strict_json"](raw)
1170
- except ValueError:
1171
- return index == len(lines) - 1 and raw.startswith("{") and not raw.endswith("}")
1172
- if (
1173
- not isinstance(item, dict)
1174
- or item.get("type") not in {"assistant", "user"}
1175
- or item.get("session_id") != session_id
1176
- ):
1177
- return False
1178
- return True
1179
-
1180
-
1181
- def detect_pair_stdout_contract_violations(
1182
- devlyn: pathlib.Path,
1183
- source_verdicts: dict[str, str | None],
1184
- ) -> list[dict[str, Any]]:
1185
- # The primary uses its explicit phase engine, or the legacy executor; only the OTHER engine's
1186
- # capture is pair evidence.
1187
- timeout_marker, timeout_violation = read_pair_timeout_marker(devlyn)
1188
- if timeout_violation is not None:
1189
- source_verdicts["pair_judge"] = "BLOCKED"
1190
- return [
1191
- pair_blocker(
1192
- timeout_violation["id"],
1193
- timeout_violation["message"],
1194
- timeout_violation["file"],
1195
- )
1196
- ]
1197
- flag_violation = pair_flag_contract_violation(devlyn)
1198
- if flag_violation is not None:
1199
- source_verdicts["pair_judge"] = "BLOCKED"
1200
- return [
1201
- pair_blocker(
1202
- flag_violation["id"],
1203
- flag_violation["message"],
1204
- flag_violation["file"],
1205
- )
1206
- ]
1207
- required, malformed_trigger = pair_trigger_status(devlyn)
1208
- if malformed_trigger is not None:
1209
- source_verdicts["pair_judge"] = "BLOCKED"
1210
- return [
1211
- pair_blocker(
1212
- malformed_trigger["id"],
1213
- malformed_trigger["message"],
1214
- malformed_trigger["file"],
1215
- )
1216
- ]
1217
- risk_profile_violation = risk_profile_contract_violation(devlyn)
1218
- if risk_profile_violation is not None:
1219
- source_verdicts["pair_judge"] = "BLOCKED"
1220
- return [
1221
- pair_blocker(
1222
- risk_profile_violation["id"],
1223
- risk_profile_violation["message"],
1224
- risk_profile_violation["file"],
1225
- )
1226
- ]
1227
- state_violation = verify_state_contract_violation(devlyn)
1228
- if state_violation is not None:
1229
- source_verdicts["pair_judge"] = "BLOCKED"
1230
- return [
1231
- pair_blocker(
1232
- state_violation["id"],
1233
- state_violation["message"],
1234
- state_violation["file"],
1235
- state_violation["rule_id"],
1236
- )
1237
- ]
1238
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
1239
- assert isinstance(state, dict)
1240
- engine = resolved_primary_engine(state)
1241
- assert isinstance(engine, str) and engine.strip()
1242
- stdout_paths = canonical_other_judge_stdout(devlyn, engine)
1243
- if not required and not pair_trigger_present(devlyn):
1244
- missing_violation = pair_trigger_missing_contract_violation(devlyn, source_verdicts)
1245
- if missing_violation is not None:
1246
- source_verdicts["pair_judge"] = "BLOCKED"
1247
- return [
1248
- pair_blocker(
1249
- missing_violation["id"],
1250
- missing_violation["message"],
1251
- missing_violation["file"],
1252
- )
1253
- ]
1254
- skip_violation = pair_trigger_skip_contract_violation(devlyn, source_verdicts)
1255
- if skip_violation is not None:
1256
- source_verdicts["pair_judge"] = "BLOCKED"
1257
- return [
1258
- pair_blocker(
1259
- skip_violation["id"],
1260
- skip_violation["message"],
1261
- skip_violation["file"],
1262
- )
1263
- ]
1264
- reason_violation = pair_trigger_reason_completeness_violation(devlyn, source_verdicts)
1265
- if reason_violation is not None:
1266
- source_verdicts["pair_judge"] = "BLOCKED"
1267
- return [
1268
- pair_blocker(
1269
- reason_violation["id"],
1270
- reason_violation["message"],
1271
- reason_violation["file"],
1272
- )
1273
- ]
1274
- trigger_present = pair_trigger_present(devlyn)
1275
- if trigger_present and not required:
1276
- if pair_capture_exists(devlyn, stdout_paths):
1277
- source_verdicts["pair_judge"] = "BLOCKED"
1278
- return [
1279
- pair_blocker(
1280
- "verify-pair-skipped-output-present",
1281
- "Pair state is skipped or ineligible, but a canonical OTHER-judge carrier exists.",
1282
- stdout_paths[0].name if stdout_paths else "verify.pair.findings.jsonl",
1283
- "verify.pair.state-capture-contradiction",
1284
- )
1285
- ]
1286
- source_verdicts["pair_judge"] = None
1287
- return []
1288
- if len(stdout_paths) > 1:
1289
- source_verdicts["pair_judge"] = "BLOCKED"
1290
- return [
1291
- pair_blocker(
1292
- "verify-pair-output-ambiguous",
1293
- "Multiple canonical OTHER-engine stdout carriers exist: "
1294
- + ", ".join(path.name for path in stdout_paths),
1295
- stdout_paths[0].name,
1296
- )
1297
- ]
1298
- if timeout_marker is not None:
1299
- expected_timeout_stdout = f"{timeout_marker['engine']}-judge.stdout"
1300
- if timeout_marker["engine"] == engine or (
1301
- stdout_paths and stdout_paths[0].name != expected_timeout_stdout
1302
- ):
1303
- source_verdicts["pair_judge"] = "BLOCKED"
1304
- return [
1305
- pair_blocker(
1306
- "verify-pair-timeout-engine-mismatch",
1307
- "Pair timeout engine does not match the canonical OTHER-engine carrier.",
1308
- "verify.pair.timeout.json",
1309
- )
1310
- ]
1311
- if not stdout_paths:
1312
- if timeout_marker is not None:
1313
- if rank(source_verdicts["pair_judge"]) <= rank("TIMEOUT"):
1314
- source_verdicts["pair_judge"] = "TIMEOUT"
1315
- return []
1316
- if required:
1317
- source_verdicts["pair_judge"] = "BLOCKED"
1318
- return [
1319
- pair_blocker(
1320
- "verify-pair-required-output-missing",
1321
- "Pair-mode was required, but the pair-JUDGE produced no canonical OTHER-engine stdout.",
1322
- "verify.pair.findings.jsonl",
1323
- )
1324
- ]
1325
- return []
1326
- if has_pair_findings(devlyn):
1327
- return []
1328
- if source_verdicts["pair_judge"] is None and timeout_marker is None:
1329
- source_verdicts["pair_judge"] = "PASS"
1330
- for stdout_path in stdout_paths:
1331
- stdout_text = stdout_path.read_text(encoding="utf-8")
1332
- if not stdout_text.strip():
1333
- if timeout_marker is not None:
1334
- continue
1335
- source_verdicts["pair_judge"] = "BLOCKED"
1336
- return [
1337
- pair_blocker(
1338
- "verify-pair-empty-output",
1339
- f"pair-JUDGE stdout {stdout_path.name} was empty; the bounded contract requires a JSONL finding or PASS line.",
1340
- stdout_path.name,
1341
- )
1342
- ]
1343
- try:
1344
- stdout_findings, stdout_summary = JUDGE_OUTPUT_PARSER["collect_stdout"](stdout_path)
1345
- except SystemExit as exc:
1346
- if timeout_marker is not None and is_result_less_message_stream(stdout_text):
1347
- continue
1348
- source_verdicts["pair_judge"] = "BLOCKED"
1349
- return [
1350
- pair_blocker(
1351
- "verify-pair-emission-contract-violated",
1352
- f"pair-JUDGE stdout {stdout_path.name} violates the shared emission parser: {exc}",
1353
- stdout_path.name,
1354
- )
1355
- ]
1356
- if stdout_findings or stdout_summary is None or stdout_summary["verdict"] != "PASS":
1357
- source_verdicts["pair_judge"] = "BLOCKED"
1358
- return [
1359
- pair_blocker(
1360
- "verify-pair-emission-contract-violated",
1361
- (
1362
- f"pair-JUDGE stdout {stdout_path.name} contained findings or a non-PASS "
1363
- "verdict, but the canonical pair findings JSONL file was empty."
1364
- ),
1365
- stdout_path.name,
1366
- )
1367
- ]
1368
- if timeout_marker is not None:
1369
- source_verdicts["pair_judge"] = "TIMEOUT"
1370
- return []
1371
-
1372
-
1373
- def write_outputs(
1374
- devlyn: pathlib.Path,
1375
- findings: list[dict[str, Any]],
1376
- source_verdicts: dict[str, str | None],
1377
- ) -> dict[str, Any]:
1378
- merged_path = devlyn / "verify-merged.findings.jsonl"
1379
- summary_path = devlyn / "verify-merge.summary.json"
1380
- with merged_path.open("w", encoding="utf-8") as handle:
1381
- for finding in findings:
1382
- handle.write(json.dumps(finding, sort_keys=True, separators=(",", ":")) + "\n")
1383
- verdict = "PASS"
1384
- for source_verdict in source_verdicts.values():
1385
- verdict = worse(verdict, source_verdict)
1386
- summary = {
1387
- "verdict": verdict,
1388
- "source_verdicts": source_verdicts,
1389
- "findings_count": len(findings),
1390
- "findings_file": str(merged_path),
1391
- }
1392
- if source_verdicts.get("pair_judge") == "TIMEOUT":
1393
- timeout_marker, _timeout_violation = read_pair_timeout_marker(devlyn)
1394
- if timeout_marker is not None:
1395
- summary["pair_timeout"] = timeout_marker
1396
- summary["report_header_note"] = "solo verdict after pair TIMEOUT"
1397
- summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
1398
- return summary
1399
-
1400
-
1401
- def write_state(devlyn: pathlib.Path, summary: dict[str, Any]) -> None:
1402
- state_path = devlyn / "pipeline.state.json"
1403
- if not state_path.is_file():
1404
- raise SystemExit(f"error: {state_path} not found")
1405
- state = loads_strict_json(state_path.read_text(encoding="utf-8"))
1406
- try:
1407
- carrier = mechanical_evidence_carrier(devlyn)
1408
- except (PROCESS_EVIDENCE["EvidenceError"], OSError, UnicodeError, ValueError) as exc:
1409
- if summary.get("source_verdicts", {}).get("mechanical") != "BLOCKED":
1410
- raise SystemExit(f"error: invalid MECHANICAL process evidence: {exc}") from exc
1411
- carrier = None
1412
- if carrier is not None:
1413
- bound = state.get("process_evidence")
1414
- if bound is None:
1415
- bound = []
1416
- elif not isinstance(bound, list):
1417
- raise SystemExit("error: state.process_evidence must be null or an array")
1418
- same_round = [
1419
- item for item in bound
1420
- if isinstance(item, dict)
1421
- and item.get("phase") == carrier["phase"]
1422
- and item.get("round") == carrier["round"]
1423
- ]
1424
- if same_round and same_round != [carrier]:
1425
- raise SystemExit("error: state-bound VERIFY evidence disagrees with MECHANICAL results")
1426
- if not same_round:
1427
- bound.append(carrier)
1428
- state["process_evidence"] = bound
1429
- phases = state.setdefault("phases", {})
1430
- verify = phases.get("verify")
1431
- if not isinstance(verify, dict):
1432
- verify = {}
1433
- phases["verify"] = verify
1434
- role_evidence = {}
1435
- try:
1436
- required_roles = JUDGE_ROLE_EVIDENCE["required_roles"](state)
1437
- except ValueError as exc:
1438
- if summary.get("source_verdicts", {}).get("judge") != "BLOCKED":
1439
- raise SystemExit(str(exc)) from exc
1440
- required_roles = [] # Persist the invalid-resolution failure; no legacy dispatch.
1441
- for role in required_roles:
1442
- source = "judge" if role == "primary_judge" else "pair_judge"
1443
- if summary.get("source_verdicts", {}).get(source) in {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK"}:
1444
- role_evidence[role] = JUDGE_ROLE_EVIDENCE["authenticate"](devlyn, state, role)
1445
- if role_evidence:
1446
- verify["role_evidence"] = role_evidence
1447
- verify["verdict"] = summary["verdict"]
1448
- sub = verify.get("sub_verdicts")
1449
- if sub is None:
1450
- # spawn (state-phase-write.py) always writes sub_verdicts: null as
1451
- # part of the per-round reset contract (state-schema.md#write-protocol)
1452
- # — legal, expected state before this function populates it.
1453
- # setdefault() would not replace an existing null, only an absent key.
1454
- sub = {}
1455
- verify["sub_verdicts"] = sub
1456
- elif not isinstance(sub, dict):
1457
- raise SystemExit(
1458
- f"error: phases.verify.sub_verdicts must be null or an object, got {type(sub).__name__}"
1459
- )
1460
- for source, source_verdict in summary["source_verdicts"].items():
1461
- if source in {"mechanical", "judge", "pair_judge"}:
1462
- sub[source] = source_verdict
1463
- verify["merged"] = {
1464
- "verdict": summary["verdict"],
1465
- "findings_file": ".devlyn/verify-merged.findings.jsonl",
1466
- "summary_file": ".devlyn/verify-merge.summary.json",
1467
- }
1468
- state_path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8")
1469
-
1470
-
1471
- def self_test() -> int:
1472
- import subprocess
1473
-
1474
- with tempfile.TemporaryDirectory() as tmp:
1475
- work = pathlib.Path(tmp).resolve()
1476
- devlyn = work / ".devlyn"
1477
- devlyn.mkdir()
1478
- role = JUDGE_ROLE_EVIDENCE["ROLE"]
1479
- (devlyn / "engines.json").write_bytes(role["encoded"]({
1480
- "roles": {"primary_judge": {"engine": "claude", "model": "fixture-claude-model", "effort": "high"}},
1481
- }))
1482
- resolution = role["resolve"](work, "claude", no_pair=True, available=lambda engine: True)
1483
- state = {"run_id": "merge-relative-role", "engine": "claude", "role_resolution": resolution,
1484
- "rounds": {"global": 0, "max_rounds": 1},
1485
- "phases": {"verify": {"engine": "claude", "round": 0, "verdict": None,
1486
- "started_at": "2026-01-01T00:00:00Z", "completed_at": "2026-01-01T00:01:00Z"}}}
1487
- state_path = devlyn / "pipeline.state.json"
1488
- state_path.write_text(json.dumps(state), encoding="utf-8")
1489
- stem = "claude-judge.r0"
1490
- prompt_path = devlyn / (stem + ".prompt")
1491
- prompt = b"review"
1492
- prompt_path.write_bytes(prompt)
1493
- command = ["claude", "-p", "--model", "fixture-claude-model", "--effort", "high",
1494
- "--permission-mode", "dontAsk", "--tools", "Read,Grep,Glob",
1495
- "--allowedTools", "Read,Grep,Glob", "--setting-sources", "project",
1496
- "--output-format", "json", "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}']
1497
- argv = ["python3", "run-bounded.py", "600", "--stdin-file", str(prompt_path),
1498
- "--record-transport", "--", *command]
1499
- argv_path = devlyn / (stem + ".argv.json")
1500
- argv_path.write_bytes(role["encoded"](argv))
1501
- (devlyn / (stem + ".stderr")).write_bytes(b"")
1502
- (devlyn / (stem + ".output.json")).write_bytes(role["encoded"]({
1503
- "type": "result", "subtype": "success", "is_error": False, "stop_reason": "end_turn",
1504
- "session_id": "fixture-claude", "result": "PASS", "modelUsage": {"fixture-claude-model": {}},
1505
- }))
1506
- transport = {"schema_version": 1, "transport": "stdin-file",
1507
- "prompt": {"path": str(prompt_path), "sha256": role["digest"](prompt), "bytes": len(prompt)},
1508
- "command": command, "argv": command, "timeout_sec": 600, "isolated": False,
1509
- "status": "completed", "exit_code": 0}
1510
- (devlyn / (stem + ".prompt.transport.json")).write_text(json.dumps(transport), encoding="utf-8")
1511
- evidence, derived = JUDGE_ROLE_EVIDENCE["describe"](devlyn, state, "primary_judge", 0)
1512
- (devlyn / (stem + ".role-evidence.json")).write_bytes(role["encoded"](evidence))
1513
- (devlyn / (stem + ".stdout")).write_bytes(derived)
1514
- (devlyn / "claude-judge.stdout").write_bytes(derived)
1515
- (devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
1516
- (devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
1517
- script = str(pathlib.Path(__file__).resolve())
1518
- def merge_at(directory):
1519
- return subprocess.run([sys.executable, script, "--devlyn-dir", directory, "--write-state"],
1520
- cwd=work, capture_output=True, text=True)
1521
- relative = merge_at(".devlyn")
1522
- assert relative.returncode == 0, relative.stderr
1523
- relative_state = json.loads(state_path.read_text())
1524
- assert relative_state["phases"]["verify"]["verdict"] == "PASS", relative.stdout
1525
- state_path.write_text(json.dumps(state), encoding="utf-8")
1526
- absolute = merge_at(str(devlyn))
1527
- assert absolute.returncode == 0, absolute.stderr
1528
- absolute_state = json.loads(state_path.read_text())
1529
- assert absolute_state["phases"]["verify"]["role_evidence"] == relative_state["phases"]["verify"]["role_evidence"]
1530
- argv[4] = str(work / "wrong.prompt")
1531
- argv_path.write_bytes(role["encoded"](argv))
1532
- state_path.write_text(json.dumps(state), encoding="utf-8")
1533
- wrong = merge_at(".devlyn")
1534
- assert wrong.returncode == 0, wrong.stderr
1535
- blocked_state = json.loads(state_path.read_text())
1536
- assert blocked_state["phases"]["verify"]["verdict"] == "BLOCKED"
1537
- writer = runpy.run_path(str(pathlib.Path(__file__).with_name("state-phase-write.py")))
1538
- try:
1539
- writer["do_spawn"](blocked_state, "implement", 1, "verify", None, None, None)
1540
- except SystemExit as exc:
1541
- assert str(exc) == "BLOCKED:repair-edge-invalid"
1542
- else:
1543
- raise AssertionError("BLOCKED VERIFY entered product repair")
1544
- assert blocked_state["rounds"]["global"] == 0
1545
- argv[4] = str(prompt_path)
1546
- argv_path.write_bytes(role["encoded"](argv))
1547
- finding = {"id": "fixture-high", "rule_id": "fixture.binding", "severity": "HIGH",
1548
- "confidence": "high", "file": "source.txt", "line": 1,
1549
- "message": "required behavior is missing", "criterion_ref": "fixture"}
1550
- (devlyn / "verify.findings.jsonl").write_text(json.dumps(finding) + "\n", encoding="utf-8")
1551
- state_path.write_text(json.dumps(state), encoding="utf-8")
1552
- needs_work = merge_at(".devlyn")
1553
- assert needs_work.returncode == 0, needs_work.stderr
1554
- repair_state = json.loads(state_path.read_text())
1555
- assert repair_state["phases"]["verify"]["verdict"] == "NEEDS_WORK"
1556
- writer["do_spawn"](repair_state, "implement", 1, "verify", None, None, None)
1557
- assert repair_state["rounds"]["global"] == 1
1558
-
1559
- try:
1560
- loads_strict_json('{"verdict":"PASS","verdict":"BLOCKED"}')
1561
- except ValueError as exc:
1562
- assert "duplicate JSON key" in str(exc)
1563
- else:
1564
- raise AssertionError("duplicate VERIFY authority key was accepted")
1565
- with tempfile.TemporaryDirectory() as tmp:
1566
- devlyn = pathlib.Path(tmp)
1567
-
1568
- # Every completed VERIFY has deterministic mechanical and primary
1569
- # judge carriers. Missing either one is an evidence failure, not PASS.
1570
- (devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
1571
-
1572
- # state-phase-write.py's spawn always writes sub_verdicts: null (the
1573
- # per-round reset contract, state-schema.md#write-protocol) — this is
1574
- # the real shape write_state() sees on every VERIFY completion, not
1575
- # the pre-populated {} the other scenarios below seed for brevity.
1576
- (devlyn / "pipeline.state.json").write_text(
1577
- json.dumps({
1578
- "engine": "claude",
1579
- "phases": {
1580
- "verify": {
1581
- "verdict": None,
1582
- "sub_verdicts": None,
1583
- "judge_durations_ms": {"judge": 23, "pair_judge": 31},
1584
- }
1585
- }
1586
- }),
1587
- encoding="utf-8",
1588
- )
1589
- (devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
1590
-
1591
- sealed_work = pathlib.Path(tmp) / "sealed-work"
1592
- sealed_devlyn = sealed_work / ".devlyn"
1593
- sealed_devlyn.mkdir(parents=True)
1594
- sealed_state = {
1595
- "run_id": "rs-sealed-mechanical",
1596
- "engine": "claude",
1597
- "process_evidence": None,
1598
- "phases": {"verify": {"round": 2, "verdict": None, "sub_verdicts": None}},
1599
- }
1600
- obligation = PROCESS_EVIDENCE["normalize_obligation"]({
1601
- "id": "mechanical-0",
1602
- "phase": "verify",
1603
- "argv": [sys.executable, "-c", "print('sealed')"],
1604
- })
1605
- manifest_rel = PROCESS_EVIDENCE["manifest_relative_path"](sealed_state, "verify")
1606
- PROCESS_EVIDENCE["capture_process"](
1607
- sealed_work, sealed_work / manifest_rel, sealed_state["run_id"],
1608
- "verify", 2, obligation,
1609
- )
1610
- carrier = PROCESS_EVIDENCE["validate_manifest"](
1611
- sealed_work, manifest_rel, sealed_state["run_id"], "verify", 2,
1612
- [obligation], require_expectations=False,
1613
- )
1614
- (sealed_devlyn / "pipeline.state.json").write_text(
1615
- json.dumps(sealed_state), encoding="utf-8",
1616
- )
1617
- sealed_commands = PROCESS_EVIDENCE["bound_carrier_summary_commands"](
1618
- sealed_work, carrier,
1619
- )
1620
- (sealed_devlyn / "spec-verify.results.json").write_text(
1621
- json.dumps({"commands": sealed_commands, "process_evidence": carrier}),
1622
- encoding="utf-8",
1623
- )
1624
- (sealed_devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
1625
- (sealed_devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
1626
- sealed_findings, sealed_verdicts = read_findings(sealed_devlyn)
1627
- assert sealed_verdicts["mechanical"] == "PASS", sealed_findings
1628
- sealed_summary = write_outputs(sealed_devlyn, sealed_findings, sealed_verdicts)
1629
- write_state(sealed_devlyn, sealed_summary)
1630
- bound_state = loads_strict_json(
1631
- (sealed_devlyn / "pipeline.state.json").read_text(encoding="utf-8")
1632
- )
1633
- assert bound_state["process_evidence"] == [carrier], bound_state
1634
-
1635
- failed_work = pathlib.Path(tmp) / "failed-work"
1636
- failed_devlyn = failed_work / ".devlyn"
1637
- failed_devlyn.mkdir(parents=True)
1638
- failed_state = {
1639
- "version": "3.0",
1640
- "run_id": "rs-failed-mechanical",
1641
- "engine": "claude",
1642
- "source": {"type": "spec", "spec_path": "docs/failed/spec.md"},
1643
- "process_evidence": None,
1644
- "phases": {"verify": {"round": 0, "verdict": None, "sub_verdicts": None}},
1645
- }
1646
- failed_spec = failed_work / "docs" / "failed"
1647
- failed_spec.mkdir(parents=True)
1648
- (failed_spec / "spec.md").write_text("# failed fixture\n", encoding="utf-8")
1649
- (failed_spec / "spec.expected.json").write_text(json.dumps({
1650
- "verification_commands": [{"cmd": "exit 7", "exit_code": 0}],
1651
- }) + "\n", encoding="utf-8")
1652
- failed_obligation = PROCESS_EVIDENCE["normalize_obligation"]({
1653
- "id": "verification-command-0001",
1654
- "phase": "verify",
1655
- "cmd": "exit 7",
1656
- })
1657
- failed_manifest = PROCESS_EVIDENCE["manifest_relative_path"](failed_state, "verify")
1658
- PROCESS_EVIDENCE["capture_process"](
1659
- failed_work, failed_work / failed_manifest, failed_state["run_id"],
1660
- "verify", 0, failed_obligation,
1661
- )
1662
- failed_carrier = PROCESS_EVIDENCE["validate_manifest"](
1663
- failed_work, failed_manifest, failed_state["run_id"], "verify", 0,
1664
- [failed_obligation], require_expectations=False,
1665
- )
1666
- (failed_devlyn / "pipeline.state.json").write_text(
1667
- json.dumps(failed_state), encoding="utf-8",
1668
- )
1669
- # Exact laundering attempt: mutable derivatives say PASS/empty while
1670
- # the state-identical sealed manifest still records the failed command.
1671
- laundered_commands = PROCESS_EVIDENCE["bound_carrier_summary_commands"](
1672
- failed_work, failed_carrier,
1673
- )
1674
- laundered_commands[0]["pass"] = True
1675
- (failed_devlyn / "spec-verify.results.json").write_text(json.dumps({
1676
- "commands": laundered_commands, "process_evidence": failed_carrier,
1677
- }), encoding="utf-8")
1678
- (failed_devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
1679
- (failed_devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
1680
- failed_findings, failed_verdicts = read_findings(failed_devlyn)
1681
- assert failed_verdicts["mechanical"] == "BLOCKED", failed_verdicts
1682
- assert any(
1683
- item.get("id") == "verify-mechanical-evidence-invalid"
1684
- for item in failed_findings
1685
- ), failed_findings
1686
- assert write_outputs(failed_devlyn, failed_findings, failed_verdicts)["verdict"] == "BLOCKED"
1687
-
1688
- (failed_devlyn / "spec-verify.results.json").write_text(json.dumps({
1689
- "commands": [], "process_evidence": None,
1690
- }), encoding="utf-8")
1691
- removed_findings, removed_verdicts = read_findings(failed_devlyn)
1692
- assert removed_verdicts["mechanical"] == "BLOCKED", removed_verdicts
1693
- assert any(
1694
- item.get("id") == "verify-mechanical-evidence-invalid"
1695
- for item in removed_findings
1696
- ), removed_findings
1697
- print("PASS iter-0112 sealed VERIFY outcome resists mutable-derivative laundering")
1698
-
1699
- sealed_stdout = sealed_work / carrier["streams"][0]["stdout"]["path"]
1700
- sealed_stdout.write_bytes(sealed_stdout.read_bytes() + b"altered")
1701
- altered_findings, altered_verdicts = read_findings(sealed_devlyn)
1702
- assert altered_verdicts["mechanical"] == "BLOCKED", altered_verdicts
1703
- assert any(
1704
- finding.get("id") == "verify-mechanical-evidence-invalid"
1705
- for finding in altered_findings
1706
- ), altered_findings
1707
-
1708
- skipped_work = pathlib.Path(tmp) / "skipped-work"
1709
- skipped_work.mkdir()
1710
- (skipped_work / "pipeline.state.json").write_text(json.dumps({
1711
- "engine": "claude",
1712
- "risk_profile": {"pair_default_enabled": False},
1713
- "phases": {"verify": {"pair_trigger": {
1714
- "eligible": False,
1715
- "reasons": [],
1716
- "skipped_reason": "user_no_pair",
1717
- }}},
1718
- }), encoding="utf-8")
1719
- (skipped_work / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
1720
- (skipped_work / "verify.findings.jsonl").write_text("", encoding="utf-8")
1721
- generic_primary = skipped_work / "verify-judge.stdout"
1722
- for generic_output in ("PASS\n", "NEEDS_WORK\n"):
1723
- generic_primary.write_text(generic_output, encoding="utf-8")
1724
- generic_findings, generic_verdicts = read_findings(skipped_work)
1725
- assert generic_verdicts["pair_judge"] is None, generic_verdicts
1726
- assert not any(
1727
- finding.get("file") == generic_primary.name for finding in generic_findings
1728
- ), generic_findings
1729
- (skipped_work / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
1730
- contradiction_findings, contradiction_verdicts = read_findings(skipped_work)
1731
- assert contradiction_verdicts["pair_judge"] == "BLOCKED", contradiction_verdicts
1732
- assert any(
1733
- finding.get("id") == "verify-pair-skipped-output-present"
1734
- for finding in contradiction_findings
1735
- ), contradiction_findings
1736
- findings, source_verdicts = read_findings(devlyn)
1737
- summary = write_outputs(devlyn, findings, source_verdicts)
1738
- write_state(devlyn, summary)
1739
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
1740
- assert summary["verdict"] == "PASS", summary
1741
- assert state["phases"]["verify"]["sub_verdicts"] == {
1742
- "mechanical": "PASS", "judge": "PASS", "pair_judge": None,
1743
- }, state
1744
- assert state["phases"]["verify"]["judge_durations_ms"] == {
1745
- "judge": 23, "pair_judge": 31,
1746
- }, state
1747
- original_state = (devlyn / "pipeline.state.json").read_bytes()
1748
- for malformed in (None, {}, {"roles": {}}):
1749
- bad_state = loads_strict_json(original_state)
1750
- bad_state["role_resolution"] = malformed
1751
- (devlyn / "pipeline.state.json").write_text(json.dumps(bad_state), encoding="utf-8")
1752
- result = subprocess.run([sys.executable, str(pathlib.Path(__file__).resolve()),
1753
- "--devlyn-dir", str(devlyn), "--write-state"], capture_output=True, text=True, encoding="utf-8")
1754
- assert result.returncode == 0 and "Traceback" not in result.stderr, result.stderr
1755
- persisted = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
1756
- assert persisted["phases"]["verify"]["verdict"] == "BLOCKED"
1757
- assert "verify-role-resolution-invalid" in (devlyn / "verify-merged.findings.jsonl").read_text(encoding="utf-8")
1758
- (devlyn / "pipeline.state.json").write_bytes(original_state)
1759
- different_primary = {"engine": "codex", "phases": {"verify": {"engine": "claude"}}}
1760
- (devlyn / "pipeline.state.json").write_text(json.dumps(different_primary), encoding="utf-8")
1761
- (devlyn / "verify.primary.timeout.json").write_text(json.dumps({"engine": "claude", "budget_seconds": 600}), encoding="utf-8")
1762
- marker, violation = read_primary_timeout_marker(devlyn)
1763
- assert marker is not None and violation is None
1764
- different_primary["phases"]["verify"]["engine"] = None
1765
- (devlyn / "pipeline.state.json").write_text(json.dumps(different_primary), encoding="utf-8")
1766
- assert read_primary_timeout_marker(devlyn)[1] is not None
1767
- (devlyn / "verify.primary.timeout.json").unlink()
1768
- (devlyn / "pipeline.state.json").write_bytes(original_state)
1769
-
1770
- (devlyn / "verify.findings.jsonl").unlink()
1771
- findings, source_verdicts = read_findings(devlyn)
1772
- summary = write_outputs(devlyn, findings, source_verdicts)
1773
- assert summary["verdict"] == "BLOCKED", summary
1774
- assert summary["source_verdicts"]["judge"] == "BLOCKED", summary
1775
- assert any(
1776
- finding["id"] == "verify-merge-required-source-missing-judge"
1777
- for finding in findings
1778
- ), findings
1779
-
1780
- # iter-0113: a valid primary timeout is an explicit BLOCKED source,
1781
- # never the generic missing-source path or pair-style TIMEOUT.
1782
- (devlyn / "verify.primary.timeout.json").write_text(
1783
- json.dumps({"engine": "claude", "budget_seconds": 600}) + "\n",
1784
- encoding="utf-8",
1785
- )
1786
- findings, source_verdicts = read_findings(devlyn)
1787
- summary = write_outputs(devlyn, findings, source_verdicts)
1788
- assert summary["verdict"] == "BLOCKED", summary
1789
- assert summary["source_verdicts"]["judge"] == "BLOCKED", summary
1790
- assert any(
1791
- finding["id"] == "verify-primary-timeout"
1792
- for finding in findings
1793
- ), findings
1794
- assert not any(
1795
- finding["id"] == "verify-merge-required-source-missing-judge"
1796
- for finding in findings
1797
- ), findings
1798
-
1799
- (devlyn / "verify.findings.jsonl").write_text(
1800
- json.dumps({"id": "primary-timeout-high", "severity": "HIGH"}) + "\n",
1801
- encoding="utf-8",
1802
- )
1803
- findings, source_verdicts = read_findings(devlyn)
1804
- assert source_verdicts["judge"] == "BLOCKED", source_verdicts
1805
- assert {finding["id"] for finding in findings} >= {
1806
- "primary-timeout-high", "verify-primary-timeout",
1807
- }, findings
1808
-
1809
- for marker, expected_id in (
1810
- ("{\n", "verify-primary-timeout-marker-malformed"),
1811
- (
1812
- json.dumps({"engine": "codex", "budget_seconds": 600}) + "\n",
1813
- "verify-primary-timeout-engine-mismatch",
1814
- ),
1815
- (
1816
- json.dumps({"engine": "claude", "budget_seconds": 601}) + "\n",
1817
- "verify-primary-timeout-budget-mismatch",
1818
- ),
1819
- ):
1820
- (devlyn / "verify.primary.timeout.json").write_text(marker, encoding="utf-8")
1821
- findings, source_verdicts = read_findings(devlyn)
1822
- assert source_verdicts["judge"] == "BLOCKED", source_verdicts
1823
- assert any(finding["id"] == expected_id for finding in findings), findings
1824
- (devlyn / "verify.primary.timeout.json").unlink()
1825
- (devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
1826
-
1827
- # iter-0072 Amendment 3: generated schema-v3 runs mechanically prove
1828
- # raw-goal persistence and required SURFACE_CLOSE dispatch.
1829
- (devlyn / "pipeline.state.json").write_text(
1830
- json.dumps({
1831
- "version": "3.0",
1832
- "engine": "claude",
1833
- "complexity": "large",
1834
- "source": {"type": "generated"},
1835
- "phases": {"verify": {"verdict": None, "sub_verdicts": None}},
1836
- }),
1837
- encoding="utf-8",
1838
- )
1839
- findings, source_verdicts = read_findings(devlyn)
1840
- summary = write_outputs(devlyn, findings, source_verdicts)
1841
- assert summary["verdict"] == "BLOCKED", summary
1842
- assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
1843
- assert any(
1844
- finding.get("rule_id") == "verify.state.goal-persistence-missing"
1845
- for finding in findings
1846
- ), findings
1847
-
1848
- (devlyn / "pipeline.state.json").write_text(
1849
- json.dumps({
1850
- "version": "3.0",
1851
- "engine": "claude",
1852
- "complexity": "large",
1853
- "source": {"type": "generated", "goal_path": "", "goal_sha256": "A" * 64},
1854
- }),
1855
- encoding="utf-8",
1856
- )
1857
- violation = verify_state_contract_violation(devlyn)
1858
- assert violation is not None, violation
1859
- assert violation["rule_id"] == "verify.state.goal-persistence-missing", violation
1860
-
1861
- generated_source = {
1862
- "type": "generated",
1863
- "goal_path": ".devlyn/goal.raw.txt",
1864
- "goal_sha256": "a" * 64,
1865
- }
1866
- (devlyn / "pipeline.state.json").write_text(
1867
- json.dumps({
1868
- "version": "3.0",
1869
- "engine": "claude",
1870
- "complexity": "medium",
1871
- "source": generated_source,
1872
- "phases": {
1873
- "surface_close": None,
1874
- "verify": {"verdict": None, "sub_verdicts": None},
1875
- },
1876
- }),
1877
- encoding="utf-8",
1878
- )
1879
- findings, source_verdicts = read_findings(devlyn)
1880
- summary = write_outputs(devlyn, findings, source_verdicts)
1881
- assert summary["verdict"] == "BLOCKED", summary
1882
- assert any(
1883
- finding.get("rule_id") == "verify.state.surface-close-skipped"
1884
- for finding in findings
1885
- ), findings
1886
-
1887
- for surface_entry in (
1888
- {"verdict": "PASS"},
1889
- {
1890
- "verdict": None,
1891
- "started_at": None,
1892
- "skipped_reason": "auto_surface_close_claude_unavailable",
1893
- },
1894
- ):
1895
- (devlyn / "pipeline.state.json").write_text(
1896
- json.dumps({
1897
- "version": "3.0",
1898
- "engine": "claude",
1899
- "complexity": "medium",
1900
- "source": generated_source,
1901
- "phases": {"surface_close": surface_entry},
1902
- }),
1903
- encoding="utf-8",
1904
- )
1905
- assert verify_state_contract_violation(devlyn) is None
1906
-
1907
- (devlyn / "pipeline.state.json").write_text(
1908
- json.dumps({
1909
- "version": "3.0",
1910
- "engine": "claude",
1911
- "complexity": "medium",
1912
- "source": generated_source,
1913
- "phases": {
1914
- "surface_close": {
1915
- "verdict": None,
1916
- "started_at": None,
1917
- "skipped_reason": "claude-unavailable",
1918
- },
1919
- },
1920
- }),
1921
- encoding="utf-8",
1922
- )
1923
- assert verify_state_contract_violation(devlyn)["rule_id"] == "verify.state.surface-close-skipped"
1924
-
1925
- for state_shape in (
1926
- {
1927
- "version": "3.0", "engine": "claude", "complexity": "medium",
1928
- "source": {"type": "spec"}, "phases": {"surface_close": None},
1929
- },
1930
- {
1931
- "version": "3.0", "engine": "claude", "complexity": "large",
1932
- "source": generated_source, "phases": {"surface_close": None},
1933
- },
1934
- {
1935
- "version": "2.0", "engine": "claude", "complexity": "medium",
1936
- "source": {"type": "generated"}, "phases": {"surface_close": None},
1937
- },
1938
- ):
1939
- (devlyn / "pipeline.state.json").write_text(
1940
- json.dumps(state_shape), encoding="utf-8",
1941
- )
1942
- assert verify_state_contract_violation(devlyn) is None
1943
-
1944
- # 2026-07-04 field bug (iter-0060 G1): an AUTO pair trigger skipped on
1945
- # OTHER-engine unavailability spawns no second judge — pair_judge must
1946
- # be recorded null, never a synthesized PASS.
1947
- (devlyn / "pipeline.state.json").write_text(
1948
- json.dumps({
1949
- "version": "3.0",
1950
- "engine": "claude",
1951
- "mode": "spec",
1952
- "risk_profile": {"pair_default_enabled": True},
1953
- "phases": {
1954
- "verify": {
1955
- "verdict": None,
1956
- "sub_verdicts": None,
1957
- "pair_trigger": {
1958
- "eligible": False,
1959
- "reasons": [],
1960
- "skipped_reason": "auto_pair_other_engine_unavailable",
1961
- },
1962
- }
1963
- },
1964
- }),
1965
- encoding="utf-8",
1966
- )
1967
- findings, source_verdicts = read_findings(devlyn)
1968
- summary = write_outputs(devlyn, findings, source_verdicts)
1969
- write_state(devlyn, summary)
1970
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
1971
- assert summary["verdict"] == "PASS", summary
1972
- assert summary["source_verdicts"]["pair_judge"] is None, summary
1973
- assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] is None, state
1974
-
1975
- # A non-null, non-dict sub_verdicts is corrupted state, not a legal
1976
- # placeholder — write_state() must fail loud, not silently coerce it.
1977
- (devlyn / "pipeline.state.json").write_text(
1978
- json.dumps({"engine": "claude", "phases": {"verify": {"verdict": None, "sub_verdicts": "corrupt"}}}),
1979
- encoding="utf-8",
1980
- )
1981
- findings, source_verdicts = read_findings(devlyn)
1982
- summary = write_outputs(devlyn, findings, source_verdicts)
1983
- try:
1984
- write_state(devlyn, summary)
1985
- except SystemExit as e:
1986
- assert "sub_verdicts must be null or an object" in str(e), e
1987
- else:
1988
- raise AssertionError("write_state() must reject non-dict, non-null sub_verdicts")
1989
-
1990
- (devlyn / "pipeline.state.json").write_text(
1991
- json.dumps({
1992
- "engine": "claude",
1993
- "phases": {
1994
- "verify": {
1995
- "verdict": "PASS",
1996
- "sub_verdicts": {},
1997
- "pair_trigger": {
1998
- "eligible": True,
1999
- "reasons": ["risk.high", "judge.warning"],
2000
- "skipped_reason": None,
2001
- },
2002
- }
2003
- }
2004
- }),
2005
- encoding="utf-8",
2006
- )
2007
- (devlyn / "verify.findings.jsonl").write_text(
2008
- json.dumps({"id": "j1", "severity": "LOW"}) + "\n",
2009
- encoding="utf-8",
2010
- )
2011
- (devlyn / "verify.pair.findings.jsonl").write_text(
2012
- json.dumps({"id": "p1", "severity": "HIGH"}) + "\n",
2013
- encoding="utf-8",
2014
- )
2015
- (devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
2016
- findings, source_verdicts = read_findings(devlyn)
2017
- summary = write_outputs(devlyn, findings, source_verdicts)
2018
- write_state(devlyn, summary)
2019
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
2020
- assert summary["verdict"] == "NEEDS_WORK", summary
2021
- assert state["phases"]["verify"]["verdict"] == "NEEDS_WORK", state
2022
- assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "NEEDS_WORK", state
2023
- assert (devlyn / "verify-merged.findings.jsonl").read_text(encoding="utf-8")
2024
- (devlyn / "codex-judge.stdout").unlink()
2025
- (devlyn / "verify.findings.jsonl").write_text(
2026
- '{"id":"nan","severity":NaN}\n',
2027
- encoding="utf-8",
2028
- )
2029
- (devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
2030
- findings, source_verdicts = read_findings(devlyn)
2031
- assert source_verdicts["judge"] == "BLOCKED", source_verdicts
2032
- assert any(
2033
- finding.get("id") == "verify-merge-invalid-json-verify.findings.jsonl-1"
2034
- and "invalid JSON numeric constant: NaN" in finding.get("message", "")
2035
- for finding in findings
2036
- ), findings
2037
- (devlyn / "pipeline.state.json").write_text(
2038
- json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
2039
- encoding="utf-8",
2040
- )
2041
- (devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
2042
- (devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
2043
- findings, source_verdicts = read_findings(devlyn)
2044
- summary = write_outputs(devlyn, findings, source_verdicts)
2045
- write_state(devlyn, summary)
2046
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
2047
- assert summary["verdict"] == "PASS", summary
2048
- assert state["phases"]["verify"]["verdict"] == "PASS", state
2049
- assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "PASS", state
2050
- (devlyn / "codex-judge.stdout").write_text(
2051
- json.dumps({"id": "cj1", "severity": "HIGH"}) + "\n"
2052
- + '# SUMMARY {"verdict":"NEEDS_WORK"}\n',
2053
- encoding="utf-8",
2054
- )
2055
- findings, source_verdicts = read_findings(devlyn)
2056
- summary = write_outputs(devlyn, findings, source_verdicts)
2057
- write_state(devlyn, summary)
2058
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
2059
- assert summary["verdict"] == "BLOCKED", summary
2060
- assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "BLOCKED", state
2061
- (devlyn / "codex-judge.stdout").unlink()
2062
-
2063
- # Engine-neutral stdout contract (iter-0060): a Claude pair-judge
2064
- # capture (claude-judge.stdout, adapters/claude.md ## Invocation)
2065
- # binds the same emission contract as the Codex one.
2066
- (devlyn / "pipeline.state.json").write_text(
2067
- json.dumps({"engine": "codex", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
2068
- encoding="utf-8",
2069
- )
2070
- (devlyn / "claude-judge.stdout").write_text(
2071
- json.dumps({"id": "clj1", "severity": "HIGH"}) + "\n",
2072
- encoding="utf-8",
2073
- )
2074
- findings, source_verdicts = read_findings(devlyn)
2075
- summary = write_outputs(devlyn, findings, source_verdicts)
2076
- assert summary["verdict"] == "BLOCKED", summary
2077
- assert any(
2078
- finding.get("id") == "verify-pair-emission-contract-violated"
2079
- and "claude-judge.stdout" in finding.get("message", "")
2080
- for finding in findings
2081
- ), findings
2082
- (devlyn / "claude-judge.stdout").unlink()
2083
-
2084
- # iter-0065 case 1: a valid pair timeout marker plus empty pair output
2085
- # records TIMEOUT and leaves the merged verdict to mechanical+primary.
2086
- (devlyn / "pipeline.state.json").write_text(
2087
- json.dumps({
2088
- "engine": "codex",
2089
- "phases": {
2090
- "verify": {
2091
- "verdict": "PASS",
2092
- "sub_verdicts": {},
2093
- "pair_trigger": {
2094
- "eligible": True,
2095
- "reasons": ["judge.warning"],
2096
- "skipped_reason": None,
2097
- },
2098
- }
2099
- }
2100
- }),
2101
- encoding="utf-8",
2102
- )
2103
- (devlyn / "verify.findings.jsonl").write_text(
2104
- json.dumps({"id": "j-timeout-low", "severity": "LOW"}) + "\n",
2105
- encoding="utf-8",
2106
- )
2107
- (devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
2108
- (devlyn / "claude-judge.stdout").write_text("", encoding="utf-8")
2109
- (devlyn / "verify.pair.timeout.json").write_text(
2110
- json.dumps({"engine": "claude", "budget_seconds": 600}),
2111
- encoding="utf-8",
2112
- )
2113
- findings, source_verdicts = read_findings(devlyn)
2114
- summary = write_outputs(devlyn, findings, source_verdicts)
2115
- write_state(devlyn, summary)
2116
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
2117
- assert summary["verdict"] == "PASS_WITH_ISSUES", summary
2118
- assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
2119
- assert summary["pair_timeout"] == {"engine": "claude", "budget_seconds": 600}, summary
2120
- assert summary["report_header_note"] == "solo verdict after pair TIMEOUT", summary
2121
- assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "TIMEOUT", state
2122
- (devlyn / "claude-judge.stdout").unlink()
2123
- (devlyn / "verify.pair.timeout.json").unlink()
2124
- (devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
2125
-
2126
- # iter-0065 case 2: canonical pair findings still bind after timeout.
2127
- (devlyn / "pipeline.state.json").write_text(
2128
- json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
2129
- encoding="utf-8",
2130
- )
2131
- (devlyn / "verify.pair.findings.jsonl").write_text(
2132
- json.dumps({"id": "p-timeout-high", "severity": "HIGH"}) + "\n",
2133
- encoding="utf-8",
2134
- )
2135
- (devlyn / "verify.pair.timeout.json").write_text(
2136
- json.dumps({"engine": "codex", "budget_seconds": 600}),
2137
- encoding="utf-8",
2138
- )
2139
- findings, source_verdicts = read_findings(devlyn)
2140
- summary = write_outputs(devlyn, findings, source_verdicts)
2141
- assert summary["verdict"] == "NEEDS_WORK", summary
2142
- assert summary["source_verdicts"]["pair_judge"] == "NEEDS_WORK", summary
2143
- assert "pair_timeout" not in summary, summary
2144
- (devlyn / "verify.pair.timeout.json").unlink()
2145
- (devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
2146
-
2147
- # iter-0065 case 2b: stdout-only HIGH still blocks on emission contract;
2148
- # timeout never converts an observed finding into a solo pass.
2149
- (devlyn / "pipeline.state.json").write_text(
2150
- json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
2151
- encoding="utf-8",
2152
- )
2153
- (devlyn / "codex-judge.stdout").write_text(
2154
- json.dumps({"id": "cj-timeout-high", "severity": "HIGH"}) + "\n",
2155
- encoding="utf-8",
2156
- )
2157
- (devlyn / "verify.pair.timeout.json").write_text(
2158
- json.dumps({"engine": "codex", "budget_seconds": 600}),
2159
- encoding="utf-8",
2160
- )
2161
- findings, source_verdicts = read_findings(devlyn)
2162
- summary = write_outputs(devlyn, findings, source_verdicts)
2163
- assert summary["verdict"] == "BLOCKED", summary
2164
- assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
2165
- assert any(
2166
- finding.get("id") == "verify-pair-emission-contract-violated"
2167
- for finding in findings
2168
- ), findings
2169
- (devlyn / "codex-judge.stdout").unlink()
2170
- (devlyn / "verify.pair.timeout.json").unlink()
2171
-
2172
- # iter-0106: a budget abort can truncate the whole-message stream before
2173
- # its terminal result. With a valid marker that capture stays TIMEOUT.
2174
- (devlyn / "pipeline.state.json").write_text(
2175
- json.dumps({
2176
- "engine": "claude",
2177
- "phases": {
2178
- "verify": {
2179
- "verdict": "PASS",
2180
- "sub_verdicts": {},
2181
- "pair_trigger": {
2182
- "eligible": True,
2183
- "reasons": ["judge.warning"],
2184
- "skipped_reason": None,
2185
- },
2186
- }
2187
- }
2188
- }),
2189
- encoding="utf-8",
2190
- )
2191
- partial_stream = (
2192
- '{"type":"system","subtype":"init","session_id":"s106"}\n'
2193
- '{"type":"assistant","session_id":"s106","message":{"stop_reason":null,'
2194
- '"content":[{"type":"text","text":"reading the diff"}]}}\n'
2195
- )
2196
- (devlyn / "grok-judge.stdout").write_text(partial_stream, encoding="utf-8")
2197
- (devlyn / "verify.pair.timeout.json").write_text(
2198
- json.dumps({"engine": "grok", "budget_seconds": 600}),
2199
- encoding="utf-8",
2200
- )
2201
- findings, source_verdicts = read_findings(devlyn)
2202
- summary = write_outputs(devlyn, findings, source_verdicts)
2203
- assert summary["verdict"] == "PASS", summary
2204
- assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
2205
- assert summary["pair_timeout"] == {"engine": "grok", "budget_seconds": 600}, summary
2206
- assert not findings, findings
2207
-
2208
- # A kill may cut the final JSON record mid-write; one malformed tail is
2209
- # still a valid prefix, while malformed/unknown complete records are not.
2210
- (devlyn / "grok-judge.stdout").write_text(
2211
- partial_stream + '{"type":"assistant","session_id":"s106"',
2212
- encoding="utf-8",
2213
- )
2214
- findings, source_verdicts = read_findings(devlyn)
2215
- summary = write_outputs(devlyn, findings, source_verdicts)
2216
- assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
2217
- assert not findings, findings
2218
-
2219
- for invalid_prefix in (
2220
- '{"type":"system","subtype":"init"}\n',
2221
- partial_stream + "trailing narration\n",
2222
- partial_stream
2223
- + '{"type":"user","session_id":"other"}\n',
2224
- partial_stream
2225
- + '{"type":"stream_event","session_id":"s106"}\n',
2226
- partial_stream
2227
- + '{"type":"system","subtype":"compact_boundary","session_id":"s106"}\n',
2228
- partial_stream
2229
- + '{"type":"assistant"\n'
2230
- + '{"type":"user","session_id":"s106"}\n',
2231
- ):
2232
- (devlyn / "grok-judge.stdout").write_text(invalid_prefix, encoding="utf-8")
2233
- findings, source_verdicts = read_findings(devlyn)
2234
- summary = write_outputs(devlyn, findings, source_verdicts)
2235
- assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
2236
- assert any(
2237
- finding.get("id") == "verify-pair-emission-contract-violated"
2238
- for finding in findings
2239
- ), findings
2240
-
2241
- # iter-0106 control: the marker conserves TIMEOUT only for that shape. A
2242
- # stream that reached its result but welded narration into the terminal
2243
- # message stays BLOCKED on the emission contract.
2244
- (devlyn / "grok-judge.stdout").write_text(
2245
- '{"type":"system","subtype":"init","session_id":"s106"}\n'
2246
- + json.dumps({
2247
- "type": "assistant",
2248
- "session_id": "s106",
2249
- "message": {
2250
- "stop_reason": "end_turn",
2251
- "content": [{"type": "text", "text": "I reviewed the diff.\nPASS"}],
2252
- },
2253
- }) + "\n"
2254
- + json.dumps({
2255
- "type": "result",
2256
- "subtype": "success",
2257
- "is_error": False,
2258
- "stop_reason": "end_turn",
2259
- "session_id": "s106",
2260
- "result": "I reviewed the diff.\nPASS",
2261
- }) + "\n",
2262
- encoding="utf-8",
2263
- )
2264
- findings, source_verdicts = read_findings(devlyn)
2265
- summary = write_outputs(devlyn, findings, source_verdicts)
2266
- assert summary["verdict"] == "BLOCKED", summary
2267
- assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
2268
- assert any(
2269
- finding.get("id") == "verify-pair-emission-contract-violated"
2270
- for finding in findings
2271
- ), findings
2272
- (devlyn / "verify.pair.timeout.json").unlink()
2273
-
2274
- # iter-0106 control: without a marker the same truncated stream is a
2275
- # plain emission-contract violation.
2276
- (devlyn / "grok-judge.stdout").write_text(partial_stream, encoding="utf-8")
2277
- findings, source_verdicts = read_findings(devlyn)
2278
- summary = write_outputs(devlyn, findings, source_verdicts)
2279
- assert summary["verdict"] == "BLOCKED", summary
2280
- assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
2281
- assert any(
2282
- finding.get("id") == "verify-pair-emission-contract-violated"
2283
- for finding in findings
2284
- ), findings
2285
- (devlyn / "grok-judge.stdout").unlink()
2286
-
2287
- # iter-0065 case 3: without a marker, required empty pair output remains BLOCKED.
2288
- (devlyn / "pipeline.state.json").write_text(
2289
- json.dumps({
2290
- "engine": "codex",
2291
- "phases": {
2292
- "verify": {
2293
- "verdict": "PASS",
2294
- "sub_verdicts": {},
2295
- "pair_trigger": {
2296
- "eligible": True,
2297
- "reasons": ["risk.high"],
2298
- "skipped_reason": None,
2299
- },
2300
- }
2301
- }
2302
- }),
2303
- encoding="utf-8",
2304
- )
2305
- (devlyn / "claude-judge.stdout").write_text("", encoding="utf-8")
2306
- findings, source_verdicts = read_findings(devlyn)
2307
- summary = write_outputs(devlyn, findings, source_verdicts)
2308
- assert summary["verdict"] == "BLOCKED", summary
2309
- assert any(
2310
- finding.get("id") == "verify-pair-empty-output"
2311
- for finding in findings
2312
- ), findings
2313
- (devlyn / "claude-judge.stdout").unlink()
2314
-
2315
- # iter-0065 malformed timeout markers fail closed as a CRITICAL pair blocker.
2316
- (devlyn / "pipeline.state.json").write_text(
2317
- json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
2318
- encoding="utf-8",
2319
- )
2320
- (devlyn / "verify.pair.timeout.json").write_text(
2321
- json.dumps({"engine": "claude", "budget_seconds": 0}),
2322
- encoding="utf-8",
2323
- )
2324
- findings, source_verdicts = read_findings(devlyn)
2325
- summary = write_outputs(devlyn, findings, source_verdicts)
2326
- assert summary["verdict"] == "BLOCKED", summary
2327
- assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
2328
- assert any(
2329
- finding.get("id") == "verify-pair-timeout-marker-malformed"
2330
- and finding.get("severity") == "CRITICAL"
2331
- for finding in findings
2332
- ), findings
2333
- (devlyn / "verify.pair.timeout.json").unlink()
2334
- (devlyn / "pipeline.state.json").write_text(
2335
- json.dumps({
2336
- "engine": "claude",
2337
- "phases": {
2338
- "verify": {
2339
- "verdict": "PASS",
2340
- "sub_verdicts": {},
2341
- "pair_trigger": {
2342
- "eligible": True,
2343
- "reasons": ["risk.high"],
2344
- "skipped_reason": None,
2345
- },
2346
- }
2347
- }
2348
- }),
2349
- encoding="utf-8",
2350
- )
2351
- findings, source_verdicts = read_findings(devlyn)
2352
- summary = write_outputs(devlyn, findings, source_verdicts)
2353
- write_state(devlyn, summary)
2354
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
2355
- assert summary["verdict"] == "BLOCKED", summary
2356
- assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "BLOCKED", state
2357
- assert any(
2358
- finding.get("id") == "verify-pair-required-output-missing"
2359
- for finding in findings
2360
- ), findings
2361
-
2362
- (devlyn / "pipeline.state.json").write_text(
2363
- json.dumps({
2364
- "engine": "claude",
2365
- "mode": "spec",
2366
- "risk_profile": {
2367
- "high_risk": True,
2368
- "risk_probes_enabled": True,
2369
- "pair_default_enabled": True,
2370
- },
2371
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2372
- }),
2373
- encoding="utf-8",
2374
- )
2375
- findings, source_verdicts = read_findings(devlyn)
2376
- summary = write_outputs(devlyn, findings, source_verdicts)
2377
- assert summary["verdict"] == "BLOCKED", summary
2378
- assert any(
2379
- finding.get("id") == "verify-pair-trigger-required-missing"
2380
- for finding in findings
2381
- ), findings
2382
-
2383
- (devlyn / "pipeline.state.json").write_text(
2384
- json.dumps({
2385
- "engine": "claude",
2386
- "mode": "spec",
2387
- "risk_profile": "enabled",
2388
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2389
- }),
2390
- encoding="utf-8",
2391
- )
2392
- findings, source_verdicts = read_findings(devlyn)
2393
- summary = write_outputs(devlyn, findings, source_verdicts)
2394
- assert summary["verdict"] == "BLOCKED", summary
2395
- assert any(
2396
- finding.get("id") == "verify-risk-profile-malformed"
2397
- and "risk_profile must be an object" in str(finding.get("message"))
2398
- for finding in findings
2399
- ), findings
2400
-
2401
- (devlyn / "pipeline.state.json").write_text(
2402
- json.dumps({
2403
- "engine": "claude",
2404
- "mode": "spec",
2405
- "risk_profile": {
2406
- "high_risk": True,
2407
- "risk_probes_enabled": "true",
2408
- "pair_default_enabled": True,
2409
- },
2410
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2411
- }),
2412
- encoding="utf-8",
2413
- )
2414
- findings, source_verdicts = read_findings(devlyn)
2415
- summary = write_outputs(devlyn, findings, source_verdicts)
2416
- assert summary["verdict"] == "BLOCKED", summary
2417
- assert any(
2418
- finding.get("id") == "verify-risk-profile-malformed"
2419
- and "risk_profile.risk_probes_enabled must be a boolean" in str(finding.get("message"))
2420
- for finding in findings
2421
- ), findings
2422
-
2423
- (devlyn / "pipeline.state.json").write_text(
2424
- json.dumps({
2425
- "engine": "claude",
2426
- "mode": "spec",
2427
- "risk_profile": {
2428
- "high_risk": True,
2429
- "risk_probes_enabled": False,
2430
- "pair_default_enabled": True,
2431
- "reasons": ["explicit", 3],
2432
- },
2433
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2434
- }),
2435
- encoding="utf-8",
2436
- )
2437
- findings, source_verdicts = read_findings(devlyn)
2438
- summary = write_outputs(devlyn, findings, source_verdicts)
2439
- assert summary["verdict"] == "BLOCKED", summary
2440
- assert any(
2441
- finding.get("id") == "verify-risk-profile-malformed"
2442
- and "risk_profile.reasons must be a list of strings" in str(finding.get("message"))
2443
- for finding in findings
2444
- ), findings
2445
-
2446
- (devlyn / "pipeline.state.json").write_text(
2447
- json.dumps({
2448
- "engine": "claude",
2449
- "mode": "spec",
2450
- "pair_verify": True,
2451
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2452
- }),
2453
- encoding="utf-8",
2454
- )
2455
- findings, source_verdicts = read_findings(devlyn)
2456
- summary = write_outputs(devlyn, findings, source_verdicts)
2457
- assert summary["verdict"] == "BLOCKED", summary
2458
- assert any(
2459
- finding.get("id") == "verify-pair-trigger-required-missing"
2460
- and "mode.pair-verify" in finding.get("message", "")
2461
- for finding in findings
2462
- ), findings
2463
-
2464
- (devlyn / "pipeline.state.json").write_text(
2465
- json.dumps({
2466
- "engine": "claude",
2467
- "mode": "spec",
2468
- "complexity": "large",
2469
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2470
- }),
2471
- encoding="utf-8",
2472
- )
2473
- findings, source_verdicts = read_findings(devlyn)
2474
- summary = write_outputs(devlyn, findings, source_verdicts)
2475
- assert summary["verdict"] == "BLOCKED", summary
2476
- assert any(
2477
- finding.get("id") == "verify-pair-trigger-required-missing"
2478
- and "complexity.large" in str(finding.get("message"))
2479
- for finding in findings
2480
- ), findings
2481
-
2482
- spec_path = devlyn / "spec.md"
2483
- spec_path.write_text(
2484
- '---\nid: "spec-high"\ncomplexity: high\n---\n\n# Spec\n',
2485
- encoding="utf-8",
2486
- )
2487
- (devlyn / "pipeline.state.json").write_text(
2488
- json.dumps({
2489
- "engine": "claude",
2490
- "mode": "spec",
2491
- "source": {"spec_path": str(spec_path)},
2492
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2493
- }),
2494
- encoding="utf-8",
2495
- )
2496
- findings, source_verdicts = read_findings(devlyn)
2497
- summary = write_outputs(devlyn, findings, source_verdicts)
2498
- assert summary["verdict"] == "BLOCKED", summary
2499
- assert any(
2500
- finding.get("id") == "verify-pair-trigger-required-missing"
2501
- and "spec.complexity.high" in str(finding.get("message"))
2502
- for finding in findings
2503
- ), findings
2504
-
2505
- spec_path.write_text(
2506
- '---\nid: "spec-large"\ncomplexity: large\n---\n\n# Spec\n',
2507
- encoding="utf-8",
2508
- )
2509
- findings, source_verdicts = read_findings(devlyn)
2510
- summary = write_outputs(devlyn, findings, source_verdicts)
2511
- assert summary["verdict"] == "BLOCKED", summary
2512
- assert any(
2513
- finding.get("id") == "verify-pair-trigger-required-missing"
2514
- and "spec.complexity.large" in str(finding.get("message"))
2515
- for finding in findings
2516
- ), findings
2517
-
2518
- spec_path.write_text(
2519
- "# Spec\n\n## Context\n\nsolo-headroom hypothesis: `SOLO_CLAUDE` should miss the priority rollback behavior; implementation token `rollback`.\n",
2520
- encoding="utf-8",
2521
- )
2522
- assert spec_has_solo_headroom_hypothesis(
2523
- {"source": {"spec_path": str(spec_path)}}
2524
- ) is False
2525
- (devlyn / "pipeline.state.json").write_text(
2526
- json.dumps({
2527
- "engine": "claude",
2528
- "mode": "spec",
2529
- "source": {"spec_path": str(spec_path)},
2530
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2531
- }),
2532
- encoding="utf-8",
2533
- )
2534
- findings, source_verdicts = read_findings(devlyn)
2535
- summary = write_outputs(devlyn, findings, source_verdicts)
2536
- assert summary["verdict"] == "PASS", summary
2537
- assert not any(
2538
- finding.get("id") == "verify-pair-trigger-required-missing"
2539
- and "spec.solo_headroom_hypothesis" in str(finding.get("message"))
2540
- for finding in findings
2541
- ), findings
2542
-
2543
- spec_path.write_text(
2544
- "# Spec\n\n## Context\n\nsolo-headroom hypothesis: solo_claude should miss the priority rollback behavior.\nObservable command: `node check.js` exposes behavior.\n",
2545
- encoding="utf-8",
2546
- )
2547
- assert spec_has_solo_headroom_hypothesis(
2548
- {"source": {"spec_path": str(spec_path)}}
2549
- ) is False
2550
-
2551
- spec_path.write_text(
2552
- "# Spec\n\n## Context\n\nsolo-headroom hypothesis: `SOLO_CLAUDE` should miss the priority rollback behavior; observable `SOLO_CLAUDE` exposes the miss.\n",
2553
- encoding="utf-8",
2554
- )
2555
- assert spec_has_solo_headroom_hypothesis(
2556
- {"source": {"spec_path": str(spec_path)}}
2557
- ) is False
2558
-
2559
- spec_path.write_text(
2560
- "# Spec\n\n## Context\n\nsolo-headroom hypothesis: solo_claude should miss behavior where observable `priority rollback` exposes the miss.\n",
2561
- encoding="utf-8",
2562
- )
2563
- assert spec_has_solo_headroom_hypothesis(
2564
- {"source": {"spec_path": str(spec_path)}}
2565
- ) is False
2566
-
2567
- spec_path.write_text(
2568
- "# Spec\n\n## Context\n\nsolo-headroom hypothesis: `SOLO_CLAUDE` should miss the priority rollback behavior exposed by `node check.js`.\n",
2569
- encoding="utf-8",
2570
- )
2571
- assert spec_has_solo_headroom_hypothesis(
2572
- {"source": {"spec_path": str(spec_path)}}
2573
- ) is True
2574
- (devlyn / "pipeline.state.json").write_text(
2575
- json.dumps({
2576
- "engine": "claude",
2577
- "mode": "spec",
2578
- "source": {"spec_path": str(spec_path)},
2579
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2580
- }),
2581
- encoding="utf-8",
2582
- )
2583
- findings, source_verdicts = read_findings(devlyn)
2584
- summary = write_outputs(devlyn, findings, source_verdicts)
2585
- assert summary["verdict"] == "BLOCKED", summary
2586
- assert any(
2587
- finding.get("id") == "verify-pair-trigger-required-missing"
2588
- and "spec.solo_headroom_hypothesis" in str(finding.get("message"))
2589
- for finding in findings
2590
- ), findings
2591
-
2592
- (devlyn / "pipeline.state.json").write_text(
2593
- json.dumps({
2594
- "engine": "claude",
2595
- "mode": "spec",
2596
- "source": {"spec_path": str(spec_path)},
2597
- "risk_profile": {
2598
- "high_risk": True,
2599
- "risk_probes_enabled": False,
2600
- "pair_default_enabled": True,
2601
- },
2602
- "phases": {
2603
- "verify": {
2604
- "verdict": "PASS",
2605
- "sub_verdicts": {},
2606
- "pair_trigger": {
2607
- "eligible": True,
2608
- "reasons": ["risk.high"],
2609
- "skipped_reason": None,
2610
- },
2611
- }
2612
- },
2613
- }),
2614
- encoding="utf-8",
2615
- )
2616
- findings, source_verdicts = read_findings(devlyn)
2617
- summary = write_outputs(devlyn, findings, source_verdicts)
2618
- assert summary["verdict"] == "BLOCKED", summary
2619
- assert any(
2620
- finding.get("id") == "verify-pair-trigger-reasons-incomplete"
2621
- and "spec.solo_headroom_hypothesis" in str(finding.get("message"))
2622
- for finding in findings
2623
- ), findings
2624
-
2625
- criteria_path = devlyn / "criteria.generated.md"
2626
- criteria_path.write_text(
2627
- "# Criteria\n\nsolo-headroom hypothesis: `SOLO_CLAUDE` should miss the priority rollback behavior exposed by `node check.js`.\n",
2628
- encoding="utf-8",
2629
- )
2630
- assert spec_has_solo_headroom_hypothesis(
2631
- {"source": {"criteria_path": str(criteria_path)}}
2632
- ) is True
2633
- (devlyn / "pipeline.state.json").write_text(
2634
- json.dumps({
2635
- "engine": "claude",
2636
- "mode": "free-form",
2637
- "source": {"criteria_path": str(criteria_path)},
2638
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
2639
- }),
2640
- encoding="utf-8",
2641
- )
2642
- findings, source_verdicts = read_findings(devlyn)
2643
- summary = write_outputs(devlyn, findings, source_verdicts)
2644
- assert summary["verdict"] == "BLOCKED", summary
2645
- assert any(
2646
- finding.get("id") == "verify-pair-trigger-required-missing"
2647
- and "spec.solo_headroom_hypothesis" in str(finding.get("message"))
2648
- for finding in findings
2649
- ), findings
2650
-
2651
- (devlyn / "verify-mechanical.findings.jsonl").write_text(
2652
- json.dumps({"id": "m0", "severity": "HIGH"}) + "\n",
2653
- encoding="utf-8",
2654
- )
2655
- findings, source_verdicts = read_findings(devlyn)
2656
- summary = write_outputs(devlyn, findings, source_verdicts)
2657
- assert summary["verdict"] == "NEEDS_WORK", summary
2658
- assert not any(
2659
- finding.get("id") == "verify-pair-trigger-required-missing"
2660
- for finding in findings
2661
- ), findings
2662
- (devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
2663
-
2664
- (devlyn / "pipeline.state.json").write_text(
2665
- json.dumps({
2666
- "engine": "claude",
2667
- "phases": {
2668
- "verify": {
2669
- "verdict": "PASS",
2670
- "sub_verdicts": {},
2671
- "pair_trigger": {
2672
- "eligible": "true",
2673
- "reasons": ["risk.high"],
2674
- "skipped_reason": None,
2675
- },
2676
- }
2677
- }
2678
- }),
2679
- encoding="utf-8",
2680
- )
2681
- findings, source_verdicts = read_findings(devlyn)
2682
- summary = write_outputs(devlyn, findings, source_verdicts)
2683
- assert summary["verdict"] == "BLOCKED", summary
2684
- assert any(
2685
- finding.get("id") == "verify-pair-trigger-eligible-malformed"
2686
- for finding in findings
2687
- ), findings
2688
-
2689
- (devlyn / "pipeline.state.json").write_text(
2690
- json.dumps({
2691
- "engine": "claude",
2692
- "phases": {
2693
- "verify": {
2694
- "verdict": "PASS",
2695
- "sub_verdicts": {},
2696
- "pair_trigger": {
2697
- "eligible": True,
2698
- "reasons": "risk.high",
2699
- "skipped_reason": None,
2700
- },
2701
- }
2702
- }
2703
- }),
2704
- encoding="utf-8",
2705
- )
2706
- findings, source_verdicts = read_findings(devlyn)
2707
- summary = write_outputs(devlyn, findings, source_verdicts)
2708
- assert summary["verdict"] == "BLOCKED", summary
2709
- assert any(
2710
- finding.get("id") == "verify-pair-trigger-reasons-malformed"
2711
- for finding in findings
2712
- ), findings
2713
-
2714
- (devlyn / "pipeline.state.json").write_text(
2715
- json.dumps({
2716
- "engine": "claude",
2717
- "phases": {
2718
- "verify": {
2719
- "verdict": "PASS",
2720
- "sub_verdicts": {},
2721
- "pair_trigger": {
2722
- "eligible": True,
2723
- "reasons": ["risk.high", "looks-hard"],
2724
- "skipped_reason": None,
2725
- },
2726
- }
2727
- }
2728
- }),
2729
- encoding="utf-8",
2730
- )
2731
- findings, source_verdicts = read_findings(devlyn)
2732
- summary = write_outputs(devlyn, findings, source_verdicts)
2733
- assert summary["verdict"] == "BLOCKED", summary
2734
- assert any(
2735
- finding.get("id") == "verify-pair-trigger-reasons-unknown"
2736
- and "only include known" in finding.get("message", "")
2737
- for finding in findings
2738
- ), findings
2739
-
2740
- (devlyn / "pipeline.state.json").write_text(
2741
- json.dumps({
2742
- "engine": "claude",
2743
- "phases": {
2744
- "verify": {
2745
- "verdict": "PASS",
2746
- "sub_verdicts": {},
2747
- "pair_trigger": {
2748
- "eligible": True,
2749
- "reasons": ["risk high"],
2750
- "skipped_reason": None,
2751
- },
2752
- }
2753
- }
2754
- }),
2755
- encoding="utf-8",
2756
- )
2757
- findings, source_verdicts = read_findings(devlyn)
2758
- summary = write_outputs(devlyn, findings, source_verdicts)
2759
- assert summary["verdict"] == "BLOCKED", summary
2760
- assert any(
2761
- finding.get("id") == "verify-pair-trigger-reasons-unknown"
2762
- and "include a known" in finding.get("message", "")
2763
- for finding in findings
2764
- ), findings
2765
-
2766
- (devlyn / "pipeline.state.json").write_text(
2767
- json.dumps({
2768
- "engine": "claude",
2769
- "phases": {
2770
- "verify": {
2771
- "verdict": "PASS",
2772
- "sub_verdicts": {},
2773
- "pair_trigger": {
2774
- "eligible": True,
2775
- "reasons": ["risk_profile.high_risk", "risk_probes_enabled"],
2776
- "skipped_reason": None,
2777
- },
2778
- }
2779
- }
2780
- }),
2781
- encoding="utf-8",
2782
- )
2783
- findings, source_verdicts = read_findings(devlyn)
2784
- summary = write_outputs(devlyn, findings, source_verdicts)
2785
- assert summary["verdict"] == "BLOCKED", summary
2786
- assert any(
2787
- finding.get("id") == "verify-pair-trigger-reasons-unknown"
2788
- and "include a known" in finding.get("message", "")
2789
- for finding in findings
2790
- ), findings
2791
-
2792
- (devlyn / "pipeline.state.json").write_text(
2793
- json.dumps({
2794
- "engine": "claude",
2795
- "phases": {
2796
- "verify": {
2797
- "verdict": "PASS",
2798
- "sub_verdicts": {},
2799
- "pair_trigger": {
2800
- "eligible": True,
2801
- "reasons": ["risk.high", 3],
2802
- "skipped_reason": None,
2803
- },
2804
- }
2805
- }
2806
- }),
2807
- encoding="utf-8",
2808
- )
2809
- findings, source_verdicts = read_findings(devlyn)
2810
- summary = write_outputs(devlyn, findings, source_verdicts)
2811
- assert summary["verdict"] == "BLOCKED", summary
2812
- assert any(
2813
- finding.get("id") == "verify-pair-trigger-reasons-malformed"
2814
- for finding in findings
2815
- ), findings
2816
-
2817
- (devlyn / "pipeline.state.json").write_text(
2818
- json.dumps({
2819
- "engine": "claude",
2820
- "phases": {
2821
- "verify": {
2822
- "verdict": "PASS",
2823
- "sub_verdicts": {},
2824
- "pair_trigger": {
2825
- "eligible": True,
2826
- "reasons": [],
2827
- "skipped_reason": None,
2828
- },
2829
- }
2830
- }
2831
- }),
2832
- encoding="utf-8",
2833
- )
2834
- findings, source_verdicts = read_findings(devlyn)
2835
- summary = write_outputs(devlyn, findings, source_verdicts)
2836
- assert summary["verdict"] == "BLOCKED", summary
2837
- assert any(
2838
- finding.get("id") == "verify-pair-trigger-reasons-empty"
2839
- for finding in findings
2840
- ), findings
2841
-
2842
- (devlyn / "pipeline.state.json").write_text(
2843
- json.dumps({
2844
- "engine": "claude",
2845
- "phases": {
2846
- "verify": {
2847
- "verdict": "PASS",
2848
- "sub_verdicts": {},
2849
- "pair_trigger": {
2850
- "eligible": True,
2851
- "reasons": ["risk.high"],
2852
- "skipped_reason": "user_no_pair",
2853
- },
2854
- }
2855
- }
2856
- }),
2857
- encoding="utf-8",
2858
- )
2859
- findings, source_verdicts = read_findings(devlyn)
2860
- summary = write_outputs(devlyn, findings, source_verdicts)
2861
- assert summary["verdict"] == "BLOCKED", summary
2862
- assert any(
2863
- finding.get("id") == "verify-pair-trigger-skip-contradiction"
2864
- for finding in findings
2865
- ), findings
2866
-
2867
- (devlyn / "pipeline.state.json").write_text(
2868
- json.dumps({
2869
- "engine": "claude",
2870
- "phases": {
2871
- "verify": {
2872
- "verdict": "PASS",
2873
- "sub_verdicts": {},
2874
- "pair_trigger": {
2875
- "eligible": False,
2876
- "reasons": ["risk.high"],
2877
- "skipped_reason": "user_no_pair",
2878
- },
2879
- }
2880
- }
2881
- }),
2882
- encoding="utf-8",
2883
- )
2884
- findings, source_verdicts = read_findings(devlyn)
2885
- summary = write_outputs(devlyn, findings, source_verdicts)
2886
- assert summary["verdict"] == "BLOCKED", summary
2887
- assert any(
2888
- finding.get("id") == "verify-pair-trigger-ineligible-reasons"
2889
- for finding in findings
2890
- ), findings
2891
-
2892
- (devlyn / "pipeline.state.json").write_text(
2893
- json.dumps({
2894
- "engine": "claude",
2895
- "mode": "spec",
2896
- "risk_profile": {
2897
- "high_risk": True,
2898
- "risk_probes_enabled": False,
2899
- "pair_default_enabled": True,
2900
- },
2901
- "phases": {
2902
- "verify": {
2903
- "verdict": "PASS",
2904
- "sub_verdicts": {},
2905
- "pair_trigger": {
2906
- "eligible": False,
2907
- "reasons": [],
2908
- "skipped_reason": None,
2909
- },
2910
- }
2911
- },
2912
- }),
2913
- encoding="utf-8",
2914
- )
2915
- findings, source_verdicts = read_findings(devlyn)
2916
- summary = write_outputs(devlyn, findings, source_verdicts)
2917
- assert summary["verdict"] == "BLOCKED", summary
2918
- assert any(
2919
- finding.get("id") == "verify-pair-trigger-ineligible-unjustified"
2920
- and "risk.high" in str(finding.get("message"))
2921
- for finding in findings
2922
- ), findings
2923
-
2924
- (devlyn / "pipeline.state.json").write_text(
2925
- json.dumps({
2926
- "engine": "claude",
2927
- "mode": "spec",
2928
- "risk_profile": {
2929
- "high_risk": True,
2930
- "risk_probes_enabled": True,
2931
- "pair_default_enabled": True,
2932
- },
2933
- "phases": {
2934
- "verify": {
2935
- "verdict": "PASS",
2936
- "sub_verdicts": {},
2937
- "pair_trigger": {
2938
- "eligible": False,
2939
- "reasons": [],
2940
- "skipped_reason": "user_no_pair",
2941
- },
2942
- }
2943
- },
2944
- }),
2945
- encoding="utf-8",
2946
- )
2947
- findings, source_verdicts = read_findings(devlyn)
2948
- summary = write_outputs(devlyn, findings, source_verdicts)
2949
- assert summary["verdict"] == "BLOCKED", summary
2950
- assert any(
2951
- finding.get("id") == "verify-pair-trigger-user-no-pair-unsupported"
2952
- and "pair_default_enabled false" in str(finding.get("message"))
2953
- for finding in findings
2954
- ), findings
2955
-
2956
- (devlyn / "pipeline.state.json").write_text(
2957
- json.dumps({
2958
- "engine": "claude",
2959
- "pair_verify": True,
2960
- "risk_profile": {
2961
- "high_risk": True,
2962
- "risk_probes_enabled": False,
2963
- "pair_default_enabled": False,
2964
- },
2965
- "phases": {
2966
- "verify": {
2967
- "verdict": "PASS",
2968
- "sub_verdicts": {},
2969
- "pair_trigger": {
2970
- "eligible": False,
2971
- "reasons": [],
2972
- "skipped_reason": "user_no_pair",
2973
- },
2974
- }
2975
- },
2976
- }),
2977
- encoding="utf-8",
2978
- )
2979
- findings, source_verdicts = read_findings(devlyn)
2980
- summary = write_outputs(devlyn, findings, source_verdicts)
2981
- assert summary["verdict"] == "BLOCKED", summary
2982
- assert any(
2983
- finding.get("id") == "verify-pair-trigger-conflicting-pair-flags"
2984
- for finding in findings
2985
- ), findings
2986
-
2987
- (devlyn / "pipeline.state.json").write_text(
2988
- json.dumps({
2989
- "engine": "claude",
2990
- "mode": "spec",
2991
- "risk_profile": {
2992
- "high_risk": True,
2993
- "risk_probes_enabled": True,
2994
- "pair_default_enabled": False,
2995
- },
2996
- "phases": {
2997
- "verify": {
2998
- "verdict": "PASS",
2999
- "sub_verdicts": {},
3000
- "pair_trigger": {
3001
- "eligible": False,
3002
- "reasons": [],
3003
- "skipped_reason": "user_no_pair",
3004
- },
3005
- }
3006
- },
3007
- }),
3008
- encoding="utf-8",
3009
- )
3010
- findings, source_verdicts = read_findings(devlyn)
3011
- summary = write_outputs(devlyn, findings, source_verdicts)
3012
- assert summary["verdict"] == "PASS", summary
3013
- assert not any(
3014
- finding.get("id") == "verify-pair-trigger-required-missing"
3015
- for finding in findings
3016
- ), findings
3017
-
3018
- (devlyn / "pipeline.state.json").write_text(
3019
- json.dumps({
3020
- "engine": "claude",
3021
- "phases": {
3022
- "verify": {
3023
- "verdict": "PASS",
3024
- "sub_verdicts": {},
3025
- "pair_trigger": {
3026
- "eligible": False,
3027
- "reasons": [],
3028
- "skipped_reason": ["user_no_pair"],
3029
- },
3030
- }
3031
- }
3032
- }),
3033
- encoding="utf-8",
3034
- )
3035
- findings, source_verdicts = read_findings(devlyn)
3036
- summary = write_outputs(devlyn, findings, source_verdicts)
3037
- assert summary["verdict"] == "BLOCKED", summary
3038
- assert any(
3039
- finding.get("id") == "verify-pair-trigger-skipped-reason-malformed"
3040
- for finding in findings
3041
- ), findings
3042
-
3043
- (devlyn / "pipeline.state.json").write_text(
3044
- json.dumps({
3045
- "engine": "claude",
3046
- "phases": {
3047
- "verify": {
3048
- "verdict": "PASS",
3049
- "sub_verdicts": {},
3050
- "pair_trigger": {
3051
- "eligible": False,
3052
- "reasons": [],
3053
- "skipped_reason": "codex_unavailable",
3054
- },
3055
- }
3056
- }
3057
- }),
3058
- encoding="utf-8",
3059
- )
3060
- findings, source_verdicts = read_findings(devlyn)
3061
- summary = write_outputs(devlyn, findings, source_verdicts)
3062
- assert summary["verdict"] == "BLOCKED", summary
3063
- assert any(
3064
- finding.get("id") == "verify-pair-trigger-skipped-reason-unsupported"
3065
- for finding in findings
3066
- ), findings
3067
-
3068
- (devlyn / "pipeline.state.json").write_text(
3069
- json.dumps({
3070
- "engine": "claude",
3071
- "phases": {
3072
- "verify": {
3073
- "verdict": "PASS",
3074
- "sub_verdicts": {},
3075
- "pair_trigger": {
3076
- "eligible": False,
3077
- "reasons": [],
3078
- "skipped_reason": "mechanical_blocker",
3079
- },
3080
- }
3081
- }
3082
- }),
3083
- encoding="utf-8",
3084
- )
3085
- findings, source_verdicts = read_findings(devlyn)
3086
- summary = write_outputs(devlyn, findings, source_verdicts)
3087
- assert summary["verdict"] == "BLOCKED", summary
3088
- assert any(
3089
- finding.get("id") == "verify-pair-trigger-mechanical-blocker-unsupported"
3090
- for finding in findings
3091
- ), findings
3092
-
3093
- (devlyn / "verify-mechanical.findings.jsonl").write_text(
3094
- json.dumps({"id": "m1", "severity": "HIGH"}) + "\n",
3095
- encoding="utf-8",
3096
- )
3097
- findings, source_verdicts = read_findings(devlyn)
3098
- summary = write_outputs(devlyn, findings, source_verdicts)
3099
- assert summary["verdict"] == "NEEDS_WORK", summary
3100
- assert not any(
3101
- finding.get("id") == "verify-pair-trigger-mechanical-blocker-unsupported"
3102
- for finding in findings
3103
- ), findings
3104
- (devlyn / "pipeline.state.json").write_text(
3105
- json.dumps({
3106
- "engine": "claude",
3107
- "phases": {
3108
- "verify": {
3109
- "verdict": None,
3110
- "sub_verdicts": None,
3111
- "judge_durations_ms": {"judge": None, "pair_judge": None},
3112
- }
3113
- }
3114
- }),
3115
- encoding="utf-8",
3116
- )
3117
- (devlyn / "verify.findings.jsonl").unlink()
3118
- findings, source_verdicts = read_findings(devlyn)
3119
- summary = write_outputs(devlyn, findings, source_verdicts)
3120
- write_state(devlyn, summary)
3121
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
3122
- assert summary["verdict"] == "NEEDS_WORK", summary
3123
- assert summary["source_verdicts"]["judge"] is None, summary
3124
- assert state["phases"]["verify"]["verdict"] == "NEEDS_WORK", state
3125
- assert state["phases"]["verify"]["sub_verdicts"]["judge"] is None, state
3126
- (devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
3127
- (devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
3128
-
3129
- (devlyn / "pipeline.state.json").write_text(
3130
- json.dumps({
3131
- "engine": "claude",
3132
- "phases": {
3133
- "verify": {
3134
- "verdict": "PASS",
3135
- "sub_verdicts": {},
3136
- "pair_trigger": {
3137
- "eligible": False,
3138
- "reasons": [],
3139
- "skipped_reason": "primary_judge_blocker",
3140
- },
3141
- }
3142
- }
3143
- }),
3144
- encoding="utf-8",
3145
- )
3146
- findings, source_verdicts = read_findings(devlyn)
3147
- summary = write_outputs(devlyn, findings, source_verdicts)
3148
- assert summary["verdict"] == "BLOCKED", summary
3149
- assert any(
3150
- finding.get("id") == "verify-pair-trigger-primary-judge-blocker-unsupported"
3151
- for finding in findings
3152
- ), findings
3153
-
3154
- # Self-test: preknown_primary_blocker_requires_pair.
3155
- (devlyn / "verify.pair.findings.jsonl").unlink(missing_ok=True)
3156
- (devlyn / "verify.findings.jsonl").write_text(
3157
- json.dumps({"id": "j-preknown", "severity": "HIGH"}) + "\n",
3158
- encoding="utf-8",
3159
- )
3160
- (devlyn / "pipeline.state.json").write_text(
3161
- json.dumps({
3162
- "version": "2.0",
3163
- "engine": "claude",
3164
- "mode": "spec",
3165
- "pair_verify": True,
3166
- "phases": {
3167
- "verify": {
3168
- "verdict": "PASS",
3169
- "sub_verdicts": {},
3170
- "pair_trigger": {
3171
- "eligible": False,
3172
- "reasons": [],
3173
- "skipped_reason": "primary_judge_blocker",
3174
- },
3175
- }
3176
- },
3177
- }),
3178
- encoding="utf-8",
3179
- )
3180
- findings, source_verdicts = read_findings(devlyn)
3181
- summary = write_outputs(devlyn, findings, source_verdicts)
3182
- assert summary["verdict"] == "BLOCKED", summary
3183
- assert any(
3184
- finding.get("id") == "verify-pair-trigger-primary-judge-blocker-preknown"
3185
- and "mode.pair-verify" in str(finding.get("message"))
3186
- for finding in findings
3187
- ), findings
3188
-
3189
- # Self-test: schema-v3 default pair reason is required before merge.
3190
- (devlyn / "pipeline.state.json").write_text(
3191
- json.dumps({
3192
- "version": "3.0",
3193
- "engine": "claude",
3194
- "mode": "spec",
3195
- "pair_verify": True,
3196
- "phases": {
3197
- "verify": {
3198
- "verdict": "PASS",
3199
- "sub_verdicts": {},
3200
- "pair_trigger": {
3201
- "eligible": True,
3202
- "reasons": ["mode.pair-verify"],
3203
- "skipped_reason": None,
3204
- },
3205
- }
3206
- },
3207
- }),
3208
- encoding="utf-8",
3209
- )
3210
- (devlyn / "verify.pair.findings.jsonl").write_text(
3211
- json.dumps({"id": "p-preknown", "severity": "LOW"}) + "\n",
3212
- encoding="utf-8",
3213
- )
3214
- (devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
3215
- findings, source_verdicts = read_findings(devlyn)
3216
- summary = write_outputs(devlyn, findings, source_verdicts)
3217
- assert summary["verdict"] == "BLOCKED", summary
3218
- assert any(
3219
- finding.get("id") == "verify-pair-trigger-reasons-incomplete"
3220
- and "pair.default" in str(finding.get("message"))
3221
- for finding in findings
3222
- ), findings
3223
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
3224
- state["phases"]["verify"]["pair_trigger"]["reasons"].insert(0, "pair.default")
3225
- (devlyn / "pipeline.state.json").write_text(json.dumps(state), encoding="utf-8")
3226
- findings, source_verdicts = read_findings(devlyn)
3227
- summary = write_outputs(devlyn, findings, source_verdicts)
3228
- assert summary["verdict"] == "NEEDS_WORK", summary
3229
- assert summary["source_verdicts"]["judge"] == "NEEDS_WORK", summary
3230
- assert summary["source_verdicts"]["pair_judge"] == "PASS_WITH_ISSUES", summary
3231
- assert '"id":"p-preknown"' in (
3232
- devlyn / "verify-merged.findings.jsonl"
3233
- ).read_text(encoding="utf-8"), findings
3234
- (devlyn / "codex-judge.stdout").unlink()
3235
-
3236
- # Self-test: archived-v2 sequential primary blocker remains legal.
3237
- (devlyn / "verify.pair.findings.jsonl").unlink()
3238
- (devlyn / "pipeline.state.json").write_text(
3239
- json.dumps({
3240
- "version": "2.0",
3241
- "engine": "claude",
3242
- "mode": "spec",
3243
- "phases": {
3244
- "verify": {
3245
- "verdict": "PASS",
3246
- "sub_verdicts": {},
3247
- "pair_trigger": {
3248
- "eligible": False,
3249
- "reasons": [],
3250
- "skipped_reason": "primary_judge_blocker",
3251
- },
3252
- }
3253
- },
3254
- }),
3255
- encoding="utf-8",
3256
- )
3257
- findings, source_verdicts = read_findings(devlyn)
3258
- summary = write_outputs(devlyn, findings, source_verdicts)
3259
- assert summary["verdict"] == "NEEDS_WORK", summary
3260
- assert summary["source_verdicts"]["pair_judge"] is None, summary
3261
- assert not any(
3262
- finding.get("id") in {
3263
- "verify-pair-trigger-primary-judge-blocker-unsupported",
3264
- "verify-pair-trigger-primary-judge-blocker-preknown",
3265
- }
3266
- for finding in findings
3267
- ), findings
3268
- # Replay the same state as schema v3 and require the retired-skip blocker.
3269
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
3270
- state["version"] = "3.0"
3271
- (devlyn / "pipeline.state.json").write_text(json.dumps(state), encoding="utf-8")
3272
- findings, source_verdicts = read_findings(devlyn)
3273
- summary = write_outputs(devlyn, findings, source_verdicts)
3274
- assert summary["verdict"] == "BLOCKED", summary
3275
- assert any(
3276
- finding.get("id") == "verify-pair-trigger-primary-judge-blocker-retired"
3277
- for finding in findings
3278
- ), findings
3279
- (devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
3280
- (devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
3281
-
3282
- (devlyn / "pipeline.state.json").write_text(
3283
- json.dumps({
3284
- "engine": "claude",
3285
- "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
3286
- "verify": {
3287
- "pair_trigger": {
3288
- "eligible": True,
3289
- "reasons": ["looks-hard"],
3290
- "skipped_reason": None,
3291
- }
3292
- },
3293
- }),
3294
- encoding="utf-8",
3295
- )
3296
- findings, source_verdicts = read_findings(devlyn)
3297
- summary = write_outputs(devlyn, findings, source_verdicts)
3298
- assert summary["verdict"] == "BLOCKED", summary
3299
- assert any(
3300
- finding.get("id") == "verify-pair-trigger-reasons-unknown"
3301
- for finding in findings
3302
- ), findings
3303
-
3304
- # stdout-only spawn evidence: no pair findings file at all; a clean
3305
- # PASS stdout from a claude judge promotes pair_judge null -> PASS.
3306
- (devlyn / "verify.pair.findings.jsonl").unlink()
3307
- (devlyn / "pipeline.state.json").write_text(
3308
- json.dumps({"engine": "codex", "phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
3309
- encoding="utf-8",
3310
- )
3311
- (devlyn / "claude-judge.stdout").write_text("PASS\n", encoding="utf-8")
3312
- findings, source_verdicts = read_findings(devlyn)
3313
- summary = write_outputs(devlyn, findings, source_verdicts)
3314
- write_state(devlyn, summary)
3315
- state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
3316
- assert summary["verdict"] == "PASS", summary
3317
- assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "PASS", state
3318
- (devlyn / "claude-judge.stdout").unlink()
3319
-
3320
- # The primary capture must not be misattributed to the OTHER-engine
3321
- # pair seat merely because both adapters use *-judge.stdout names.
3322
- (devlyn / "pipeline.state.json").write_text(
3323
- json.dumps({"engine": "claude", "phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
3324
- encoding="utf-8",
3325
- )
3326
- (devlyn / "claude-judge.stdout").write_text("primary prose\n", encoding="utf-8")
3327
- (devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
3328
- findings, source_verdicts = read_findings(devlyn)
3329
- assert source_verdicts["pair_judge"] == "PASS", source_verdicts
3330
- assert not any(finding["source"] == "pair_judge" for finding in findings), findings
3331
- (devlyn / "codex-judge.stdout").write_text("pair prose\n", encoding="utf-8")
3332
- findings, source_verdicts = read_findings(devlyn)
3333
- assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
3334
- assert any(finding["id"] == "verify-pair-emission-contract-violated" for finding in findings)
3335
- (devlyn / "claude-judge.stdout").unlink()
3336
- (devlyn / "codex-judge.stdout").unlink()
3337
-
3338
- # A primary capture cannot become pair evidence when the state needed
3339
- # to identify the primary seat is absent, incomplete, or malformed.
3340
- (devlyn / "claude-judge.stdout").write_text("primary prose\n", encoding="utf-8")
3341
- (devlyn / "pipeline.state.json").unlink()
3342
- findings, source_verdicts = read_findings(devlyn)
3343
- assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
3344
- assert any(finding["id"] == "verify-state-missing" for finding in findings), findings
3345
- assert not any(finding.get("file") == "claude-judge.stdout" for finding in findings), findings
3346
-
3347
- (devlyn / "pipeline.state.json").write_text(
3348
- json.dumps({"version": "3.0", "phases": {"verify": {}}}), encoding="utf-8",
3349
- )
3350
- findings, source_verdicts = read_findings(devlyn)
3351
- assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
3352
- assert any(finding.get("rule_id") == "verify.state.engine-malformed" for finding in findings), findings
3353
-
3354
- (devlyn / "pipeline.state.json").write_text("{", encoding="utf-8")
3355
- findings, source_verdicts = read_findings(devlyn)
3356
- assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
3357
- assert any(finding["id"] == "verify-pair-trigger-state-malformed" for finding in findings), findings
3358
- (devlyn / "claude-judge.stdout").unlink()
3359
-
3360
- # iter-0083: canonical summary verdict conservation.
3361
- iter_0083_paths = (
3362
- "pipeline.state.json",
3363
- "verify-mechanical.findings.jsonl",
3364
- "verify.findings.jsonl",
3365
- "verify.pair.findings.jsonl",
3366
- "verify.pair-judge.findings.jsonl",
3367
- "pair-judge.summary.json",
3368
- "codex-primary-judge.summary.json",
3369
- "grok-judge.summary.json",
3370
- "verify.pair.timeout.json",
3371
- "codex-judge.stdout",
3372
- "claude-judge.stdout",
3373
- )
3374
-
3375
- def iter_0083_reset() -> None:
3376
- for name in iter_0083_paths:
3377
- (devlyn / name).unlink(missing_ok=True)
3378
-
3379
- def iter_0083_case(
3380
- summary_payload: object | None,
3381
- severity: str | None,
3382
- *,
3383
- carrier: str | None = "verify.pair.findings.jsonl",
3384
- other_summary: str | None = None,
3385
- timeout: bool = False,
3386
- ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
3387
- iter_0083_reset()
3388
- (devlyn / "pipeline.state.json").write_text(
3389
- json.dumps({"engine": "claude", "phases": {"verify": {}}}), encoding="utf-8",
3390
- )
3391
- (devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
3392
- (devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
3393
- if carrier is not None:
3394
- content = (
3395
- json.dumps({"id": "iter-0083", "severity": severity}) + "\n"
3396
- if severity
3397
- else ""
3398
- )
3399
- (devlyn / carrier).write_text(content, encoding="utf-8")
3400
- if summary_payload is not None:
3401
- content = (
3402
- summary_payload
3403
- if isinstance(summary_payload, str)
3404
- else json.dumps(summary_payload)
3405
- )
3406
- (devlyn / "pair-judge.summary.json").write_text(content, encoding="utf-8")
3407
- if other_summary is not None:
3408
- (devlyn / other_summary).write_text(
3409
- json.dumps({"verdict": "BLOCKED"}), encoding="utf-8"
3410
- )
3411
- if timeout:
3412
- (devlyn / "verify.pair.timeout.json").write_text(
3413
- json.dumps({"engine": "codex", "budget_seconds": 600}),
3414
- encoding="utf-8",
3415
- )
3416
- case_findings, case_source_verdicts = read_findings(devlyn)
3417
- return case_findings, write_outputs(devlyn, case_findings, case_source_verdicts)
3418
-
3419
- for case_id, verdict, severity, expected in (
3420
- ("P1", "NEEDS_WORK", "INFO", "NEEDS_WORK"),
3421
- ("P2", "NEEDS_WORK", "LOW", "NEEDS_WORK"),
3422
- ("P3", "NEEDS_WORK", "MEDIUM", "NEEDS_WORK"),
3423
- ("P4", "PASS", "HIGH", "NEEDS_WORK"),
3424
- ("P5", "BLOCKED", "INFO", "BLOCKED"),
3425
- ("P6", "FAIL", "INFO", "NEEDS_WORK"),
3426
- ("P7", "PASS", None, "PASS"),
3427
- ):
3428
- _, summary = iter_0083_case({"verdict": verdict}, severity)
3429
- assert summary["source_verdicts"]["pair_judge"] == expected, case_id
3430
- assert summary["verdict"] == expected, case_id
3431
-
3432
- for case_id, severity, expected in (
3433
- ("P8-INFO", "INFO", "PASS"),
3434
- ("P8-LOW", "LOW", "PASS_WITH_ISSUES"),
3435
- ):
3436
- _, summary = iter_0083_case(None, severity)
3437
- assert summary["source_verdicts"]["pair_judge"] == expected, case_id
3438
- assert summary["verdict"] == expected, case_id
3439
-
3440
- def assert_iter_0083_blocked(
3441
- case_id: str,
3442
- case_findings: list[dict[str, Any]],
3443
- summary: dict[str, Any],
3444
- ) -> None:
3445
- assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", case_id
3446
- assert summary["verdict"] == "BLOCKED", case_id
3447
- assert any(
3448
- finding.get("source") == "pair_judge"
3449
- and finding.get("severity") == "CRITICAL"
3450
- and finding.get("file") == "pair-judge.summary.json"
3451
- for finding in case_findings
3452
- ), case_id
3453
-
3454
- for case_id, payload in (
3455
- ("N1-malformed", "{"),
3456
- ("N1-non-object", []),
3457
- ("N1-unknown", {"verdict": "UNKNOWN"}),
3458
- ("N2", {}),
3459
- ("N7", {"verdict": "TIMEOUT"}),
3460
- ):
3461
- case_findings, summary = iter_0083_case(payload, "INFO")
3462
- assert_iter_0083_blocked(case_id, case_findings, summary)
3463
-
3464
- _, summary = iter_0083_case({"verdict": "BLOCKED"}, None, carrier=None)
3465
- assert summary["source_verdicts"]["pair_judge"] is None, summary
3466
- assert summary["verdict"] == "PASS", summary
3467
-
3468
- _, summary = iter_0083_case(
3469
- {"verdict": "PASS"},
3470
- "INFO",
3471
- other_summary="codex-primary-judge.summary.json",
3472
- )
3473
- assert summary["source_verdicts"]["pair_judge"] == "PASS", "N4"
3474
- assert summary["verdict"] == "PASS", "N4"
3475
-
3476
- _, summary = iter_0083_case(
3477
- None,
3478
- "INFO",
3479
- other_summary="grok-judge.summary.json",
3480
- )
3481
- assert summary["source_verdicts"]["pair_judge"] == "PASS", "N5"
3482
- assert summary["verdict"] == "PASS", "N5"
3483
-
3484
- _, summary = iter_0083_case(
3485
- {"verdict": "NEEDS_WORK"},
3486
- "INFO",
3487
- carrier="verify.pair-judge.findings.jsonl",
3488
- )
3489
- assert summary["source_verdicts"]["pair_judge"] == "NEEDS_WORK", "N6"
3490
- assert summary["verdict"] == "NEEDS_WORK", "N6"
3491
-
3492
- case_findings, summary = iter_0083_case({"verdict": "BLOCKED"}, None, timeout=True)
3493
- assert case_findings == [], "VERIFY-JUDGE-001"
3494
- assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", "VERIFY-JUDGE-001"
3495
- assert summary["verdict"] == "BLOCKED", "VERIFY-JUDGE-001"
3496
- return 0
3497
-
3498
-
3499
- def main() -> int:
3500
- parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
3501
- parser.add_argument("--devlyn-dir", default=".devlyn")
3502
- parser.add_argument("--write-state", action="store_true")
3503
- parser.add_argument("--self-test", action="store_true")
3504
- args = parser.parse_args()
3505
- if args.self_test:
3506
- return self_test()
3507
-
3508
- devlyn = pathlib.Path(args.devlyn_dir)
3509
- if not devlyn.is_dir():
3510
- sys.stderr.write(f"error: {devlyn} is not a directory\n")
3511
- return 1
3512
- if args.write_state:
3513
- lock = runpy.run_path(str(pathlib.Path(__file__).with_name("platform-support.py")))["file_lock"]
3514
- with lock(devlyn.resolve() / "pipeline.state.lock", blocking=True):
3515
- findings, source_verdicts = read_findings(devlyn)
3516
- summary = write_outputs(devlyn, findings, source_verdicts)
3517
- write_state(devlyn, summary)
3518
- print(json.dumps(summary, sort_keys=True))
3519
- return 0
3520
- findings, source_verdicts = read_findings(devlyn)
3521
- summary = write_outputs(devlyn, findings, source_verdicts)
3522
- print(json.dumps(summary, sort_keys=True))
3523
- return 0
3524
-
3525
-
3526
- if __name__ == "__main__":
3527
- runpy.run_path(str(pathlib.Path(__file__).with_name("platform-support.py")))["configure_utf8"]()
3528
- raise SystemExit(main())