devlyn-cli 4.0.1 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +11 -95
- package/CLAUDE.md +7 -160
- package/LICENSE +21 -0
- package/README.md +83 -114
- package/bin/devlyn.js +330 -472
- package/bin/instruction-templates.json +120 -2
- package/bin/instructions.js +35 -8
- package/config/skills/_shared/adapters/claude.md +8 -85
- package/config/skills/_shared/adapters/codex.md +14 -34
- package/config/skills/_shared/adapters/omp.md +1 -11
- package/config/skills/_shared/codex-monitored.sh +4 -54
- package/config/skills/_shared/engine-doctor.sh +18 -58
- package/config/skills/_shared/engine-preflight.md +15 -47
- package/config/skills/_shared/expected-contract.py +242 -0
- package/config/skills/_shared/expected.schema.json +10 -95
- package/config/skills/_shared/invocation-receipt.py +26 -757
- package/config/skills/_shared/platform-support.py +37 -28
- package/config/skills/_shared/role-config.py +100 -421
- package/config/skills/_shared/runtime-principles.md +17 -93
- package/config/skills/_shared/task-complete.py +490 -285
- package/config/skills/_shared/task-completion.md +235 -0
- package/config/skills/devlyn-engines/SKILL.md +9 -11
- package/config/skills/devlyn-ideate/SKILL.md +57 -107
- package/config/skills/devlyn-ideate/references/elicitation.md +18 -99
- package/config/skills/devlyn-ideate/references/loop.md +110 -0
- package/config/skills/devlyn-ideate/references/package-format.md +128 -0
- package/config/skills/devlyn-ideate/scripts/acceptance.py +795 -0
- package/config/skills/devlyn-ideate/scripts/queue.py +1780 -0
- package/package.json +2 -3
- package/config/commit-conventions.md +0 -28
- package/config/skills/_shared/adapters/README.md +0 -74
- package/config/skills/_shared/adapters/grok.md +0 -100
- package/config/skills/_shared/archive_run.py +0 -969
- package/config/skills/_shared/codex-config.md +0 -81
- package/config/skills/_shared/collect-codex-findings.py +0 -505
- package/config/skills/_shared/finish-gate.py +0 -521
- package/config/skills/_shared/grok-anchor-guard.py +0 -78
- package/config/skills/_shared/judge-output-parser.py +0 -200
- package/config/skills/_shared/judge-role-evidence.py +0 -320
- package/config/skills/_shared/phase-prompt-render.py +0 -183
- package/config/skills/_shared/process-evidence.py +0 -926
- package/config/skills/_shared/resolve-bootstrap.py +0 -1410
- package/config/skills/_shared/resolve-stop-hook.py +0 -359
- package/config/skills/_shared/run-bounded.py +0 -58
- package/config/skills/_shared/spec-verify-check.py +0 -5083
- package/config/skills/_shared/state-phase-write.py +0 -5989
- package/config/skills/_shared/terminal-claim-check.py +0 -777
- package/config/skills/_shared/verify-merge-findings.py +0 -3528
- package/config/skills/devlyn-design-ui/SKILL.md +0 -650
- package/config/skills/devlyn-ideate/references/from-spec-mode.md +0 -61
- package/config/skills/devlyn-ideate/references/project-mode.md +0 -80
- package/config/skills/devlyn-ideate/references/spec-template.md +0 -121
- package/config/skills/devlyn-ideate/references/templates/decision.md +0 -48
- package/config/skills/devlyn-ideate/references/templates/roadmap.md +0 -50
- package/config/skills/devlyn-ideate/references/templates/vision.md +0 -44
- package/config/skills/devlyn-queue/SKILL.md +0 -66
- package/config/skills/devlyn-queue/scripts/append.py +0 -94
- package/config/skills/devlyn-resolve/SKILL.md +0 -379
- package/config/skills/devlyn-resolve/references/free-form-mode.md +0 -71
- package/config/skills/devlyn-resolve/references/outer-loop.md +0 -23
- package/config/skills/devlyn-resolve/references/phases/build-gate.md +0 -75
- package/config/skills/devlyn-resolve/references/phases/cleanup.md +0 -33
- package/config/skills/devlyn-resolve/references/phases/implement.md +0 -44
- package/config/skills/devlyn-resolve/references/phases/plan.md +0 -39
- package/config/skills/devlyn-resolve/references/phases/probe-derive.md +0 -261
- package/config/skills/devlyn-resolve/references/phases/surface-close.md +0 -34
- package/config/skills/devlyn-resolve/references/phases/verify.md +0 -310
- package/config/skills/devlyn-resolve/references/state-schema.md +0 -152
- package/config/skills/devlyn-resolve/references/task-completion.md +0 -160
- package/config/templates/prompt-templates.md +0 -71
- package/config/templates/template-feature.spec.md +0 -255
- package/config/templates/template-product-spec.md +0 -680
|
@@ -1,3528 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Merge VERIFY findings and derive a deterministic verdict.
|
|
3
|
-
|
|
4
|
-
VERIFY judges are model-written, but routing on finding severity must be
|
|
5
|
-
mechanical. This script reads the known VERIFY JSONL finding files, writes a
|
|
6
|
-
merged JSONL artifact, computes source-level and overall verdicts, and can
|
|
7
|
-
write the merged verdict back to `.devlyn/pipeline.state.json`.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
from __future__ import annotations
|
|
11
|
-
|
|
12
|
-
import argparse
|
|
13
|
-
import json
|
|
14
|
-
import pathlib
|
|
15
|
-
import re
|
|
16
|
-
import sys
|
|
17
|
-
import tempfile
|
|
18
|
-
import runpy
|
|
19
|
-
from typing import Any
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
JUDGE_OUTPUT_PARSER = runpy.run_path(pathlib.Path(__file__).with_name("judge-output-parser.py"))
|
|
23
|
-
PROCESS_EVIDENCE = runpy.run_path(pathlib.Path(__file__).with_name("process-evidence.py"))
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
SOURCE_FILES = (
|
|
27
|
-
("mechanical", "verify-mechanical.findings.jsonl"),
|
|
28
|
-
("judge", "verify.findings.jsonl"),
|
|
29
|
-
("pair_judge", "verify.pair.findings.jsonl"),
|
|
30
|
-
("pair_judge", "verify.pair-judge.findings.jsonl"),
|
|
31
|
-
)
|
|
32
|
-
REQUIRED_SOURCE_FILES = {
|
|
33
|
-
"mechanical": "verify-mechanical.findings.jsonl",
|
|
34
|
-
"judge": "verify.findings.jsonl",
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
VERDICT_RANK = {
|
|
38
|
-
"PASS": 0,
|
|
39
|
-
"TIMEOUT": 0,
|
|
40
|
-
"PASS_WITH_ISSUES": 1,
|
|
41
|
-
"FAIL": 2,
|
|
42
|
-
"NEEDS_WORK": 2,
|
|
43
|
-
"BLOCKED": 3,
|
|
44
|
-
}
|
|
45
|
-
RANK_VERDICT = {0: "PASS", 1: "PASS_WITH_ISSUES", 2: "NEEDS_WORK", 3: "BLOCKED"}
|
|
46
|
-
ALLOWED_PAIR_SKIP_REASONS = {
|
|
47
|
-
"user_no_pair",
|
|
48
|
-
"mechanical_blocker",
|
|
49
|
-
"primary_judge_blocker",
|
|
50
|
-
"auto_pair_other_engine_unavailable",
|
|
51
|
-
}
|
|
52
|
-
KNOWN_PAIR_TRIGGER_REASONS = {
|
|
53
|
-
"pair.default",
|
|
54
|
-
"mode.verify-only",
|
|
55
|
-
"mode.pair-verify",
|
|
56
|
-
"complexity.high",
|
|
57
|
-
"complexity.large",
|
|
58
|
-
"spec.complexity.high",
|
|
59
|
-
"spec.complexity.large",
|
|
60
|
-
"spec.solo_headroom_hypothesis",
|
|
61
|
-
"risk.high",
|
|
62
|
-
"risk_probes.enabled",
|
|
63
|
-
"risk_probes.present",
|
|
64
|
-
"coverage.failed",
|
|
65
|
-
"mechanical.warning",
|
|
66
|
-
"judge.warning",
|
|
67
|
-
}
|
|
68
|
-
OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")
|
|
69
|
-
BACKTICKED_TEXT_RE = re.compile(r"`[^`\n]+`")
|
|
70
|
-
RESERVED_BACKTICK_TERMS = {"solo-headroom hypothesis", "solo_claude", "miss"}
|
|
71
|
-
COMMAND_PREFIXES = {
|
|
72
|
-
"bash",
|
|
73
|
-
"bun",
|
|
74
|
-
"cargo",
|
|
75
|
-
"git",
|
|
76
|
-
"go",
|
|
77
|
-
"jest",
|
|
78
|
-
"make",
|
|
79
|
-
"node",
|
|
80
|
-
"npm",
|
|
81
|
-
"pnpm",
|
|
82
|
-
"printf",
|
|
83
|
-
"pytest",
|
|
84
|
-
"python",
|
|
85
|
-
"python3",
|
|
86
|
-
"ruff",
|
|
87
|
-
"sh",
|
|
88
|
-
"uv",
|
|
89
|
-
"vitest",
|
|
90
|
-
"yarn",
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
def reject_json_constant(token: str) -> None:
|
|
95
|
-
raise ValueError(f"invalid JSON numeric constant: {token}")
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict:
|
|
99
|
-
result = {}
|
|
100
|
-
for key, value in pairs:
|
|
101
|
-
if key in result:
|
|
102
|
-
raise ValueError(f"duplicate JSON key: {key}")
|
|
103
|
-
result[key] = value
|
|
104
|
-
return result
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def loads_strict_json(text: str) -> Any:
|
|
108
|
-
return json.loads(
|
|
109
|
-
text,
|
|
110
|
-
parse_constant=reject_json_constant,
|
|
111
|
-
object_pairs_hook=reject_duplicate_keys,
|
|
112
|
-
)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def rank(verdict: str | None) -> int:
|
|
116
|
-
return VERDICT_RANK.get(verdict or "PASS", 0)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
def worse(a: str | None, b: str | None) -> str:
|
|
120
|
-
return RANK_VERDICT[max(rank(a), rank(b))]
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def is_known_pair_trigger_reason(reason: str) -> bool:
|
|
124
|
-
return reason in KNOWN_PAIR_TRIGGER_REASONS
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def has_known_pair_trigger_reason(reasons: list[str]) -> bool:
|
|
128
|
-
return any(is_known_pair_trigger_reason(reason) for reason in reasons)
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
def all_known_pair_trigger_reasons(reasons: list[str]) -> bool:
|
|
132
|
-
return all(is_known_pair_trigger_reason(reason) for reason in reasons)
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
def state_uses_default_pair_contract(state: dict[str, Any]) -> bool:
|
|
136
|
-
return state.get("version") == "3.0"
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
def finding_rank(finding: dict[str, Any]) -> int:
|
|
140
|
-
severity = str(finding.get("severity") or "").upper()
|
|
141
|
-
if severity in {"CRITICAL", "HIGH"}:
|
|
142
|
-
return 2
|
|
143
|
-
if severity == "MEDIUM" and finding.get("verdict_binding") is True:
|
|
144
|
-
return 2
|
|
145
|
-
if severity in {"LOW", "MEDIUM"}:
|
|
146
|
-
return 1
|
|
147
|
-
return 0
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
def mechanical_evidence_required(devlyn: pathlib.Path, state: dict[str, Any]) -> bool:
|
|
151
|
-
return PROCESS_EVIDENCE["mechanical_evidence_required"](devlyn.parent.resolve(), state)
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
def mechanical_evidence_carrier(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
155
|
-
state_path = devlyn / "pipeline.state.json"
|
|
156
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
157
|
-
if not isinstance(state, dict):
|
|
158
|
-
raise ValueError("pipeline.state.json must contain a JSON object")
|
|
159
|
-
results_path = devlyn / "spec-verify.results.json"
|
|
160
|
-
if not results_path.is_file():
|
|
161
|
-
if state.get("version") == "3.0" and mechanical_evidence_required(devlyn, state):
|
|
162
|
-
raise ValueError("spec-verify.results.json is required for VERIFY MECHANICAL evidence")
|
|
163
|
-
return None
|
|
164
|
-
results = loads_strict_json(results_path.read_text(encoding="utf-8"))
|
|
165
|
-
if not isinstance(results, dict):
|
|
166
|
-
raise ValueError("spec-verify.results.json must contain a JSON object")
|
|
167
|
-
commands = results.get("commands")
|
|
168
|
-
if not isinstance(commands, list):
|
|
169
|
-
raise ValueError("spec-verify.results.json commands must be an array")
|
|
170
|
-
carrier = results.get("process_evidence")
|
|
171
|
-
if carrier is None:
|
|
172
|
-
if commands:
|
|
173
|
-
raise ValueError("MECHANICAL commands exist without a process-evidence carrier")
|
|
174
|
-
if mechanical_evidence_required(devlyn, state):
|
|
175
|
-
raise ValueError("required VERIFY MECHANICAL evidence has no process-evidence carrier")
|
|
176
|
-
run_id = state.get("run_id")
|
|
177
|
-
phases = state.get("phases")
|
|
178
|
-
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
179
|
-
round_ = verify_phase.get("round") if isinstance(verify_phase, dict) else None
|
|
180
|
-
if isinstance(run_id, str) and isinstance(round_, int) and not isinstance(round_, bool):
|
|
181
|
-
manifest_path = (
|
|
182
|
-
devlyn / "process-evidence" / run_id / "verify"
|
|
183
|
-
/ f"round-{round_}" / "manifest.json"
|
|
184
|
-
)
|
|
185
|
-
if manifest_path.exists():
|
|
186
|
-
raise ValueError("VERIFY manifest exists without a process-evidence carrier")
|
|
187
|
-
return None
|
|
188
|
-
run_id = state.get("run_id")
|
|
189
|
-
phases = state.get("phases")
|
|
190
|
-
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
191
|
-
round_ = verify_phase.get("round") if isinstance(verify_phase, dict) else None
|
|
192
|
-
if not isinstance(run_id, str) or not run_id:
|
|
193
|
-
raise ValueError("state.run_id is required for sealed MECHANICAL evidence")
|
|
194
|
-
if isinstance(round_, bool) or not isinstance(round_, int) or round_ < 0:
|
|
195
|
-
raise ValueError("phases.verify.round is required for sealed MECHANICAL evidence")
|
|
196
|
-
if not isinstance(carrier, dict) or carrier.get("phase") != "verify" or carrier.get("round") != round_:
|
|
197
|
-
raise ValueError("MECHANICAL process-evidence carrier does not match the VERIFY round")
|
|
198
|
-
PROCESS_EVIDENCE["validate_summary_commands"](
|
|
199
|
-
devlyn.parent.resolve(), commands, carrier,
|
|
200
|
-
)
|
|
201
|
-
expected_prefix = f".devlyn/process-evidence/{run_id}/verify/round-{round_}/"
|
|
202
|
-
manifest = carrier.get("manifest")
|
|
203
|
-
if not isinstance(manifest, dict) or not str(manifest.get("path", "")).startswith(expected_prefix):
|
|
204
|
-
raise ValueError("MECHANICAL process-evidence carrier does not match state.run_id")
|
|
205
|
-
|
|
206
|
-
bound = state.get("process_evidence")
|
|
207
|
-
if bound is not None:
|
|
208
|
-
if not isinstance(bound, list):
|
|
209
|
-
raise ValueError("state.process_evidence must be null or an array")
|
|
210
|
-
same_round = [
|
|
211
|
-
item for item in bound
|
|
212
|
-
if isinstance(item, dict)
|
|
213
|
-
and item.get("phase") == "verify"
|
|
214
|
-
and item.get("round") == round_
|
|
215
|
-
]
|
|
216
|
-
if same_round and same_round != [carrier]:
|
|
217
|
-
raise ValueError("state-bound VERIFY evidence disagrees with MECHANICAL results")
|
|
218
|
-
return carrier
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
def mechanical_evidence_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
222
|
-
try:
|
|
223
|
-
mechanical_evidence_carrier(devlyn)
|
|
224
|
-
except (PROCESS_EVIDENCE["EvidenceError"], OSError, UnicodeError, ValueError) as exc:
|
|
225
|
-
return {
|
|
226
|
-
"id": "verify-mechanical-evidence-invalid",
|
|
227
|
-
"rule_id": "invariant.process-evidence-invalid",
|
|
228
|
-
"severity": "CRITICAL",
|
|
229
|
-
"confidence": "high",
|
|
230
|
-
"file": "spec-verify.results.json",
|
|
231
|
-
"line": 1,
|
|
232
|
-
"message": f"Sealed VERIFY MECHANICAL evidence failed rehash before merge: {exc}",
|
|
233
|
-
"criterion_ref": "process-evidence://mechanical",
|
|
234
|
-
"source": "mechanical",
|
|
235
|
-
}
|
|
236
|
-
return None
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
def mechanical_evidence_outcome(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
240
|
-
carrier = mechanical_evidence_carrier(devlyn)
|
|
241
|
-
if carrier is None:
|
|
242
|
-
return None
|
|
243
|
-
return PROCESS_EVIDENCE["bound_carrier_outcome"](devlyn.parent.resolve(), carrier)
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
ROLE_CONFIG = runpy.run_path(pathlib.Path(__file__).with_name("role-config.py"))
|
|
247
|
-
JUDGE_ROLE_EVIDENCE = runpy.run_path(pathlib.Path(__file__).with_name("judge-role-evidence.py"))
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
def resolved_primary_engine(state):
|
|
251
|
-
return ROLE_CONFIG["primary_engine"](state)
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
def primary_timeout_blocker(id_: str, message: str) -> dict[str, Any]:
|
|
255
|
-
return {
|
|
256
|
-
"id": id_,
|
|
257
|
-
"rule_id": "verify.primary.timeout-contract",
|
|
258
|
-
"severity": "CRITICAL",
|
|
259
|
-
"confidence": "high",
|
|
260
|
-
"file": "verify.primary.timeout.json",
|
|
261
|
-
"line": 1,
|
|
262
|
-
"message": message,
|
|
263
|
-
"criterion_ref": "verify.primary.timeout",
|
|
264
|
-
"source": "judge",
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
def read_primary_timeout_marker(
|
|
269
|
-
devlyn: pathlib.Path,
|
|
270
|
-
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
271
|
-
path = devlyn / "verify.primary.timeout.json"
|
|
272
|
-
if not path.is_file():
|
|
273
|
-
return None, None
|
|
274
|
-
try:
|
|
275
|
-
marker = loads_strict_json(path.read_text(encoding="utf-8"))
|
|
276
|
-
except (OSError, UnicodeError, ValueError) as exc:
|
|
277
|
-
return None, primary_timeout_blocker(
|
|
278
|
-
"verify-primary-timeout-marker-malformed",
|
|
279
|
-
f"verify.primary.timeout.json is malformed: {exc}",
|
|
280
|
-
)
|
|
281
|
-
if not isinstance(marker, dict) or set(marker) != {"engine", "budget_seconds"}:
|
|
282
|
-
return None, primary_timeout_blocker(
|
|
283
|
-
"verify-primary-timeout-marker-malformed",
|
|
284
|
-
"verify.primary.timeout.json must contain exactly engine and budget_seconds.",
|
|
285
|
-
)
|
|
286
|
-
engine = marker["engine"]
|
|
287
|
-
budget_seconds = marker["budget_seconds"]
|
|
288
|
-
if not isinstance(engine, str) or not engine:
|
|
289
|
-
return None, primary_timeout_blocker(
|
|
290
|
-
"verify-primary-timeout-marker-malformed",
|
|
291
|
-
"verify.primary.timeout.json engine must be a non-empty string.",
|
|
292
|
-
)
|
|
293
|
-
if (
|
|
294
|
-
not isinstance(budget_seconds, int)
|
|
295
|
-
or isinstance(budget_seconds, bool)
|
|
296
|
-
or budget_seconds != 600
|
|
297
|
-
):
|
|
298
|
-
return None, primary_timeout_blocker(
|
|
299
|
-
"verify-primary-timeout-budget-mismatch",
|
|
300
|
-
"verify.primary.timeout.json budget_seconds must equal 600.",
|
|
301
|
-
)
|
|
302
|
-
try:
|
|
303
|
-
state = loads_strict_json(
|
|
304
|
-
(devlyn / "pipeline.state.json").read_text(encoding="utf-8")
|
|
305
|
-
)
|
|
306
|
-
except (OSError, UnicodeError, ValueError) as exc:
|
|
307
|
-
return None, primary_timeout_blocker(
|
|
308
|
-
"verify-primary-timeout-engine-mismatch",
|
|
309
|
-
f"Cannot authenticate the primary timeout engine from pipeline.state.json: {exc}",
|
|
310
|
-
)
|
|
311
|
-
try:
|
|
312
|
-
expected_engine = resolved_primary_engine(state) if isinstance(state, dict) else None
|
|
313
|
-
except ValueError:
|
|
314
|
-
expected_engine = None
|
|
315
|
-
if (
|
|
316
|
-
not isinstance(expected_engine, str)
|
|
317
|
-
or not expected_engine
|
|
318
|
-
or engine != expected_engine
|
|
319
|
-
):
|
|
320
|
-
return None, primary_timeout_blocker(
|
|
321
|
-
"verify-primary-timeout-engine-mismatch",
|
|
322
|
-
"Primary timeout engine does not match the resolved primary judge.",
|
|
323
|
-
)
|
|
324
|
-
return {"engine": engine, "budget_seconds": budget_seconds}, None
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str, str | None]]:
|
|
328
|
-
findings: list[dict[str, Any]] = []
|
|
329
|
-
source_verdicts: dict[str, str | None] = {source: "PASS" for source, _ in SOURCE_FILES}
|
|
330
|
-
primary_timeout_marker, primary_timeout_violation = read_primary_timeout_marker(devlyn)
|
|
331
|
-
# A verdict must come from a judge that ran: pair_judge stays null until
|
|
332
|
-
# spawn evidence exists (a pair findings file or pair stdout). verify.md's
|
|
333
|
-
# pair contract records null when no second agent is spawned.
|
|
334
|
-
source_verdicts["pair_judge"] = None
|
|
335
|
-
for source, name in SOURCE_FILES:
|
|
336
|
-
path = devlyn / name
|
|
337
|
-
if not path.is_file():
|
|
338
|
-
# A verdict-binding mechanical result contractually skips both
|
|
339
|
-
# judges. Its absent primary carrier is therefore not evidence a
|
|
340
|
-
# dispatched primary judge failed to produce.
|
|
341
|
-
if source == "judge" and rank(source_verdicts.get("mechanical")) >= 2:
|
|
342
|
-
source_verdicts[source] = None
|
|
343
|
-
continue
|
|
344
|
-
if source == "judge" and primary_timeout_marker is not None:
|
|
345
|
-
continue
|
|
346
|
-
if REQUIRED_SOURCE_FILES.get(source) == name:
|
|
347
|
-
findings.append({
|
|
348
|
-
"id": f"verify-merge-required-source-missing-{source}",
|
|
349
|
-
"rule_id": "verify.findings.required-source-missing",
|
|
350
|
-
"severity": "CRITICAL",
|
|
351
|
-
"confidence": "high",
|
|
352
|
-
"file": name,
|
|
353
|
-
"line": 1,
|
|
354
|
-
"message": f"Required VERIFY {source} findings file is missing: {name}",
|
|
355
|
-
"criterion_ref": "verify-merge",
|
|
356
|
-
"source": source,
|
|
357
|
-
})
|
|
358
|
-
source_verdicts[source] = "BLOCKED"
|
|
359
|
-
continue
|
|
360
|
-
if source_verdicts[source] is None:
|
|
361
|
-
source_verdicts[source] = "PASS"
|
|
362
|
-
with path.open(encoding="utf-8") as handle:
|
|
363
|
-
for line_no, line in enumerate(handle, 1):
|
|
364
|
-
raw = line.strip()
|
|
365
|
-
if not raw:
|
|
366
|
-
continue
|
|
367
|
-
try:
|
|
368
|
-
item = loads_strict_json(raw)
|
|
369
|
-
except ValueError as exc:
|
|
370
|
-
blocked = {
|
|
371
|
-
"id": f"verify-merge-invalid-json-{name}-{line_no}",
|
|
372
|
-
"rule_id": "verify.findings.invalid-json",
|
|
373
|
-
"severity": "CRITICAL",
|
|
374
|
-
"confidence": "high",
|
|
375
|
-
"file": name,
|
|
376
|
-
"line": line_no,
|
|
377
|
-
"message": f"Invalid JSONL finding: {exc}",
|
|
378
|
-
"criterion_ref": "verify-merge",
|
|
379
|
-
"source": source,
|
|
380
|
-
}
|
|
381
|
-
findings.append(blocked)
|
|
382
|
-
source_verdicts[source] = "BLOCKED"
|
|
383
|
-
continue
|
|
384
|
-
if not isinstance(item, dict):
|
|
385
|
-
continue
|
|
386
|
-
item = dict(item)
|
|
387
|
-
item.setdefault("source", source)
|
|
388
|
-
findings.append(item)
|
|
389
|
-
source_verdicts[source] = worse(
|
|
390
|
-
source_verdicts[source], RANK_VERDICT[finding_rank(item)]
|
|
391
|
-
)
|
|
392
|
-
evidence_violation = mechanical_evidence_violation(devlyn)
|
|
393
|
-
if evidence_violation is not None:
|
|
394
|
-
findings.append(evidence_violation)
|
|
395
|
-
source_verdicts["mechanical"] = "BLOCKED"
|
|
396
|
-
else:
|
|
397
|
-
outcome = mechanical_evidence_outcome(devlyn)
|
|
398
|
-
if outcome is not None and outcome["verdict"] != "PASS":
|
|
399
|
-
blocked = outcome["verdict"] == "BLOCKED"
|
|
400
|
-
ids = (
|
|
401
|
-
[item["id"] for item in outcome["capability_denials"]]
|
|
402
|
-
if blocked else outcome["failed_ids"]
|
|
403
|
-
)
|
|
404
|
-
findings.append({
|
|
405
|
-
"id": (
|
|
406
|
-
"verify-mechanical-capability-denied"
|
|
407
|
-
if blocked else "verify-mechanical-expectation-mismatch"
|
|
408
|
-
),
|
|
409
|
-
"rule_id": (
|
|
410
|
-
"invariant.build-env-underprovisioned"
|
|
411
|
-
if blocked else "invariant.mechanical-expectation-mismatch"
|
|
412
|
-
),
|
|
413
|
-
"severity": "CRITICAL" if blocked else "HIGH",
|
|
414
|
-
"confidence": "high",
|
|
415
|
-
"file": "spec-verify.results.json",
|
|
416
|
-
"line": 1,
|
|
417
|
-
"message": (
|
|
418
|
-
"Sealed VERIFY MECHANICAL evidence records a capability denial: "
|
|
419
|
-
if blocked else
|
|
420
|
-
"Sealed VERIFY MECHANICAL evidence records failed expectations: "
|
|
421
|
-
) + ",".join(ids),
|
|
422
|
-
"criterion_ref": "process-evidence://mechanical",
|
|
423
|
-
"source": "mechanical",
|
|
424
|
-
"verdict_binding": True,
|
|
425
|
-
})
|
|
426
|
-
source_verdicts["mechanical"] = outcome["verdict"]
|
|
427
|
-
if primary_timeout_violation is not None:
|
|
428
|
-
findings.append(primary_timeout_violation)
|
|
429
|
-
source_verdicts["judge"] = "BLOCKED"
|
|
430
|
-
elif primary_timeout_marker is not None:
|
|
431
|
-
findings.append(primary_timeout_blocker(
|
|
432
|
-
"verify-primary-timeout",
|
|
433
|
-
"Primary JUDGE exceeded its authenticated 600-second wall budget.",
|
|
434
|
-
))
|
|
435
|
-
source_verdicts["judge"] = "BLOCKED"
|
|
436
|
-
findings.extend(detect_pair_stdout_contract_violations(devlyn, source_verdicts))
|
|
437
|
-
pair_summary_path = devlyn / "pair-judge.summary.json"
|
|
438
|
-
pair_carrier_exists = any(
|
|
439
|
-
(devlyn / name).is_file()
|
|
440
|
-
for name in ("verify.pair.findings.jsonl", "verify.pair-judge.findings.jsonl")
|
|
441
|
-
)
|
|
442
|
-
if pair_carrier_exists and pair_summary_path.is_file():
|
|
443
|
-
try:
|
|
444
|
-
pair_summary = loads_strict_json(pair_summary_path.read_text(encoding="utf-8"))
|
|
445
|
-
except ValueError as exc:
|
|
446
|
-
summary_error = f"pair-judge.summary.json is malformed JSON: {exc}"
|
|
447
|
-
else:
|
|
448
|
-
pair_verdict = pair_summary.get("verdict") if isinstance(pair_summary, dict) else None
|
|
449
|
-
summary_error = None
|
|
450
|
-
if not isinstance(pair_summary, dict):
|
|
451
|
-
summary_error = "pair-judge.summary.json must be a JSON object."
|
|
452
|
-
elif not isinstance(pair_verdict, str) or pair_verdict not in {
|
|
453
|
-
"PASS", "PASS_WITH_ISSUES", "FAIL", "NEEDS_WORK", "BLOCKED"
|
|
454
|
-
}:
|
|
455
|
-
summary_error = (
|
|
456
|
-
"pair-judge.summary.json verdict must be PASS, PASS_WITH_ISSUES, "
|
|
457
|
-
"FAIL, NEEDS_WORK, or BLOCKED."
|
|
458
|
-
)
|
|
459
|
-
if summary_error is not None:
|
|
460
|
-
findings.append(
|
|
461
|
-
pair_blocker(
|
|
462
|
-
"verify-pair-summary-invalid",
|
|
463
|
-
summary_error,
|
|
464
|
-
pair_summary_path.name,
|
|
465
|
-
)
|
|
466
|
-
)
|
|
467
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
468
|
-
elif (
|
|
469
|
-
source_verdicts["pair_judge"] != "TIMEOUT"
|
|
470
|
-
or rank(pair_verdict) > rank("TIMEOUT")
|
|
471
|
-
):
|
|
472
|
-
source_verdicts["pair_judge"] = worse(
|
|
473
|
-
source_verdicts["pair_judge"], pair_verdict
|
|
474
|
-
)
|
|
475
|
-
if (devlyn / "pipeline.state.json").is_file():
|
|
476
|
-
try:
|
|
477
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
478
|
-
required = JUDGE_ROLE_EVIDENCE["required_roles"](state)
|
|
479
|
-
if rank(source_verdicts.get("mechanical")) >= 2:
|
|
480
|
-
required = []
|
|
481
|
-
for role in required:
|
|
482
|
-
source = "judge" if role == "primary_judge" else "pair_judge"
|
|
483
|
-
if source_verdicts.get(source) in {"BLOCKED", "TIMEOUT"}:
|
|
484
|
-
continue
|
|
485
|
-
try:
|
|
486
|
-
JUDGE_ROLE_EVIDENCE["authenticate"](devlyn, state, role)
|
|
487
|
-
except (ValueError, OSError, TypeError, KeyError) as exc:
|
|
488
|
-
source_verdicts[source] = "BLOCKED"
|
|
489
|
-
findings.append({**primary_timeout_blocker("verify-role-evidence-invalid", str(exc)),
|
|
490
|
-
"source": source, "rule_id": "verify.role-evidence"})
|
|
491
|
-
except (ValueError, OSError, TypeError, KeyError) as exc:
|
|
492
|
-
source_verdicts["judge"] = "BLOCKED"
|
|
493
|
-
findings.append({**primary_timeout_blocker("verify-role-resolution-invalid", str(exc)),
|
|
494
|
-
"rule_id": "verify.role-resolution"})
|
|
495
|
-
return findings, source_verdicts
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
def has_pair_findings(devlyn: pathlib.Path) -> bool:
|
|
499
|
-
for name in ("verify.pair.findings.jsonl", "verify.pair-judge.findings.jsonl"):
|
|
500
|
-
path = devlyn / name
|
|
501
|
-
if path.is_file() and path.read_text(encoding="utf-8").strip():
|
|
502
|
-
return True
|
|
503
|
-
return False
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
def canonical_other_judge_stdout(devlyn: pathlib.Path, primary_engine: str) -> list[pathlib.Path]:
|
|
507
|
-
adapters = pathlib.Path(__file__).with_name("adapters")
|
|
508
|
-
engine_names = {
|
|
509
|
-
path.stem for path in adapters.glob("*.md")
|
|
510
|
-
if path.stem != "README"
|
|
511
|
-
}
|
|
512
|
-
return sorted(
|
|
513
|
-
devlyn / f"{engine}-judge.stdout"
|
|
514
|
-
for engine in engine_names
|
|
515
|
-
if engine != primary_engine and (devlyn / f"{engine}-judge.stdout").is_file()
|
|
516
|
-
)
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
def pair_capture_exists(devlyn: pathlib.Path, stdout_paths: list[pathlib.Path]) -> bool:
|
|
520
|
-
return bool(stdout_paths) or has_pair_findings(devlyn) or any(
|
|
521
|
-
(devlyn / name).is_file()
|
|
522
|
-
for name in (
|
|
523
|
-
"verify.pair.timeout.json",
|
|
524
|
-
"pair-judge.summary.json",
|
|
525
|
-
)
|
|
526
|
-
)
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
def pair_trigger_status(devlyn: pathlib.Path) -> tuple[bool, dict[str, Any] | None]:
|
|
530
|
-
state_path = devlyn / "pipeline.state.json"
|
|
531
|
-
if not state_path.is_file():
|
|
532
|
-
return False, None
|
|
533
|
-
try:
|
|
534
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
535
|
-
except ValueError:
|
|
536
|
-
return False, {
|
|
537
|
-
"id": "verify-pair-trigger-state-malformed",
|
|
538
|
-
"message": "pipeline.state.json is malformed; cannot verify pair_trigger contract.",
|
|
539
|
-
"file": "pipeline.state.json",
|
|
540
|
-
}
|
|
541
|
-
phases = state.get("phases") if isinstance(state, dict) else {}
|
|
542
|
-
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
543
|
-
trigger = None
|
|
544
|
-
if isinstance(verify_phase, dict):
|
|
545
|
-
trigger = verify_phase.get("pair_trigger")
|
|
546
|
-
if trigger is None and isinstance(state, dict):
|
|
547
|
-
verify_state = state.get("verify")
|
|
548
|
-
if isinstance(verify_state, dict):
|
|
549
|
-
trigger = verify_state.get("pair_trigger")
|
|
550
|
-
if trigger is None:
|
|
551
|
-
return False, None
|
|
552
|
-
if not isinstance(trigger, dict):
|
|
553
|
-
return False, {
|
|
554
|
-
"id": "verify-pair-trigger-malformed",
|
|
555
|
-
"message": "pair_trigger must be an object.",
|
|
556
|
-
"file": "pipeline.state.json",
|
|
557
|
-
}
|
|
558
|
-
eligible = trigger.get("eligible")
|
|
559
|
-
if not isinstance(eligible, bool):
|
|
560
|
-
return False, {
|
|
561
|
-
"id": "verify-pair-trigger-eligible-malformed",
|
|
562
|
-
"message": "pair_trigger.eligible must be a boolean.",
|
|
563
|
-
"file": "pipeline.state.json",
|
|
564
|
-
}
|
|
565
|
-
reasons = trigger.get("reasons")
|
|
566
|
-
if not isinstance(reasons, list) or not all(isinstance(item, str) for item in reasons):
|
|
567
|
-
return False, {
|
|
568
|
-
"id": "verify-pair-trigger-reasons-malformed",
|
|
569
|
-
"message": "pair_trigger.reasons must be a list of strings.",
|
|
570
|
-
"file": "pipeline.state.json",
|
|
571
|
-
}
|
|
572
|
-
skipped_reason = trigger.get("skipped_reason")
|
|
573
|
-
if skipped_reason is not None and not isinstance(skipped_reason, str):
|
|
574
|
-
return False, {
|
|
575
|
-
"id": "verify-pair-trigger-skipped-reason-malformed",
|
|
576
|
-
"message": "pair_trigger.skipped_reason must be a string or null.",
|
|
577
|
-
"file": "pipeline.state.json",
|
|
578
|
-
}
|
|
579
|
-
if eligible is True and not reasons:
|
|
580
|
-
return False, {
|
|
581
|
-
"id": "verify-pair-trigger-reasons-empty",
|
|
582
|
-
"message": "pair_trigger.eligible cannot be true with an empty reasons list.",
|
|
583
|
-
"file": "pipeline.state.json",
|
|
584
|
-
}
|
|
585
|
-
if eligible is True and not has_known_pair_trigger_reason(reasons):
|
|
586
|
-
return False, {
|
|
587
|
-
"id": "verify-pair-trigger-reasons-unknown",
|
|
588
|
-
"message": "pair_trigger.reasons must include a known pair-trigger reason.",
|
|
589
|
-
"file": "pipeline.state.json",
|
|
590
|
-
}
|
|
591
|
-
if eligible is True and not all_known_pair_trigger_reasons(reasons):
|
|
592
|
-
return False, {
|
|
593
|
-
"id": "verify-pair-trigger-reasons-unknown",
|
|
594
|
-
"message": "pair_trigger.reasons must only include known pair-trigger reasons.",
|
|
595
|
-
"file": "pipeline.state.json",
|
|
596
|
-
}
|
|
597
|
-
if eligible is True and skipped_reason is not None:
|
|
598
|
-
return False, {
|
|
599
|
-
"id": "verify-pair-trigger-skip-contradiction",
|
|
600
|
-
"message": "pair_trigger.eligible cannot be true while skipped_reason is set.",
|
|
601
|
-
"file": "pipeline.state.json",
|
|
602
|
-
}
|
|
603
|
-
if eligible is False and reasons:
|
|
604
|
-
return False, {
|
|
605
|
-
"id": "verify-pair-trigger-ineligible-reasons",
|
|
606
|
-
"message": "pair_trigger.reasons must be empty when pair_trigger.eligible is false.",
|
|
607
|
-
"file": "pipeline.state.json",
|
|
608
|
-
}
|
|
609
|
-
return eligible is True and len(reasons) > 0, None
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
def pair_trigger_required(devlyn: pathlib.Path) -> bool:
|
|
613
|
-
required, _malformed = pair_trigger_status(devlyn)
|
|
614
|
-
return required
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
def pair_trigger_present(devlyn: pathlib.Path) -> bool:
|
|
618
|
-
state_path = devlyn / "pipeline.state.json"
|
|
619
|
-
if not state_path.is_file():
|
|
620
|
-
return False
|
|
621
|
-
try:
|
|
622
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
623
|
-
except ValueError:
|
|
624
|
-
return False
|
|
625
|
-
phases = state.get("phases") if isinstance(state, dict) else {}
|
|
626
|
-
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
627
|
-
if isinstance(verify_phase, dict) and "pair_trigger" in verify_phase:
|
|
628
|
-
return True
|
|
629
|
-
if isinstance(state, dict):
|
|
630
|
-
verify_state = state.get("verify")
|
|
631
|
-
if isinstance(verify_state, dict) and "pair_trigger" in verify_state:
|
|
632
|
-
return True
|
|
633
|
-
return False
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
def pair_flag_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
637
|
-
state_path = devlyn / "pipeline.state.json"
|
|
638
|
-
if not state_path.is_file():
|
|
639
|
-
return None
|
|
640
|
-
try:
|
|
641
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
642
|
-
except ValueError:
|
|
643
|
-
return None
|
|
644
|
-
if not isinstance(state, dict) or state.get("pair_verify") is not True:
|
|
645
|
-
return None
|
|
646
|
-
risk_profile = state.get("risk_profile")
|
|
647
|
-
if isinstance(risk_profile, dict) and risk_profile.get("pair_default_enabled") is False:
|
|
648
|
-
return {
|
|
649
|
-
"id": "verify-pair-trigger-conflicting-pair-flags",
|
|
650
|
-
"message": "--pair-verify and --no-pair are mutually exclusive.",
|
|
651
|
-
"file": "pipeline.state.json",
|
|
652
|
-
}
|
|
653
|
-
return None
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
def risk_profile_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
657
|
-
state_path = devlyn / "pipeline.state.json"
|
|
658
|
-
if not state_path.is_file():
|
|
659
|
-
return None
|
|
660
|
-
try:
|
|
661
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
662
|
-
except ValueError:
|
|
663
|
-
return None
|
|
664
|
-
if not isinstance(state, dict) or "risk_profile" not in state:
|
|
665
|
-
return None
|
|
666
|
-
risk_profile = state.get("risk_profile")
|
|
667
|
-
if not isinstance(risk_profile, dict):
|
|
668
|
-
return {
|
|
669
|
-
"id": "verify-risk-profile-malformed",
|
|
670
|
-
"message": "risk_profile must be an object.",
|
|
671
|
-
"file": "pipeline.state.json",
|
|
672
|
-
}
|
|
673
|
-
for field in ("high_risk", "risk_probes_enabled", "pair_default_enabled"):
|
|
674
|
-
if field in risk_profile and not isinstance(risk_profile.get(field), bool):
|
|
675
|
-
return {
|
|
676
|
-
"id": "verify-risk-profile-malformed",
|
|
677
|
-
"message": f"risk_profile.{field} must be a boolean.",
|
|
678
|
-
"file": "pipeline.state.json",
|
|
679
|
-
}
|
|
680
|
-
reasons = risk_profile.get("reasons")
|
|
681
|
-
if "reasons" in risk_profile and (
|
|
682
|
-
not isinstance(reasons, list) or not all(isinstance(item, str) for item in reasons)
|
|
683
|
-
):
|
|
684
|
-
return {
|
|
685
|
-
"id": "verify-risk-profile-malformed",
|
|
686
|
-
"message": "risk_profile.reasons must be a list of strings.",
|
|
687
|
-
"file": "pipeline.state.json",
|
|
688
|
-
}
|
|
689
|
-
return None
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
def verify_state_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
693
|
-
state_path = devlyn / "pipeline.state.json"
|
|
694
|
-
if not state_path.is_file():
|
|
695
|
-
return {
|
|
696
|
-
"id": "verify-state-missing",
|
|
697
|
-
"rule_id": "verify.state.missing",
|
|
698
|
-
"message": "pipeline.state.json is required before VERIFY merge.",
|
|
699
|
-
"file": "pipeline.state.json",
|
|
700
|
-
}
|
|
701
|
-
try:
|
|
702
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
703
|
-
except ValueError:
|
|
704
|
-
return {
|
|
705
|
-
"id": "verify-pair-trigger-state-malformed",
|
|
706
|
-
"rule_id": "verify.pair.emission-contract",
|
|
707
|
-
"message": "pipeline.state.json is malformed; cannot verify pair_trigger contract.",
|
|
708
|
-
"file": "pipeline.state.json",
|
|
709
|
-
}
|
|
710
|
-
if not isinstance(state, dict):
|
|
711
|
-
return {
|
|
712
|
-
"id": "verify-state-malformed",
|
|
713
|
-
"rule_id": "verify.state.malformed",
|
|
714
|
-
"message": "pipeline.state.json must be a JSON object before VERIFY merge.",
|
|
715
|
-
"file": "pipeline.state.json",
|
|
716
|
-
}
|
|
717
|
-
try:
|
|
718
|
-
engine = resolved_primary_engine(state)
|
|
719
|
-
except ValueError:
|
|
720
|
-
engine = None
|
|
721
|
-
if not isinstance(engine, str) or not engine.strip():
|
|
722
|
-
rule = "verify.state.engine-malformed"
|
|
723
|
-
return {
|
|
724
|
-
"id": rule,
|
|
725
|
-
"rule_id": rule,
|
|
726
|
-
"message": "pipeline.state.json requires a valid primary engine before VERIFY merge.",
|
|
727
|
-
"file": "pipeline.state.json",
|
|
728
|
-
}
|
|
729
|
-
if not state_uses_default_pair_contract(state):
|
|
730
|
-
return None
|
|
731
|
-
source = state.get("source")
|
|
732
|
-
if not isinstance(source, dict) or source.get("type") != "generated":
|
|
733
|
-
return None
|
|
734
|
-
goal_path = source.get("goal_path")
|
|
735
|
-
goal_sha256 = source.get("goal_sha256")
|
|
736
|
-
if (
|
|
737
|
-
not isinstance(goal_path, str)
|
|
738
|
-
or not goal_path
|
|
739
|
-
or not isinstance(goal_sha256, str)
|
|
740
|
-
or re.fullmatch(r"[0-9a-f]{64}", goal_sha256) is None
|
|
741
|
-
):
|
|
742
|
-
rule = "verify.state.goal-persistence-missing"
|
|
743
|
-
return {
|
|
744
|
-
"id": rule,
|
|
745
|
-
"rule_id": rule,
|
|
746
|
-
"message": (
|
|
747
|
-
"schema-v3 generated runs require source.goal_path as a non-empty string "
|
|
748
|
-
"and source.goal_sha256 as 64 lowercase hexadecimal characters."
|
|
749
|
-
),
|
|
750
|
-
"file": "pipeline.state.json",
|
|
751
|
-
}
|
|
752
|
-
if state.get("complexity") in {"trivial", "medium"}:
|
|
753
|
-
phases = state.get("phases")
|
|
754
|
-
surface_close = phases.get("surface_close") if isinstance(phases, dict) else None
|
|
755
|
-
skipped = (
|
|
756
|
-
isinstance(surface_close, dict)
|
|
757
|
-
and surface_close.get("verdict") is None
|
|
758
|
-
and surface_close.get("started_at") is None
|
|
759
|
-
and surface_close.get("skipped_reason") == "auto_surface_close_claude_unavailable"
|
|
760
|
-
)
|
|
761
|
-
if (
|
|
762
|
-
not isinstance(surface_close, dict)
|
|
763
|
-
or (surface_close.get("verdict") is None and not skipped)
|
|
764
|
-
):
|
|
765
|
-
rule = "verify.state.surface-close-skipped"
|
|
766
|
-
return {
|
|
767
|
-
"id": rule,
|
|
768
|
-
"rule_id": rule,
|
|
769
|
-
"message": (
|
|
770
|
-
"schema-v3 generated trivial/medium runs require completed SURFACE_CLOSE "
|
|
771
|
-
"or its canonical automatic Claude-unavailable skip before VERIFY merge."
|
|
772
|
-
),
|
|
773
|
-
"file": "pipeline.state.json",
|
|
774
|
-
}
|
|
775
|
-
return None
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
def source_spec_text(state: dict[str, Any]) -> str | None:
|
|
779
|
-
source = state.get("source") if isinstance(state.get("source"), dict) else {}
|
|
780
|
-
for key in ("spec_path", "criteria_path"):
|
|
781
|
-
raw_path = source.get(key)
|
|
782
|
-
if not isinstance(raw_path, str) or not raw_path:
|
|
783
|
-
continue
|
|
784
|
-
path = pathlib.Path(raw_path)
|
|
785
|
-
if not path.is_absolute():
|
|
786
|
-
path = pathlib.Path.cwd() / path
|
|
787
|
-
try:
|
|
788
|
-
return path.read_text(encoding="utf-8")
|
|
789
|
-
except OSError:
|
|
790
|
-
continue
|
|
791
|
-
return None
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
def spec_frontmatter_complexity(state: dict[str, Any]) -> str | None:
|
|
795
|
-
text = source_spec_text(state)
|
|
796
|
-
if text is None:
|
|
797
|
-
return None
|
|
798
|
-
if not text.startswith("---"):
|
|
799
|
-
return None
|
|
800
|
-
end = text.find("\n---", 3)
|
|
801
|
-
if end == -1:
|
|
802
|
-
return None
|
|
803
|
-
for line in text[3:end].splitlines():
|
|
804
|
-
match = re.match(r"\s*complexity\s*:\s*[\"']?([A-Za-z_-]+)", line)
|
|
805
|
-
if match:
|
|
806
|
-
return match.group(1).lower()
|
|
807
|
-
return None
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
def spec_has_solo_headroom_hypothesis(state: dict[str, Any]) -> bool:
|
|
811
|
-
text = source_spec_text(state)
|
|
812
|
-
if text is None:
|
|
813
|
-
return False
|
|
814
|
-
lower = text.lower()
|
|
815
|
-
return (
|
|
816
|
-
"solo-headroom hypothesis" in lower
|
|
817
|
-
and "solo_claude" in lower
|
|
818
|
-
and "miss" in lower
|
|
819
|
-
and has_backticked_observable_command(text)
|
|
820
|
-
)
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
def has_backticked_observable_command(text: str) -> bool:
|
|
824
|
-
for line in text.splitlines():
|
|
825
|
-
lower = line.lower()
|
|
826
|
-
if "miss" not in lower or not any(marker in lower for marker in OBSERVABLE_COMMAND_MARKERS):
|
|
827
|
-
continue
|
|
828
|
-
if any(is_command_like_backtick(match.group(0).strip("`")) for match in BACKTICKED_TEXT_RE.finditer(line)):
|
|
829
|
-
return True
|
|
830
|
-
return False
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
def is_command_like_backtick(value: str) -> bool:
|
|
834
|
-
stripped = value.strip()
|
|
835
|
-
lower = stripped.lower()
|
|
836
|
-
if not stripped or lower in RESERVED_BACKTICK_TERMS:
|
|
837
|
-
return False
|
|
838
|
-
first = lower.split(maxsplit=1)[0]
|
|
839
|
-
return (
|
|
840
|
-
first in COMMAND_PREFIXES
|
|
841
|
-
or any(marker in stripped for marker in ("/", "$", "=", "|", "&&", ";"))
|
|
842
|
-
or stripped.endswith((".js", ".py", ".sh"))
|
|
843
|
-
)
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
def state_pair_trigger_reasons(
|
|
847
|
-
devlyn: pathlib.Path,
|
|
848
|
-
source_verdicts: dict[str, str | None],
|
|
849
|
-
) -> list[str]:
|
|
850
|
-
state_path = devlyn / "pipeline.state.json"
|
|
851
|
-
if not state_path.is_file():
|
|
852
|
-
return []
|
|
853
|
-
try:
|
|
854
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
855
|
-
except ValueError:
|
|
856
|
-
return []
|
|
857
|
-
if not isinstance(state, dict):
|
|
858
|
-
return []
|
|
859
|
-
phases = state.get("phases") if isinstance(state.get("phases"), dict) else {}
|
|
860
|
-
verify_phase = phases.get("verify") if isinstance(phases, dict) else {}
|
|
861
|
-
verify_state = state.get("verify") if isinstance(state.get("verify"), dict) else {}
|
|
862
|
-
risk_profile = state.get("risk_profile") if isinstance(state.get("risk_profile"), dict) else {}
|
|
863
|
-
reasons: list[str] = []
|
|
864
|
-
if (
|
|
865
|
-
state_uses_default_pair_contract(state)
|
|
866
|
-
and risk_profile.get("pair_default_enabled") is not False
|
|
867
|
-
):
|
|
868
|
-
reasons.append("pair.default")
|
|
869
|
-
if state.get("mode") == "verify-only":
|
|
870
|
-
reasons.append("mode.verify-only")
|
|
871
|
-
if state.get("pair_verify") is True:
|
|
872
|
-
reasons.append("mode.pair-verify")
|
|
873
|
-
if state.get("complexity") in {"high", "large"}:
|
|
874
|
-
reasons.append(f"complexity.{state.get('complexity')}")
|
|
875
|
-
spec_complexity = spec_frontmatter_complexity(state)
|
|
876
|
-
if spec_complexity in {"high", "large"}:
|
|
877
|
-
reasons.append(f"spec.complexity.{spec_complexity}")
|
|
878
|
-
if spec_has_solo_headroom_hypothesis(state):
|
|
879
|
-
reasons.append("spec.solo_headroom_hypothesis")
|
|
880
|
-
if risk_profile.get("high_risk") is True:
|
|
881
|
-
reasons.append("risk.high")
|
|
882
|
-
if risk_profile.get("risk_probes_enabled") is True:
|
|
883
|
-
reasons.append("risk_probes.enabled")
|
|
884
|
-
if (devlyn / "risk-probes.jsonl").is_file():
|
|
885
|
-
reasons.append("risk_probes.present")
|
|
886
|
-
coverage_failed = False
|
|
887
|
-
if isinstance(verify_state, dict) and verify_state.get("coverage_failed") is True:
|
|
888
|
-
coverage_failed = True
|
|
889
|
-
if isinstance(verify_phase, dict) and verify_phase.get("coverage_failed") is True:
|
|
890
|
-
coverage_failed = True
|
|
891
|
-
if coverage_failed:
|
|
892
|
-
reasons.append("coverage.failed")
|
|
893
|
-
if rank(source_verdicts.get("mechanical")) == 1:
|
|
894
|
-
reasons.append("mechanical.warning")
|
|
895
|
-
if rank(source_verdicts.get("judge")) == 1:
|
|
896
|
-
reasons.append("judge.warning")
|
|
897
|
-
return reasons
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
def outcome_independent_reasons(devlyn: pathlib.Path) -> list[str]:
|
|
901
|
-
return [
|
|
902
|
-
reason
|
|
903
|
-
for reason in state_pair_trigger_reasons(devlyn, {})
|
|
904
|
-
if reason not in ("coverage.failed", "mechanical.warning", "judge.warning")
|
|
905
|
-
]
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
def pair_trigger_missing_contract_violation(
|
|
909
|
-
devlyn: pathlib.Path,
|
|
910
|
-
source_verdicts: dict[str, str | None],
|
|
911
|
-
) -> dict[str, Any] | None:
|
|
912
|
-
if rank(source_verdicts.get("mechanical")) >= 2:
|
|
913
|
-
return None
|
|
914
|
-
reasons = state_pair_trigger_reasons(devlyn, source_verdicts)
|
|
915
|
-
if not reasons:
|
|
916
|
-
return None
|
|
917
|
-
return {
|
|
918
|
-
"id": "verify-pair-trigger-required-missing",
|
|
919
|
-
"message": (
|
|
920
|
-
"pair_trigger is missing even though VERIFY state requires a pair decision: "
|
|
921
|
-
+ ", ".join(reasons)
|
|
922
|
-
),
|
|
923
|
-
"file": "pipeline.state.json",
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
def pair_trigger_skip_contract_violation(
|
|
928
|
-
devlyn: pathlib.Path,
|
|
929
|
-
source_verdicts: dict[str, str | None],
|
|
930
|
-
) -> dict[str, Any] | None:
|
|
931
|
-
state_path = devlyn / "pipeline.state.json"
|
|
932
|
-
if not state_path.is_file():
|
|
933
|
-
return None
|
|
934
|
-
try:
|
|
935
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
936
|
-
except ValueError:
|
|
937
|
-
return None
|
|
938
|
-
phases = state.get("phases") if isinstance(state, dict) else {}
|
|
939
|
-
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
940
|
-
trigger = None
|
|
941
|
-
if isinstance(verify_phase, dict):
|
|
942
|
-
trigger = verify_phase.get("pair_trigger")
|
|
943
|
-
if trigger is None and isinstance(state, dict):
|
|
944
|
-
verify_state = state.get("verify")
|
|
945
|
-
if isinstance(verify_state, dict):
|
|
946
|
-
trigger = verify_state.get("pair_trigger")
|
|
947
|
-
if not isinstance(trigger, dict):
|
|
948
|
-
return None
|
|
949
|
-
skipped_reason = trigger.get("skipped_reason")
|
|
950
|
-
if trigger.get("eligible") is False and skipped_reason is None:
|
|
951
|
-
natural_reasons = state_pair_trigger_reasons(devlyn, source_verdicts)
|
|
952
|
-
if natural_reasons:
|
|
953
|
-
return {
|
|
954
|
-
"id": "verify-pair-trigger-ineligible-unjustified",
|
|
955
|
-
"message": (
|
|
956
|
-
"pair_trigger is ineligible without a skip reason even though "
|
|
957
|
-
"VERIFY state requires a pair decision: "
|
|
958
|
-
+ ", ".join(natural_reasons)
|
|
959
|
-
),
|
|
960
|
-
"file": "pipeline.state.json",
|
|
961
|
-
}
|
|
962
|
-
if skipped_reason is None:
|
|
963
|
-
return None
|
|
964
|
-
if skipped_reason not in ALLOWED_PAIR_SKIP_REASONS:
|
|
965
|
-
return {
|
|
966
|
-
"id": "verify-pair-trigger-skipped-reason-unsupported",
|
|
967
|
-
"message": (
|
|
968
|
-
"pair_trigger.skipped_reason must be user_no_pair, "
|
|
969
|
-
"mechanical_blocker, primary_judge_blocker, "
|
|
970
|
-
"auto_pair_other_engine_unavailable, or null."
|
|
971
|
-
),
|
|
972
|
-
"file": "pipeline.state.json",
|
|
973
|
-
}
|
|
974
|
-
if skipped_reason == "auto_pair_other_engine_unavailable":
|
|
975
|
-
# alpha+ capability-gating: an AUTOMATIC pair trigger may skip to solo
|
|
976
|
-
# VERIFY when the OTHER engine is unavailable (single-LLM users stay
|
|
977
|
-
# first-class). An EXPLICIT --pair-verify route is a promise and must
|
|
978
|
-
# BLOCK on an unavailable engine, never skip — enforce that here so the
|
|
979
|
-
# auto-skip cannot launder an explicit request.
|
|
980
|
-
pair_verify = state.get("pair_verify") if isinstance(state, dict) else None
|
|
981
|
-
if pair_verify is True:
|
|
982
|
-
return {
|
|
983
|
-
"id": "verify-pair-trigger-auto-skip-explicit-conflict",
|
|
984
|
-
"message": (
|
|
985
|
-
"pair_trigger skipped_reason auto_pair_other_engine_unavailable is "
|
|
986
|
-
"only valid for an automatic trigger; an explicit --pair-verify run "
|
|
987
|
-
"must BLOCK on an unavailable OTHER engine, not skip."
|
|
988
|
-
),
|
|
989
|
-
"file": "pipeline.state.json",
|
|
990
|
-
}
|
|
991
|
-
if skipped_reason == "user_no_pair":
|
|
992
|
-
risk_profile = state.get("risk_profile") if isinstance(state, dict) else {}
|
|
993
|
-
if not isinstance(risk_profile, dict) or risk_profile.get("pair_default_enabled") is not False:
|
|
994
|
-
return {
|
|
995
|
-
"id": "verify-pair-trigger-user-no-pair-unsupported",
|
|
996
|
-
"message": (
|
|
997
|
-
"pair_trigger skipped_reason user_no_pair requires "
|
|
998
|
-
"risk_profile.pair_default_enabled false from an explicit --no-pair opt-out."
|
|
999
|
-
),
|
|
1000
|
-
"file": "pipeline.state.json",
|
|
1001
|
-
}
|
|
1002
|
-
if skipped_reason == "mechanical_blocker" and rank(source_verdicts.get("mechanical")) < 2:
|
|
1003
|
-
return {
|
|
1004
|
-
"id": "verify-pair-trigger-mechanical-blocker-unsupported",
|
|
1005
|
-
"message": (
|
|
1006
|
-
"pair_trigger skipped_reason mechanical_blocker requires a "
|
|
1007
|
-
"verdict-binding MECHANICAL finding."
|
|
1008
|
-
),
|
|
1009
|
-
"file": "pipeline.state.json",
|
|
1010
|
-
}
|
|
1011
|
-
if skipped_reason == "primary_judge_blocker":
|
|
1012
|
-
if state_uses_default_pair_contract(state):
|
|
1013
|
-
return {
|
|
1014
|
-
"id": "verify-pair-trigger-primary-judge-blocker-retired",
|
|
1015
|
-
"message": (
|
|
1016
|
-
"pair_trigger skipped_reason primary_judge_blocker is archived-v2.0 "
|
|
1017
|
-
"state only; schema-v3 runs must dispatch the pair-JUDGE."
|
|
1018
|
-
),
|
|
1019
|
-
"file": "pipeline.state.json",
|
|
1020
|
-
}
|
|
1021
|
-
if rank(source_verdicts.get("judge")) < 2:
|
|
1022
|
-
return {
|
|
1023
|
-
"id": "verify-pair-trigger-primary-judge-blocker-unsupported",
|
|
1024
|
-
"message": (
|
|
1025
|
-
"pair_trigger skipped_reason primary_judge_blocker requires a "
|
|
1026
|
-
"verdict-binding primary JUDGE finding."
|
|
1027
|
-
),
|
|
1028
|
-
"file": "pipeline.state.json",
|
|
1029
|
-
}
|
|
1030
|
-
preknown_reasons = outcome_independent_reasons(devlyn)
|
|
1031
|
-
if preknown_reasons:
|
|
1032
|
-
return {
|
|
1033
|
-
"id": "verify-pair-trigger-primary-judge-blocker-preknown",
|
|
1034
|
-
"message": (
|
|
1035
|
-
"pair_trigger cannot skip the pair-JUDGE for a primary JUDGE blocker "
|
|
1036
|
-
"when outcome-independent reasons applied at spawn: "
|
|
1037
|
-
+ ", ".join(preknown_reasons)
|
|
1038
|
-
),
|
|
1039
|
-
"file": "pipeline.state.json",
|
|
1040
|
-
}
|
|
1041
|
-
return None
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
def pair_trigger_reason_completeness_violation(
|
|
1045
|
-
devlyn: pathlib.Path,
|
|
1046
|
-
source_verdicts: dict[str, str | None],
|
|
1047
|
-
) -> dict[str, Any] | None:
|
|
1048
|
-
if rank(source_verdicts.get("mechanical")) >= 2:
|
|
1049
|
-
return None
|
|
1050
|
-
state_path = devlyn / "pipeline.state.json"
|
|
1051
|
-
if not state_path.is_file():
|
|
1052
|
-
return None
|
|
1053
|
-
try:
|
|
1054
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
1055
|
-
except ValueError:
|
|
1056
|
-
return None
|
|
1057
|
-
phases = state.get("phases") if isinstance(state, dict) else {}
|
|
1058
|
-
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
1059
|
-
trigger = None
|
|
1060
|
-
if isinstance(verify_phase, dict):
|
|
1061
|
-
trigger = verify_phase.get("pair_trigger")
|
|
1062
|
-
if trigger is None and isinstance(state, dict):
|
|
1063
|
-
verify_state = state.get("verify")
|
|
1064
|
-
if isinstance(verify_state, dict):
|
|
1065
|
-
trigger = verify_state.get("pair_trigger")
|
|
1066
|
-
if not isinstance(trigger, dict) or trigger.get("eligible") is not True:
|
|
1067
|
-
return None
|
|
1068
|
-
reasons = trigger.get("reasons")
|
|
1069
|
-
if not isinstance(reasons, list) or not all(isinstance(item, str) for item in reasons):
|
|
1070
|
-
return None
|
|
1071
|
-
missing = [
|
|
1072
|
-
reason
|
|
1073
|
-
for reason in state_pair_trigger_reasons(devlyn, source_verdicts)
|
|
1074
|
-
if reason not in reasons
|
|
1075
|
-
]
|
|
1076
|
-
if not missing:
|
|
1077
|
-
return None
|
|
1078
|
-
return {
|
|
1079
|
-
"id": "verify-pair-trigger-reasons-incomplete",
|
|
1080
|
-
"message": (
|
|
1081
|
-
"pair_trigger.reasons is missing applicable canonical reason(s): "
|
|
1082
|
-
+ ", ".join(missing)
|
|
1083
|
-
),
|
|
1084
|
-
"file": "pipeline.state.json",
|
|
1085
|
-
}
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
def pair_blocker(
|
|
1089
|
-
id_: str,
|
|
1090
|
-
message: str,
|
|
1091
|
-
file_: str | None = None,
|
|
1092
|
-
rule_id: str = "verify.pair.emission-contract",
|
|
1093
|
-
) -> dict[str, Any]:
|
|
1094
|
-
return {
|
|
1095
|
-
"id": id_,
|
|
1096
|
-
"rule_id": rule_id,
|
|
1097
|
-
"severity": "CRITICAL",
|
|
1098
|
-
"confidence": "high",
|
|
1099
|
-
"file": file_,
|
|
1100
|
-
"line": 1 if file_ else None,
|
|
1101
|
-
"message": message,
|
|
1102
|
-
"criterion_ref": "verify.pair.findings",
|
|
1103
|
-
"source": "pair_judge",
|
|
1104
|
-
}
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
def read_pair_timeout_marker(devlyn: pathlib.Path) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
1108
|
-
path = devlyn / "verify.pair.timeout.json"
|
|
1109
|
-
if not path.is_file():
|
|
1110
|
-
return None, None
|
|
1111
|
-
try:
|
|
1112
|
-
marker = loads_strict_json(path.read_text(encoding="utf-8"))
|
|
1113
|
-
except ValueError as exc:
|
|
1114
|
-
return None, {
|
|
1115
|
-
"id": "verify-pair-timeout-marker-malformed",
|
|
1116
|
-
"message": f"verify.pair.timeout.json is malformed JSON: {exc}",
|
|
1117
|
-
"file": "verify.pair.timeout.json",
|
|
1118
|
-
}
|
|
1119
|
-
if not isinstance(marker, dict):
|
|
1120
|
-
return None, {
|
|
1121
|
-
"id": "verify-pair-timeout-marker-malformed",
|
|
1122
|
-
"message": "verify.pair.timeout.json must be a JSON object.",
|
|
1123
|
-
"file": "verify.pair.timeout.json",
|
|
1124
|
-
}
|
|
1125
|
-
engine = marker.get("engine")
|
|
1126
|
-
budget_seconds = marker.get("budget_seconds")
|
|
1127
|
-
if not isinstance(engine, str) or not engine:
|
|
1128
|
-
return None, {
|
|
1129
|
-
"id": "verify-pair-timeout-marker-malformed",
|
|
1130
|
-
"message": "verify.pair.timeout.json engine must be a non-empty string.",
|
|
1131
|
-
"file": "verify.pair.timeout.json",
|
|
1132
|
-
}
|
|
1133
|
-
if (
|
|
1134
|
-
not isinstance(budget_seconds, int)
|
|
1135
|
-
or isinstance(budget_seconds, bool)
|
|
1136
|
-
or budget_seconds <= 0
|
|
1137
|
-
):
|
|
1138
|
-
return None, {
|
|
1139
|
-
"id": "verify-pair-timeout-marker-malformed",
|
|
1140
|
-
"message": "verify.pair.timeout.json budget_seconds must be a positive integer.",
|
|
1141
|
-
"file": "verify.pair.timeout.json",
|
|
1142
|
-
}
|
|
1143
|
-
return {"engine": engine, "budget_seconds": budget_seconds}, None
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
def is_result_less_message_stream(stdout_text: str) -> bool:
|
|
1147
|
-
"""True when a declared whole-message stream stops before its terminal result.
|
|
1148
|
-
|
|
1149
|
-
The timeout marker attests the killed process. This classifier admits only a
|
|
1150
|
-
valid stream prefix, optionally ending in one truncated JSON record.
|
|
1151
|
-
"""
|
|
1152
|
-
lines = [raw for line in stdout_text.splitlines() if (raw := line.strip())]
|
|
1153
|
-
if not lines:
|
|
1154
|
-
return False
|
|
1155
|
-
try:
|
|
1156
|
-
first = JUDGE_OUTPUT_PARSER["loads_strict_json"](lines[0])
|
|
1157
|
-
except ValueError:
|
|
1158
|
-
return False
|
|
1159
|
-
if (
|
|
1160
|
-
not isinstance(first, dict)
|
|
1161
|
-
or first.get("type") != "system"
|
|
1162
|
-
or first.get("subtype") != "init"
|
|
1163
|
-
or not isinstance(first.get("session_id"), str)
|
|
1164
|
-
):
|
|
1165
|
-
return False
|
|
1166
|
-
session_id = first["session_id"]
|
|
1167
|
-
for index, raw in enumerate(lines[1:], 1):
|
|
1168
|
-
try:
|
|
1169
|
-
item = JUDGE_OUTPUT_PARSER["loads_strict_json"](raw)
|
|
1170
|
-
except ValueError:
|
|
1171
|
-
return index == len(lines) - 1 and raw.startswith("{") and not raw.endswith("}")
|
|
1172
|
-
if (
|
|
1173
|
-
not isinstance(item, dict)
|
|
1174
|
-
or item.get("type") not in {"assistant", "user"}
|
|
1175
|
-
or item.get("session_id") != session_id
|
|
1176
|
-
):
|
|
1177
|
-
return False
|
|
1178
|
-
return True
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
def detect_pair_stdout_contract_violations(
|
|
1182
|
-
devlyn: pathlib.Path,
|
|
1183
|
-
source_verdicts: dict[str, str | None],
|
|
1184
|
-
) -> list[dict[str, Any]]:
|
|
1185
|
-
# The primary uses its explicit phase engine, or the legacy executor; only the OTHER engine's
|
|
1186
|
-
# capture is pair evidence.
|
|
1187
|
-
timeout_marker, timeout_violation = read_pair_timeout_marker(devlyn)
|
|
1188
|
-
if timeout_violation is not None:
|
|
1189
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1190
|
-
return [
|
|
1191
|
-
pair_blocker(
|
|
1192
|
-
timeout_violation["id"],
|
|
1193
|
-
timeout_violation["message"],
|
|
1194
|
-
timeout_violation["file"],
|
|
1195
|
-
)
|
|
1196
|
-
]
|
|
1197
|
-
flag_violation = pair_flag_contract_violation(devlyn)
|
|
1198
|
-
if flag_violation is not None:
|
|
1199
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1200
|
-
return [
|
|
1201
|
-
pair_blocker(
|
|
1202
|
-
flag_violation["id"],
|
|
1203
|
-
flag_violation["message"],
|
|
1204
|
-
flag_violation["file"],
|
|
1205
|
-
)
|
|
1206
|
-
]
|
|
1207
|
-
required, malformed_trigger = pair_trigger_status(devlyn)
|
|
1208
|
-
if malformed_trigger is not None:
|
|
1209
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1210
|
-
return [
|
|
1211
|
-
pair_blocker(
|
|
1212
|
-
malformed_trigger["id"],
|
|
1213
|
-
malformed_trigger["message"],
|
|
1214
|
-
malformed_trigger["file"],
|
|
1215
|
-
)
|
|
1216
|
-
]
|
|
1217
|
-
risk_profile_violation = risk_profile_contract_violation(devlyn)
|
|
1218
|
-
if risk_profile_violation is not None:
|
|
1219
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1220
|
-
return [
|
|
1221
|
-
pair_blocker(
|
|
1222
|
-
risk_profile_violation["id"],
|
|
1223
|
-
risk_profile_violation["message"],
|
|
1224
|
-
risk_profile_violation["file"],
|
|
1225
|
-
)
|
|
1226
|
-
]
|
|
1227
|
-
state_violation = verify_state_contract_violation(devlyn)
|
|
1228
|
-
if state_violation is not None:
|
|
1229
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1230
|
-
return [
|
|
1231
|
-
pair_blocker(
|
|
1232
|
-
state_violation["id"],
|
|
1233
|
-
state_violation["message"],
|
|
1234
|
-
state_violation["file"],
|
|
1235
|
-
state_violation["rule_id"],
|
|
1236
|
-
)
|
|
1237
|
-
]
|
|
1238
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
1239
|
-
assert isinstance(state, dict)
|
|
1240
|
-
engine = resolved_primary_engine(state)
|
|
1241
|
-
assert isinstance(engine, str) and engine.strip()
|
|
1242
|
-
stdout_paths = canonical_other_judge_stdout(devlyn, engine)
|
|
1243
|
-
if not required and not pair_trigger_present(devlyn):
|
|
1244
|
-
missing_violation = pair_trigger_missing_contract_violation(devlyn, source_verdicts)
|
|
1245
|
-
if missing_violation is not None:
|
|
1246
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1247
|
-
return [
|
|
1248
|
-
pair_blocker(
|
|
1249
|
-
missing_violation["id"],
|
|
1250
|
-
missing_violation["message"],
|
|
1251
|
-
missing_violation["file"],
|
|
1252
|
-
)
|
|
1253
|
-
]
|
|
1254
|
-
skip_violation = pair_trigger_skip_contract_violation(devlyn, source_verdicts)
|
|
1255
|
-
if skip_violation is not None:
|
|
1256
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1257
|
-
return [
|
|
1258
|
-
pair_blocker(
|
|
1259
|
-
skip_violation["id"],
|
|
1260
|
-
skip_violation["message"],
|
|
1261
|
-
skip_violation["file"],
|
|
1262
|
-
)
|
|
1263
|
-
]
|
|
1264
|
-
reason_violation = pair_trigger_reason_completeness_violation(devlyn, source_verdicts)
|
|
1265
|
-
if reason_violation is not None:
|
|
1266
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1267
|
-
return [
|
|
1268
|
-
pair_blocker(
|
|
1269
|
-
reason_violation["id"],
|
|
1270
|
-
reason_violation["message"],
|
|
1271
|
-
reason_violation["file"],
|
|
1272
|
-
)
|
|
1273
|
-
]
|
|
1274
|
-
trigger_present = pair_trigger_present(devlyn)
|
|
1275
|
-
if trigger_present and not required:
|
|
1276
|
-
if pair_capture_exists(devlyn, stdout_paths):
|
|
1277
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1278
|
-
return [
|
|
1279
|
-
pair_blocker(
|
|
1280
|
-
"verify-pair-skipped-output-present",
|
|
1281
|
-
"Pair state is skipped or ineligible, but a canonical OTHER-judge carrier exists.",
|
|
1282
|
-
stdout_paths[0].name if stdout_paths else "verify.pair.findings.jsonl",
|
|
1283
|
-
"verify.pair.state-capture-contradiction",
|
|
1284
|
-
)
|
|
1285
|
-
]
|
|
1286
|
-
source_verdicts["pair_judge"] = None
|
|
1287
|
-
return []
|
|
1288
|
-
if len(stdout_paths) > 1:
|
|
1289
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1290
|
-
return [
|
|
1291
|
-
pair_blocker(
|
|
1292
|
-
"verify-pair-output-ambiguous",
|
|
1293
|
-
"Multiple canonical OTHER-engine stdout carriers exist: "
|
|
1294
|
-
+ ", ".join(path.name for path in stdout_paths),
|
|
1295
|
-
stdout_paths[0].name,
|
|
1296
|
-
)
|
|
1297
|
-
]
|
|
1298
|
-
if timeout_marker is not None:
|
|
1299
|
-
expected_timeout_stdout = f"{timeout_marker['engine']}-judge.stdout"
|
|
1300
|
-
if timeout_marker["engine"] == engine or (
|
|
1301
|
-
stdout_paths and stdout_paths[0].name != expected_timeout_stdout
|
|
1302
|
-
):
|
|
1303
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1304
|
-
return [
|
|
1305
|
-
pair_blocker(
|
|
1306
|
-
"verify-pair-timeout-engine-mismatch",
|
|
1307
|
-
"Pair timeout engine does not match the canonical OTHER-engine carrier.",
|
|
1308
|
-
"verify.pair.timeout.json",
|
|
1309
|
-
)
|
|
1310
|
-
]
|
|
1311
|
-
if not stdout_paths:
|
|
1312
|
-
if timeout_marker is not None:
|
|
1313
|
-
if rank(source_verdicts["pair_judge"]) <= rank("TIMEOUT"):
|
|
1314
|
-
source_verdicts["pair_judge"] = "TIMEOUT"
|
|
1315
|
-
return []
|
|
1316
|
-
if required:
|
|
1317
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1318
|
-
return [
|
|
1319
|
-
pair_blocker(
|
|
1320
|
-
"verify-pair-required-output-missing",
|
|
1321
|
-
"Pair-mode was required, but the pair-JUDGE produced no canonical OTHER-engine stdout.",
|
|
1322
|
-
"verify.pair.findings.jsonl",
|
|
1323
|
-
)
|
|
1324
|
-
]
|
|
1325
|
-
return []
|
|
1326
|
-
if has_pair_findings(devlyn):
|
|
1327
|
-
return []
|
|
1328
|
-
if source_verdicts["pair_judge"] is None and timeout_marker is None:
|
|
1329
|
-
source_verdicts["pair_judge"] = "PASS"
|
|
1330
|
-
for stdout_path in stdout_paths:
|
|
1331
|
-
stdout_text = stdout_path.read_text(encoding="utf-8")
|
|
1332
|
-
if not stdout_text.strip():
|
|
1333
|
-
if timeout_marker is not None:
|
|
1334
|
-
continue
|
|
1335
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1336
|
-
return [
|
|
1337
|
-
pair_blocker(
|
|
1338
|
-
"verify-pair-empty-output",
|
|
1339
|
-
f"pair-JUDGE stdout {stdout_path.name} was empty; the bounded contract requires a JSONL finding or PASS line.",
|
|
1340
|
-
stdout_path.name,
|
|
1341
|
-
)
|
|
1342
|
-
]
|
|
1343
|
-
try:
|
|
1344
|
-
stdout_findings, stdout_summary = JUDGE_OUTPUT_PARSER["collect_stdout"](stdout_path)
|
|
1345
|
-
except SystemExit as exc:
|
|
1346
|
-
if timeout_marker is not None and is_result_less_message_stream(stdout_text):
|
|
1347
|
-
continue
|
|
1348
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1349
|
-
return [
|
|
1350
|
-
pair_blocker(
|
|
1351
|
-
"verify-pair-emission-contract-violated",
|
|
1352
|
-
f"pair-JUDGE stdout {stdout_path.name} violates the shared emission parser: {exc}",
|
|
1353
|
-
stdout_path.name,
|
|
1354
|
-
)
|
|
1355
|
-
]
|
|
1356
|
-
if stdout_findings or stdout_summary is None or stdout_summary["verdict"] != "PASS":
|
|
1357
|
-
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1358
|
-
return [
|
|
1359
|
-
pair_blocker(
|
|
1360
|
-
"verify-pair-emission-contract-violated",
|
|
1361
|
-
(
|
|
1362
|
-
f"pair-JUDGE stdout {stdout_path.name} contained findings or a non-PASS "
|
|
1363
|
-
"verdict, but the canonical pair findings JSONL file was empty."
|
|
1364
|
-
),
|
|
1365
|
-
stdout_path.name,
|
|
1366
|
-
)
|
|
1367
|
-
]
|
|
1368
|
-
if timeout_marker is not None:
|
|
1369
|
-
source_verdicts["pair_judge"] = "TIMEOUT"
|
|
1370
|
-
return []
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
def write_outputs(
|
|
1374
|
-
devlyn: pathlib.Path,
|
|
1375
|
-
findings: list[dict[str, Any]],
|
|
1376
|
-
source_verdicts: dict[str, str | None],
|
|
1377
|
-
) -> dict[str, Any]:
|
|
1378
|
-
merged_path = devlyn / "verify-merged.findings.jsonl"
|
|
1379
|
-
summary_path = devlyn / "verify-merge.summary.json"
|
|
1380
|
-
with merged_path.open("w", encoding="utf-8") as handle:
|
|
1381
|
-
for finding in findings:
|
|
1382
|
-
handle.write(json.dumps(finding, sort_keys=True, separators=(",", ":")) + "\n")
|
|
1383
|
-
verdict = "PASS"
|
|
1384
|
-
for source_verdict in source_verdicts.values():
|
|
1385
|
-
verdict = worse(verdict, source_verdict)
|
|
1386
|
-
summary = {
|
|
1387
|
-
"verdict": verdict,
|
|
1388
|
-
"source_verdicts": source_verdicts,
|
|
1389
|
-
"findings_count": len(findings),
|
|
1390
|
-
"findings_file": str(merged_path),
|
|
1391
|
-
}
|
|
1392
|
-
if source_verdicts.get("pair_judge") == "TIMEOUT":
|
|
1393
|
-
timeout_marker, _timeout_violation = read_pair_timeout_marker(devlyn)
|
|
1394
|
-
if timeout_marker is not None:
|
|
1395
|
-
summary["pair_timeout"] = timeout_marker
|
|
1396
|
-
summary["report_header_note"] = "solo verdict after pair TIMEOUT"
|
|
1397
|
-
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
1398
|
-
return summary
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
def write_state(devlyn: pathlib.Path, summary: dict[str, Any]) -> None:
|
|
1402
|
-
state_path = devlyn / "pipeline.state.json"
|
|
1403
|
-
if not state_path.is_file():
|
|
1404
|
-
raise SystemExit(f"error: {state_path} not found")
|
|
1405
|
-
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
1406
|
-
try:
|
|
1407
|
-
carrier = mechanical_evidence_carrier(devlyn)
|
|
1408
|
-
except (PROCESS_EVIDENCE["EvidenceError"], OSError, UnicodeError, ValueError) as exc:
|
|
1409
|
-
if summary.get("source_verdicts", {}).get("mechanical") != "BLOCKED":
|
|
1410
|
-
raise SystemExit(f"error: invalid MECHANICAL process evidence: {exc}") from exc
|
|
1411
|
-
carrier = None
|
|
1412
|
-
if carrier is not None:
|
|
1413
|
-
bound = state.get("process_evidence")
|
|
1414
|
-
if bound is None:
|
|
1415
|
-
bound = []
|
|
1416
|
-
elif not isinstance(bound, list):
|
|
1417
|
-
raise SystemExit("error: state.process_evidence must be null or an array")
|
|
1418
|
-
same_round = [
|
|
1419
|
-
item for item in bound
|
|
1420
|
-
if isinstance(item, dict)
|
|
1421
|
-
and item.get("phase") == carrier["phase"]
|
|
1422
|
-
and item.get("round") == carrier["round"]
|
|
1423
|
-
]
|
|
1424
|
-
if same_round and same_round != [carrier]:
|
|
1425
|
-
raise SystemExit("error: state-bound VERIFY evidence disagrees with MECHANICAL results")
|
|
1426
|
-
if not same_round:
|
|
1427
|
-
bound.append(carrier)
|
|
1428
|
-
state["process_evidence"] = bound
|
|
1429
|
-
phases = state.setdefault("phases", {})
|
|
1430
|
-
verify = phases.get("verify")
|
|
1431
|
-
if not isinstance(verify, dict):
|
|
1432
|
-
verify = {}
|
|
1433
|
-
phases["verify"] = verify
|
|
1434
|
-
role_evidence = {}
|
|
1435
|
-
try:
|
|
1436
|
-
required_roles = JUDGE_ROLE_EVIDENCE["required_roles"](state)
|
|
1437
|
-
except ValueError as exc:
|
|
1438
|
-
if summary.get("source_verdicts", {}).get("judge") != "BLOCKED":
|
|
1439
|
-
raise SystemExit(str(exc)) from exc
|
|
1440
|
-
required_roles = [] # Persist the invalid-resolution failure; no legacy dispatch.
|
|
1441
|
-
for role in required_roles:
|
|
1442
|
-
source = "judge" if role == "primary_judge" else "pair_judge"
|
|
1443
|
-
if summary.get("source_verdicts", {}).get(source) in {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK"}:
|
|
1444
|
-
role_evidence[role] = JUDGE_ROLE_EVIDENCE["authenticate"](devlyn, state, role)
|
|
1445
|
-
if role_evidence:
|
|
1446
|
-
verify["role_evidence"] = role_evidence
|
|
1447
|
-
verify["verdict"] = summary["verdict"]
|
|
1448
|
-
sub = verify.get("sub_verdicts")
|
|
1449
|
-
if sub is None:
|
|
1450
|
-
# spawn (state-phase-write.py) always writes sub_verdicts: null as
|
|
1451
|
-
# part of the per-round reset contract (state-schema.md#write-protocol)
|
|
1452
|
-
# — legal, expected state before this function populates it.
|
|
1453
|
-
# setdefault() would not replace an existing null, only an absent key.
|
|
1454
|
-
sub = {}
|
|
1455
|
-
verify["sub_verdicts"] = sub
|
|
1456
|
-
elif not isinstance(sub, dict):
|
|
1457
|
-
raise SystemExit(
|
|
1458
|
-
f"error: phases.verify.sub_verdicts must be null or an object, got {type(sub).__name__}"
|
|
1459
|
-
)
|
|
1460
|
-
for source, source_verdict in summary["source_verdicts"].items():
|
|
1461
|
-
if source in {"mechanical", "judge", "pair_judge"}:
|
|
1462
|
-
sub[source] = source_verdict
|
|
1463
|
-
verify["merged"] = {
|
|
1464
|
-
"verdict": summary["verdict"],
|
|
1465
|
-
"findings_file": ".devlyn/verify-merged.findings.jsonl",
|
|
1466
|
-
"summary_file": ".devlyn/verify-merge.summary.json",
|
|
1467
|
-
}
|
|
1468
|
-
state_path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
def self_test() -> int:
|
|
1472
|
-
import subprocess
|
|
1473
|
-
|
|
1474
|
-
with tempfile.TemporaryDirectory() as tmp:
|
|
1475
|
-
work = pathlib.Path(tmp).resolve()
|
|
1476
|
-
devlyn = work / ".devlyn"
|
|
1477
|
-
devlyn.mkdir()
|
|
1478
|
-
role = JUDGE_ROLE_EVIDENCE["ROLE"]
|
|
1479
|
-
(devlyn / "engines.json").write_bytes(role["encoded"]({
|
|
1480
|
-
"roles": {"primary_judge": {"engine": "claude", "model": "fixture-claude-model", "effort": "high"}},
|
|
1481
|
-
}))
|
|
1482
|
-
resolution = role["resolve"](work, "claude", no_pair=True, available=lambda engine: True)
|
|
1483
|
-
state = {"run_id": "merge-relative-role", "engine": "claude", "role_resolution": resolution,
|
|
1484
|
-
"rounds": {"global": 0, "max_rounds": 1},
|
|
1485
|
-
"phases": {"verify": {"engine": "claude", "round": 0, "verdict": None,
|
|
1486
|
-
"started_at": "2026-01-01T00:00:00Z", "completed_at": "2026-01-01T00:01:00Z"}}}
|
|
1487
|
-
state_path = devlyn / "pipeline.state.json"
|
|
1488
|
-
state_path.write_text(json.dumps(state), encoding="utf-8")
|
|
1489
|
-
stem = "claude-judge.r0"
|
|
1490
|
-
prompt_path = devlyn / (stem + ".prompt")
|
|
1491
|
-
prompt = b"review"
|
|
1492
|
-
prompt_path.write_bytes(prompt)
|
|
1493
|
-
command = ["claude", "-p", "--model", "fixture-claude-model", "--effort", "high",
|
|
1494
|
-
"--permission-mode", "dontAsk", "--tools", "Read,Grep,Glob",
|
|
1495
|
-
"--allowedTools", "Read,Grep,Glob", "--setting-sources", "project",
|
|
1496
|
-
"--output-format", "json", "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}']
|
|
1497
|
-
argv = ["python3", "run-bounded.py", "600", "--stdin-file", str(prompt_path),
|
|
1498
|
-
"--record-transport", "--", *command]
|
|
1499
|
-
argv_path = devlyn / (stem + ".argv.json")
|
|
1500
|
-
argv_path.write_bytes(role["encoded"](argv))
|
|
1501
|
-
(devlyn / (stem + ".stderr")).write_bytes(b"")
|
|
1502
|
-
(devlyn / (stem + ".output.json")).write_bytes(role["encoded"]({
|
|
1503
|
-
"type": "result", "subtype": "success", "is_error": False, "stop_reason": "end_turn",
|
|
1504
|
-
"session_id": "fixture-claude", "result": "PASS", "modelUsage": {"fixture-claude-model": {}},
|
|
1505
|
-
}))
|
|
1506
|
-
transport = {"schema_version": 1, "transport": "stdin-file",
|
|
1507
|
-
"prompt": {"path": str(prompt_path), "sha256": role["digest"](prompt), "bytes": len(prompt)},
|
|
1508
|
-
"command": command, "argv": command, "timeout_sec": 600, "isolated": False,
|
|
1509
|
-
"status": "completed", "exit_code": 0}
|
|
1510
|
-
(devlyn / (stem + ".prompt.transport.json")).write_text(json.dumps(transport), encoding="utf-8")
|
|
1511
|
-
evidence, derived = JUDGE_ROLE_EVIDENCE["describe"](devlyn, state, "primary_judge", 0)
|
|
1512
|
-
(devlyn / (stem + ".role-evidence.json")).write_bytes(role["encoded"](evidence))
|
|
1513
|
-
(devlyn / (stem + ".stdout")).write_bytes(derived)
|
|
1514
|
-
(devlyn / "claude-judge.stdout").write_bytes(derived)
|
|
1515
|
-
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1516
|
-
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1517
|
-
script = str(pathlib.Path(__file__).resolve())
|
|
1518
|
-
def merge_at(directory):
|
|
1519
|
-
return subprocess.run([sys.executable, script, "--devlyn-dir", directory, "--write-state"],
|
|
1520
|
-
cwd=work, capture_output=True, text=True)
|
|
1521
|
-
relative = merge_at(".devlyn")
|
|
1522
|
-
assert relative.returncode == 0, relative.stderr
|
|
1523
|
-
relative_state = json.loads(state_path.read_text())
|
|
1524
|
-
assert relative_state["phases"]["verify"]["verdict"] == "PASS", relative.stdout
|
|
1525
|
-
state_path.write_text(json.dumps(state), encoding="utf-8")
|
|
1526
|
-
absolute = merge_at(str(devlyn))
|
|
1527
|
-
assert absolute.returncode == 0, absolute.stderr
|
|
1528
|
-
absolute_state = json.loads(state_path.read_text())
|
|
1529
|
-
assert absolute_state["phases"]["verify"]["role_evidence"] == relative_state["phases"]["verify"]["role_evidence"]
|
|
1530
|
-
argv[4] = str(work / "wrong.prompt")
|
|
1531
|
-
argv_path.write_bytes(role["encoded"](argv))
|
|
1532
|
-
state_path.write_text(json.dumps(state), encoding="utf-8")
|
|
1533
|
-
wrong = merge_at(".devlyn")
|
|
1534
|
-
assert wrong.returncode == 0, wrong.stderr
|
|
1535
|
-
blocked_state = json.loads(state_path.read_text())
|
|
1536
|
-
assert blocked_state["phases"]["verify"]["verdict"] == "BLOCKED"
|
|
1537
|
-
writer = runpy.run_path(str(pathlib.Path(__file__).with_name("state-phase-write.py")))
|
|
1538
|
-
try:
|
|
1539
|
-
writer["do_spawn"](blocked_state, "implement", 1, "verify", None, None, None)
|
|
1540
|
-
except SystemExit as exc:
|
|
1541
|
-
assert str(exc) == "BLOCKED:repair-edge-invalid"
|
|
1542
|
-
else:
|
|
1543
|
-
raise AssertionError("BLOCKED VERIFY entered product repair")
|
|
1544
|
-
assert blocked_state["rounds"]["global"] == 0
|
|
1545
|
-
argv[4] = str(prompt_path)
|
|
1546
|
-
argv_path.write_bytes(role["encoded"](argv))
|
|
1547
|
-
finding = {"id": "fixture-high", "rule_id": "fixture.binding", "severity": "HIGH",
|
|
1548
|
-
"confidence": "high", "file": "source.txt", "line": 1,
|
|
1549
|
-
"message": "required behavior is missing", "criterion_ref": "fixture"}
|
|
1550
|
-
(devlyn / "verify.findings.jsonl").write_text(json.dumps(finding) + "\n", encoding="utf-8")
|
|
1551
|
-
state_path.write_text(json.dumps(state), encoding="utf-8")
|
|
1552
|
-
needs_work = merge_at(".devlyn")
|
|
1553
|
-
assert needs_work.returncode == 0, needs_work.stderr
|
|
1554
|
-
repair_state = json.loads(state_path.read_text())
|
|
1555
|
-
assert repair_state["phases"]["verify"]["verdict"] == "NEEDS_WORK"
|
|
1556
|
-
writer["do_spawn"](repair_state, "implement", 1, "verify", None, None, None)
|
|
1557
|
-
assert repair_state["rounds"]["global"] == 1
|
|
1558
|
-
|
|
1559
|
-
try:
|
|
1560
|
-
loads_strict_json('{"verdict":"PASS","verdict":"BLOCKED"}')
|
|
1561
|
-
except ValueError as exc:
|
|
1562
|
-
assert "duplicate JSON key" in str(exc)
|
|
1563
|
-
else:
|
|
1564
|
-
raise AssertionError("duplicate VERIFY authority key was accepted")
|
|
1565
|
-
with tempfile.TemporaryDirectory() as tmp:
|
|
1566
|
-
devlyn = pathlib.Path(tmp)
|
|
1567
|
-
|
|
1568
|
-
# Every completed VERIFY has deterministic mechanical and primary
|
|
1569
|
-
# judge carriers. Missing either one is an evidence failure, not PASS.
|
|
1570
|
-
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1571
|
-
|
|
1572
|
-
# state-phase-write.py's spawn always writes sub_verdicts: null (the
|
|
1573
|
-
# per-round reset contract, state-schema.md#write-protocol) — this is
|
|
1574
|
-
# the real shape write_state() sees on every VERIFY completion, not
|
|
1575
|
-
# the pre-populated {} the other scenarios below seed for brevity.
|
|
1576
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1577
|
-
json.dumps({
|
|
1578
|
-
"engine": "claude",
|
|
1579
|
-
"phases": {
|
|
1580
|
-
"verify": {
|
|
1581
|
-
"verdict": None,
|
|
1582
|
-
"sub_verdicts": None,
|
|
1583
|
-
"judge_durations_ms": {"judge": 23, "pair_judge": 31},
|
|
1584
|
-
}
|
|
1585
|
-
}
|
|
1586
|
-
}),
|
|
1587
|
-
encoding="utf-8",
|
|
1588
|
-
)
|
|
1589
|
-
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1590
|
-
|
|
1591
|
-
sealed_work = pathlib.Path(tmp) / "sealed-work"
|
|
1592
|
-
sealed_devlyn = sealed_work / ".devlyn"
|
|
1593
|
-
sealed_devlyn.mkdir(parents=True)
|
|
1594
|
-
sealed_state = {
|
|
1595
|
-
"run_id": "rs-sealed-mechanical",
|
|
1596
|
-
"engine": "claude",
|
|
1597
|
-
"process_evidence": None,
|
|
1598
|
-
"phases": {"verify": {"round": 2, "verdict": None, "sub_verdicts": None}},
|
|
1599
|
-
}
|
|
1600
|
-
obligation = PROCESS_EVIDENCE["normalize_obligation"]({
|
|
1601
|
-
"id": "mechanical-0",
|
|
1602
|
-
"phase": "verify",
|
|
1603
|
-
"argv": [sys.executable, "-c", "print('sealed')"],
|
|
1604
|
-
})
|
|
1605
|
-
manifest_rel = PROCESS_EVIDENCE["manifest_relative_path"](sealed_state, "verify")
|
|
1606
|
-
PROCESS_EVIDENCE["capture_process"](
|
|
1607
|
-
sealed_work, sealed_work / manifest_rel, sealed_state["run_id"],
|
|
1608
|
-
"verify", 2, obligation,
|
|
1609
|
-
)
|
|
1610
|
-
carrier = PROCESS_EVIDENCE["validate_manifest"](
|
|
1611
|
-
sealed_work, manifest_rel, sealed_state["run_id"], "verify", 2,
|
|
1612
|
-
[obligation], require_expectations=False,
|
|
1613
|
-
)
|
|
1614
|
-
(sealed_devlyn / "pipeline.state.json").write_text(
|
|
1615
|
-
json.dumps(sealed_state), encoding="utf-8",
|
|
1616
|
-
)
|
|
1617
|
-
sealed_commands = PROCESS_EVIDENCE["bound_carrier_summary_commands"](
|
|
1618
|
-
sealed_work, carrier,
|
|
1619
|
-
)
|
|
1620
|
-
(sealed_devlyn / "spec-verify.results.json").write_text(
|
|
1621
|
-
json.dumps({"commands": sealed_commands, "process_evidence": carrier}),
|
|
1622
|
-
encoding="utf-8",
|
|
1623
|
-
)
|
|
1624
|
-
(sealed_devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1625
|
-
(sealed_devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1626
|
-
sealed_findings, sealed_verdicts = read_findings(sealed_devlyn)
|
|
1627
|
-
assert sealed_verdicts["mechanical"] == "PASS", sealed_findings
|
|
1628
|
-
sealed_summary = write_outputs(sealed_devlyn, sealed_findings, sealed_verdicts)
|
|
1629
|
-
write_state(sealed_devlyn, sealed_summary)
|
|
1630
|
-
bound_state = loads_strict_json(
|
|
1631
|
-
(sealed_devlyn / "pipeline.state.json").read_text(encoding="utf-8")
|
|
1632
|
-
)
|
|
1633
|
-
assert bound_state["process_evidence"] == [carrier], bound_state
|
|
1634
|
-
|
|
1635
|
-
failed_work = pathlib.Path(tmp) / "failed-work"
|
|
1636
|
-
failed_devlyn = failed_work / ".devlyn"
|
|
1637
|
-
failed_devlyn.mkdir(parents=True)
|
|
1638
|
-
failed_state = {
|
|
1639
|
-
"version": "3.0",
|
|
1640
|
-
"run_id": "rs-failed-mechanical",
|
|
1641
|
-
"engine": "claude",
|
|
1642
|
-
"source": {"type": "spec", "spec_path": "docs/failed/spec.md"},
|
|
1643
|
-
"process_evidence": None,
|
|
1644
|
-
"phases": {"verify": {"round": 0, "verdict": None, "sub_verdicts": None}},
|
|
1645
|
-
}
|
|
1646
|
-
failed_spec = failed_work / "docs" / "failed"
|
|
1647
|
-
failed_spec.mkdir(parents=True)
|
|
1648
|
-
(failed_spec / "spec.md").write_text("# failed fixture\n", encoding="utf-8")
|
|
1649
|
-
(failed_spec / "spec.expected.json").write_text(json.dumps({
|
|
1650
|
-
"verification_commands": [{"cmd": "exit 7", "exit_code": 0}],
|
|
1651
|
-
}) + "\n", encoding="utf-8")
|
|
1652
|
-
failed_obligation = PROCESS_EVIDENCE["normalize_obligation"]({
|
|
1653
|
-
"id": "verification-command-0001",
|
|
1654
|
-
"phase": "verify",
|
|
1655
|
-
"cmd": "exit 7",
|
|
1656
|
-
})
|
|
1657
|
-
failed_manifest = PROCESS_EVIDENCE["manifest_relative_path"](failed_state, "verify")
|
|
1658
|
-
PROCESS_EVIDENCE["capture_process"](
|
|
1659
|
-
failed_work, failed_work / failed_manifest, failed_state["run_id"],
|
|
1660
|
-
"verify", 0, failed_obligation,
|
|
1661
|
-
)
|
|
1662
|
-
failed_carrier = PROCESS_EVIDENCE["validate_manifest"](
|
|
1663
|
-
failed_work, failed_manifest, failed_state["run_id"], "verify", 0,
|
|
1664
|
-
[failed_obligation], require_expectations=False,
|
|
1665
|
-
)
|
|
1666
|
-
(failed_devlyn / "pipeline.state.json").write_text(
|
|
1667
|
-
json.dumps(failed_state), encoding="utf-8",
|
|
1668
|
-
)
|
|
1669
|
-
# Exact laundering attempt: mutable derivatives say PASS/empty while
|
|
1670
|
-
# the state-identical sealed manifest still records the failed command.
|
|
1671
|
-
laundered_commands = PROCESS_EVIDENCE["bound_carrier_summary_commands"](
|
|
1672
|
-
failed_work, failed_carrier,
|
|
1673
|
-
)
|
|
1674
|
-
laundered_commands[0]["pass"] = True
|
|
1675
|
-
(failed_devlyn / "spec-verify.results.json").write_text(json.dumps({
|
|
1676
|
-
"commands": laundered_commands, "process_evidence": failed_carrier,
|
|
1677
|
-
}), encoding="utf-8")
|
|
1678
|
-
(failed_devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1679
|
-
(failed_devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1680
|
-
failed_findings, failed_verdicts = read_findings(failed_devlyn)
|
|
1681
|
-
assert failed_verdicts["mechanical"] == "BLOCKED", failed_verdicts
|
|
1682
|
-
assert any(
|
|
1683
|
-
item.get("id") == "verify-mechanical-evidence-invalid"
|
|
1684
|
-
for item in failed_findings
|
|
1685
|
-
), failed_findings
|
|
1686
|
-
assert write_outputs(failed_devlyn, failed_findings, failed_verdicts)["verdict"] == "BLOCKED"
|
|
1687
|
-
|
|
1688
|
-
(failed_devlyn / "spec-verify.results.json").write_text(json.dumps({
|
|
1689
|
-
"commands": [], "process_evidence": None,
|
|
1690
|
-
}), encoding="utf-8")
|
|
1691
|
-
removed_findings, removed_verdicts = read_findings(failed_devlyn)
|
|
1692
|
-
assert removed_verdicts["mechanical"] == "BLOCKED", removed_verdicts
|
|
1693
|
-
assert any(
|
|
1694
|
-
item.get("id") == "verify-mechanical-evidence-invalid"
|
|
1695
|
-
for item in removed_findings
|
|
1696
|
-
), removed_findings
|
|
1697
|
-
print("PASS iter-0112 sealed VERIFY outcome resists mutable-derivative laundering")
|
|
1698
|
-
|
|
1699
|
-
sealed_stdout = sealed_work / carrier["streams"][0]["stdout"]["path"]
|
|
1700
|
-
sealed_stdout.write_bytes(sealed_stdout.read_bytes() + b"altered")
|
|
1701
|
-
altered_findings, altered_verdicts = read_findings(sealed_devlyn)
|
|
1702
|
-
assert altered_verdicts["mechanical"] == "BLOCKED", altered_verdicts
|
|
1703
|
-
assert any(
|
|
1704
|
-
finding.get("id") == "verify-mechanical-evidence-invalid"
|
|
1705
|
-
for finding in altered_findings
|
|
1706
|
-
), altered_findings
|
|
1707
|
-
|
|
1708
|
-
skipped_work = pathlib.Path(tmp) / "skipped-work"
|
|
1709
|
-
skipped_work.mkdir()
|
|
1710
|
-
(skipped_work / "pipeline.state.json").write_text(json.dumps({
|
|
1711
|
-
"engine": "claude",
|
|
1712
|
-
"risk_profile": {"pair_default_enabled": False},
|
|
1713
|
-
"phases": {"verify": {"pair_trigger": {
|
|
1714
|
-
"eligible": False,
|
|
1715
|
-
"reasons": [],
|
|
1716
|
-
"skipped_reason": "user_no_pair",
|
|
1717
|
-
}}},
|
|
1718
|
-
}), encoding="utf-8")
|
|
1719
|
-
(skipped_work / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1720
|
-
(skipped_work / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1721
|
-
generic_primary = skipped_work / "verify-judge.stdout"
|
|
1722
|
-
for generic_output in ("PASS\n", "NEEDS_WORK\n"):
|
|
1723
|
-
generic_primary.write_text(generic_output, encoding="utf-8")
|
|
1724
|
-
generic_findings, generic_verdicts = read_findings(skipped_work)
|
|
1725
|
-
assert generic_verdicts["pair_judge"] is None, generic_verdicts
|
|
1726
|
-
assert not any(
|
|
1727
|
-
finding.get("file") == generic_primary.name for finding in generic_findings
|
|
1728
|
-
), generic_findings
|
|
1729
|
-
(skipped_work / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
1730
|
-
contradiction_findings, contradiction_verdicts = read_findings(skipped_work)
|
|
1731
|
-
assert contradiction_verdicts["pair_judge"] == "BLOCKED", contradiction_verdicts
|
|
1732
|
-
assert any(
|
|
1733
|
-
finding.get("id") == "verify-pair-skipped-output-present"
|
|
1734
|
-
for finding in contradiction_findings
|
|
1735
|
-
), contradiction_findings
|
|
1736
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
1737
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1738
|
-
write_state(devlyn, summary)
|
|
1739
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
1740
|
-
assert summary["verdict"] == "PASS", summary
|
|
1741
|
-
assert state["phases"]["verify"]["sub_verdicts"] == {
|
|
1742
|
-
"mechanical": "PASS", "judge": "PASS", "pair_judge": None,
|
|
1743
|
-
}, state
|
|
1744
|
-
assert state["phases"]["verify"]["judge_durations_ms"] == {
|
|
1745
|
-
"judge": 23, "pair_judge": 31,
|
|
1746
|
-
}, state
|
|
1747
|
-
original_state = (devlyn / "pipeline.state.json").read_bytes()
|
|
1748
|
-
for malformed in (None, {}, {"roles": {}}):
|
|
1749
|
-
bad_state = loads_strict_json(original_state)
|
|
1750
|
-
bad_state["role_resolution"] = malformed
|
|
1751
|
-
(devlyn / "pipeline.state.json").write_text(json.dumps(bad_state), encoding="utf-8")
|
|
1752
|
-
result = subprocess.run([sys.executable, str(pathlib.Path(__file__).resolve()),
|
|
1753
|
-
"--devlyn-dir", str(devlyn), "--write-state"], capture_output=True, text=True, encoding="utf-8")
|
|
1754
|
-
assert result.returncode == 0 and "Traceback" not in result.stderr, result.stderr
|
|
1755
|
-
persisted = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
1756
|
-
assert persisted["phases"]["verify"]["verdict"] == "BLOCKED"
|
|
1757
|
-
assert "verify-role-resolution-invalid" in (devlyn / "verify-merged.findings.jsonl").read_text(encoding="utf-8")
|
|
1758
|
-
(devlyn / "pipeline.state.json").write_bytes(original_state)
|
|
1759
|
-
different_primary = {"engine": "codex", "phases": {"verify": {"engine": "claude"}}}
|
|
1760
|
-
(devlyn / "pipeline.state.json").write_text(json.dumps(different_primary), encoding="utf-8")
|
|
1761
|
-
(devlyn / "verify.primary.timeout.json").write_text(json.dumps({"engine": "claude", "budget_seconds": 600}), encoding="utf-8")
|
|
1762
|
-
marker, violation = read_primary_timeout_marker(devlyn)
|
|
1763
|
-
assert marker is not None and violation is None
|
|
1764
|
-
different_primary["phases"]["verify"]["engine"] = None
|
|
1765
|
-
(devlyn / "pipeline.state.json").write_text(json.dumps(different_primary), encoding="utf-8")
|
|
1766
|
-
assert read_primary_timeout_marker(devlyn)[1] is not None
|
|
1767
|
-
(devlyn / "verify.primary.timeout.json").unlink()
|
|
1768
|
-
(devlyn / "pipeline.state.json").write_bytes(original_state)
|
|
1769
|
-
|
|
1770
|
-
(devlyn / "verify.findings.jsonl").unlink()
|
|
1771
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
1772
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1773
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
1774
|
-
assert summary["source_verdicts"]["judge"] == "BLOCKED", summary
|
|
1775
|
-
assert any(
|
|
1776
|
-
finding["id"] == "verify-merge-required-source-missing-judge"
|
|
1777
|
-
for finding in findings
|
|
1778
|
-
), findings
|
|
1779
|
-
|
|
1780
|
-
# iter-0113: a valid primary timeout is an explicit BLOCKED source,
|
|
1781
|
-
# never the generic missing-source path or pair-style TIMEOUT.
|
|
1782
|
-
(devlyn / "verify.primary.timeout.json").write_text(
|
|
1783
|
-
json.dumps({"engine": "claude", "budget_seconds": 600}) + "\n",
|
|
1784
|
-
encoding="utf-8",
|
|
1785
|
-
)
|
|
1786
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
1787
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1788
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
1789
|
-
assert summary["source_verdicts"]["judge"] == "BLOCKED", summary
|
|
1790
|
-
assert any(
|
|
1791
|
-
finding["id"] == "verify-primary-timeout"
|
|
1792
|
-
for finding in findings
|
|
1793
|
-
), findings
|
|
1794
|
-
assert not any(
|
|
1795
|
-
finding["id"] == "verify-merge-required-source-missing-judge"
|
|
1796
|
-
for finding in findings
|
|
1797
|
-
), findings
|
|
1798
|
-
|
|
1799
|
-
(devlyn / "verify.findings.jsonl").write_text(
|
|
1800
|
-
json.dumps({"id": "primary-timeout-high", "severity": "HIGH"}) + "\n",
|
|
1801
|
-
encoding="utf-8",
|
|
1802
|
-
)
|
|
1803
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
1804
|
-
assert source_verdicts["judge"] == "BLOCKED", source_verdicts
|
|
1805
|
-
assert {finding["id"] for finding in findings} >= {
|
|
1806
|
-
"primary-timeout-high", "verify-primary-timeout",
|
|
1807
|
-
}, findings
|
|
1808
|
-
|
|
1809
|
-
for marker, expected_id in (
|
|
1810
|
-
("{\n", "verify-primary-timeout-marker-malformed"),
|
|
1811
|
-
(
|
|
1812
|
-
json.dumps({"engine": "codex", "budget_seconds": 600}) + "\n",
|
|
1813
|
-
"verify-primary-timeout-engine-mismatch",
|
|
1814
|
-
),
|
|
1815
|
-
(
|
|
1816
|
-
json.dumps({"engine": "claude", "budget_seconds": 601}) + "\n",
|
|
1817
|
-
"verify-primary-timeout-budget-mismatch",
|
|
1818
|
-
),
|
|
1819
|
-
):
|
|
1820
|
-
(devlyn / "verify.primary.timeout.json").write_text(marker, encoding="utf-8")
|
|
1821
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
1822
|
-
assert source_verdicts["judge"] == "BLOCKED", source_verdicts
|
|
1823
|
-
assert any(finding["id"] == expected_id for finding in findings), findings
|
|
1824
|
-
(devlyn / "verify.primary.timeout.json").unlink()
|
|
1825
|
-
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1826
|
-
|
|
1827
|
-
# iter-0072 Amendment 3: generated schema-v3 runs mechanically prove
|
|
1828
|
-
# raw-goal persistence and required SURFACE_CLOSE dispatch.
|
|
1829
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1830
|
-
json.dumps({
|
|
1831
|
-
"version": "3.0",
|
|
1832
|
-
"engine": "claude",
|
|
1833
|
-
"complexity": "large",
|
|
1834
|
-
"source": {"type": "generated"},
|
|
1835
|
-
"phases": {"verify": {"verdict": None, "sub_verdicts": None}},
|
|
1836
|
-
}),
|
|
1837
|
-
encoding="utf-8",
|
|
1838
|
-
)
|
|
1839
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
1840
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1841
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
1842
|
-
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
1843
|
-
assert any(
|
|
1844
|
-
finding.get("rule_id") == "verify.state.goal-persistence-missing"
|
|
1845
|
-
for finding in findings
|
|
1846
|
-
), findings
|
|
1847
|
-
|
|
1848
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1849
|
-
json.dumps({
|
|
1850
|
-
"version": "3.0",
|
|
1851
|
-
"engine": "claude",
|
|
1852
|
-
"complexity": "large",
|
|
1853
|
-
"source": {"type": "generated", "goal_path": "", "goal_sha256": "A" * 64},
|
|
1854
|
-
}),
|
|
1855
|
-
encoding="utf-8",
|
|
1856
|
-
)
|
|
1857
|
-
violation = verify_state_contract_violation(devlyn)
|
|
1858
|
-
assert violation is not None, violation
|
|
1859
|
-
assert violation["rule_id"] == "verify.state.goal-persistence-missing", violation
|
|
1860
|
-
|
|
1861
|
-
generated_source = {
|
|
1862
|
-
"type": "generated",
|
|
1863
|
-
"goal_path": ".devlyn/goal.raw.txt",
|
|
1864
|
-
"goal_sha256": "a" * 64,
|
|
1865
|
-
}
|
|
1866
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1867
|
-
json.dumps({
|
|
1868
|
-
"version": "3.0",
|
|
1869
|
-
"engine": "claude",
|
|
1870
|
-
"complexity": "medium",
|
|
1871
|
-
"source": generated_source,
|
|
1872
|
-
"phases": {
|
|
1873
|
-
"surface_close": None,
|
|
1874
|
-
"verify": {"verdict": None, "sub_verdicts": None},
|
|
1875
|
-
},
|
|
1876
|
-
}),
|
|
1877
|
-
encoding="utf-8",
|
|
1878
|
-
)
|
|
1879
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
1880
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1881
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
1882
|
-
assert any(
|
|
1883
|
-
finding.get("rule_id") == "verify.state.surface-close-skipped"
|
|
1884
|
-
for finding in findings
|
|
1885
|
-
), findings
|
|
1886
|
-
|
|
1887
|
-
for surface_entry in (
|
|
1888
|
-
{"verdict": "PASS"},
|
|
1889
|
-
{
|
|
1890
|
-
"verdict": None,
|
|
1891
|
-
"started_at": None,
|
|
1892
|
-
"skipped_reason": "auto_surface_close_claude_unavailable",
|
|
1893
|
-
},
|
|
1894
|
-
):
|
|
1895
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1896
|
-
json.dumps({
|
|
1897
|
-
"version": "3.0",
|
|
1898
|
-
"engine": "claude",
|
|
1899
|
-
"complexity": "medium",
|
|
1900
|
-
"source": generated_source,
|
|
1901
|
-
"phases": {"surface_close": surface_entry},
|
|
1902
|
-
}),
|
|
1903
|
-
encoding="utf-8",
|
|
1904
|
-
)
|
|
1905
|
-
assert verify_state_contract_violation(devlyn) is None
|
|
1906
|
-
|
|
1907
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1908
|
-
json.dumps({
|
|
1909
|
-
"version": "3.0",
|
|
1910
|
-
"engine": "claude",
|
|
1911
|
-
"complexity": "medium",
|
|
1912
|
-
"source": generated_source,
|
|
1913
|
-
"phases": {
|
|
1914
|
-
"surface_close": {
|
|
1915
|
-
"verdict": None,
|
|
1916
|
-
"started_at": None,
|
|
1917
|
-
"skipped_reason": "claude-unavailable",
|
|
1918
|
-
},
|
|
1919
|
-
},
|
|
1920
|
-
}),
|
|
1921
|
-
encoding="utf-8",
|
|
1922
|
-
)
|
|
1923
|
-
assert verify_state_contract_violation(devlyn)["rule_id"] == "verify.state.surface-close-skipped"
|
|
1924
|
-
|
|
1925
|
-
for state_shape in (
|
|
1926
|
-
{
|
|
1927
|
-
"version": "3.0", "engine": "claude", "complexity": "medium",
|
|
1928
|
-
"source": {"type": "spec"}, "phases": {"surface_close": None},
|
|
1929
|
-
},
|
|
1930
|
-
{
|
|
1931
|
-
"version": "3.0", "engine": "claude", "complexity": "large",
|
|
1932
|
-
"source": generated_source, "phases": {"surface_close": None},
|
|
1933
|
-
},
|
|
1934
|
-
{
|
|
1935
|
-
"version": "2.0", "engine": "claude", "complexity": "medium",
|
|
1936
|
-
"source": {"type": "generated"}, "phases": {"surface_close": None},
|
|
1937
|
-
},
|
|
1938
|
-
):
|
|
1939
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1940
|
-
json.dumps(state_shape), encoding="utf-8",
|
|
1941
|
-
)
|
|
1942
|
-
assert verify_state_contract_violation(devlyn) is None
|
|
1943
|
-
|
|
1944
|
-
# 2026-07-04 field bug (iter-0060 G1): an AUTO pair trigger skipped on
|
|
1945
|
-
# OTHER-engine unavailability spawns no second judge — pair_judge must
|
|
1946
|
-
# be recorded null, never a synthesized PASS.
|
|
1947
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1948
|
-
json.dumps({
|
|
1949
|
-
"version": "3.0",
|
|
1950
|
-
"engine": "claude",
|
|
1951
|
-
"mode": "spec",
|
|
1952
|
-
"risk_profile": {"pair_default_enabled": True},
|
|
1953
|
-
"phases": {
|
|
1954
|
-
"verify": {
|
|
1955
|
-
"verdict": None,
|
|
1956
|
-
"sub_verdicts": None,
|
|
1957
|
-
"pair_trigger": {
|
|
1958
|
-
"eligible": False,
|
|
1959
|
-
"reasons": [],
|
|
1960
|
-
"skipped_reason": "auto_pair_other_engine_unavailable",
|
|
1961
|
-
},
|
|
1962
|
-
}
|
|
1963
|
-
},
|
|
1964
|
-
}),
|
|
1965
|
-
encoding="utf-8",
|
|
1966
|
-
)
|
|
1967
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
1968
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1969
|
-
write_state(devlyn, summary)
|
|
1970
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
1971
|
-
assert summary["verdict"] == "PASS", summary
|
|
1972
|
-
assert summary["source_verdicts"]["pair_judge"] is None, summary
|
|
1973
|
-
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] is None, state
|
|
1974
|
-
|
|
1975
|
-
# A non-null, non-dict sub_verdicts is corrupted state, not a legal
|
|
1976
|
-
# placeholder — write_state() must fail loud, not silently coerce it.
|
|
1977
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1978
|
-
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": None, "sub_verdicts": "corrupt"}}}),
|
|
1979
|
-
encoding="utf-8",
|
|
1980
|
-
)
|
|
1981
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
1982
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1983
|
-
try:
|
|
1984
|
-
write_state(devlyn, summary)
|
|
1985
|
-
except SystemExit as e:
|
|
1986
|
-
assert "sub_verdicts must be null or an object" in str(e), e
|
|
1987
|
-
else:
|
|
1988
|
-
raise AssertionError("write_state() must reject non-dict, non-null sub_verdicts")
|
|
1989
|
-
|
|
1990
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
1991
|
-
json.dumps({
|
|
1992
|
-
"engine": "claude",
|
|
1993
|
-
"phases": {
|
|
1994
|
-
"verify": {
|
|
1995
|
-
"verdict": "PASS",
|
|
1996
|
-
"sub_verdicts": {},
|
|
1997
|
-
"pair_trigger": {
|
|
1998
|
-
"eligible": True,
|
|
1999
|
-
"reasons": ["risk.high", "judge.warning"],
|
|
2000
|
-
"skipped_reason": None,
|
|
2001
|
-
},
|
|
2002
|
-
}
|
|
2003
|
-
}
|
|
2004
|
-
}),
|
|
2005
|
-
encoding="utf-8",
|
|
2006
|
-
)
|
|
2007
|
-
(devlyn / "verify.findings.jsonl").write_text(
|
|
2008
|
-
json.dumps({"id": "j1", "severity": "LOW"}) + "\n",
|
|
2009
|
-
encoding="utf-8",
|
|
2010
|
-
)
|
|
2011
|
-
(devlyn / "verify.pair.findings.jsonl").write_text(
|
|
2012
|
-
json.dumps({"id": "p1", "severity": "HIGH"}) + "\n",
|
|
2013
|
-
encoding="utf-8",
|
|
2014
|
-
)
|
|
2015
|
-
(devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
2016
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2017
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2018
|
-
write_state(devlyn, summary)
|
|
2019
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
2020
|
-
assert summary["verdict"] == "NEEDS_WORK", summary
|
|
2021
|
-
assert state["phases"]["verify"]["verdict"] == "NEEDS_WORK", state
|
|
2022
|
-
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "NEEDS_WORK", state
|
|
2023
|
-
assert (devlyn / "verify-merged.findings.jsonl").read_text(encoding="utf-8")
|
|
2024
|
-
(devlyn / "codex-judge.stdout").unlink()
|
|
2025
|
-
(devlyn / "verify.findings.jsonl").write_text(
|
|
2026
|
-
'{"id":"nan","severity":NaN}\n',
|
|
2027
|
-
encoding="utf-8",
|
|
2028
|
-
)
|
|
2029
|
-
(devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
|
|
2030
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2031
|
-
assert source_verdicts["judge"] == "BLOCKED", source_verdicts
|
|
2032
|
-
assert any(
|
|
2033
|
-
finding.get("id") == "verify-merge-invalid-json-verify.findings.jsonl-1"
|
|
2034
|
-
and "invalid JSON numeric constant: NaN" in finding.get("message", "")
|
|
2035
|
-
for finding in findings
|
|
2036
|
-
), findings
|
|
2037
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2038
|
-
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2039
|
-
encoding="utf-8",
|
|
2040
|
-
)
|
|
2041
|
-
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
2042
|
-
(devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
|
|
2043
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2044
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2045
|
-
write_state(devlyn, summary)
|
|
2046
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
2047
|
-
assert summary["verdict"] == "PASS", summary
|
|
2048
|
-
assert state["phases"]["verify"]["verdict"] == "PASS", state
|
|
2049
|
-
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "PASS", state
|
|
2050
|
-
(devlyn / "codex-judge.stdout").write_text(
|
|
2051
|
-
json.dumps({"id": "cj1", "severity": "HIGH"}) + "\n"
|
|
2052
|
-
+ '# SUMMARY {"verdict":"NEEDS_WORK"}\n',
|
|
2053
|
-
encoding="utf-8",
|
|
2054
|
-
)
|
|
2055
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2056
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2057
|
-
write_state(devlyn, summary)
|
|
2058
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
2059
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2060
|
-
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "BLOCKED", state
|
|
2061
|
-
(devlyn / "codex-judge.stdout").unlink()
|
|
2062
|
-
|
|
2063
|
-
# Engine-neutral stdout contract (iter-0060): a Claude pair-judge
|
|
2064
|
-
# capture (claude-judge.stdout, adapters/claude.md ## Invocation)
|
|
2065
|
-
# binds the same emission contract as the Codex one.
|
|
2066
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2067
|
-
json.dumps({"engine": "codex", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2068
|
-
encoding="utf-8",
|
|
2069
|
-
)
|
|
2070
|
-
(devlyn / "claude-judge.stdout").write_text(
|
|
2071
|
-
json.dumps({"id": "clj1", "severity": "HIGH"}) + "\n",
|
|
2072
|
-
encoding="utf-8",
|
|
2073
|
-
)
|
|
2074
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2075
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2076
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2077
|
-
assert any(
|
|
2078
|
-
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2079
|
-
and "claude-judge.stdout" in finding.get("message", "")
|
|
2080
|
-
for finding in findings
|
|
2081
|
-
), findings
|
|
2082
|
-
(devlyn / "claude-judge.stdout").unlink()
|
|
2083
|
-
|
|
2084
|
-
# iter-0065 case 1: a valid pair timeout marker plus empty pair output
|
|
2085
|
-
# records TIMEOUT and leaves the merged verdict to mechanical+primary.
|
|
2086
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2087
|
-
json.dumps({
|
|
2088
|
-
"engine": "codex",
|
|
2089
|
-
"phases": {
|
|
2090
|
-
"verify": {
|
|
2091
|
-
"verdict": "PASS",
|
|
2092
|
-
"sub_verdicts": {},
|
|
2093
|
-
"pair_trigger": {
|
|
2094
|
-
"eligible": True,
|
|
2095
|
-
"reasons": ["judge.warning"],
|
|
2096
|
-
"skipped_reason": None,
|
|
2097
|
-
},
|
|
2098
|
-
}
|
|
2099
|
-
}
|
|
2100
|
-
}),
|
|
2101
|
-
encoding="utf-8",
|
|
2102
|
-
)
|
|
2103
|
-
(devlyn / "verify.findings.jsonl").write_text(
|
|
2104
|
-
json.dumps({"id": "j-timeout-low", "severity": "LOW"}) + "\n",
|
|
2105
|
-
encoding="utf-8",
|
|
2106
|
-
)
|
|
2107
|
-
(devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
|
|
2108
|
-
(devlyn / "claude-judge.stdout").write_text("", encoding="utf-8")
|
|
2109
|
-
(devlyn / "verify.pair.timeout.json").write_text(
|
|
2110
|
-
json.dumps({"engine": "claude", "budget_seconds": 600}),
|
|
2111
|
-
encoding="utf-8",
|
|
2112
|
-
)
|
|
2113
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2114
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2115
|
-
write_state(devlyn, summary)
|
|
2116
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
2117
|
-
assert summary["verdict"] == "PASS_WITH_ISSUES", summary
|
|
2118
|
-
assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
|
|
2119
|
-
assert summary["pair_timeout"] == {"engine": "claude", "budget_seconds": 600}, summary
|
|
2120
|
-
assert summary["report_header_note"] == "solo verdict after pair TIMEOUT", summary
|
|
2121
|
-
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "TIMEOUT", state
|
|
2122
|
-
(devlyn / "claude-judge.stdout").unlink()
|
|
2123
|
-
(devlyn / "verify.pair.timeout.json").unlink()
|
|
2124
|
-
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
2125
|
-
|
|
2126
|
-
# iter-0065 case 2: canonical pair findings still bind after timeout.
|
|
2127
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2128
|
-
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2129
|
-
encoding="utf-8",
|
|
2130
|
-
)
|
|
2131
|
-
(devlyn / "verify.pair.findings.jsonl").write_text(
|
|
2132
|
-
json.dumps({"id": "p-timeout-high", "severity": "HIGH"}) + "\n",
|
|
2133
|
-
encoding="utf-8",
|
|
2134
|
-
)
|
|
2135
|
-
(devlyn / "verify.pair.timeout.json").write_text(
|
|
2136
|
-
json.dumps({"engine": "codex", "budget_seconds": 600}),
|
|
2137
|
-
encoding="utf-8",
|
|
2138
|
-
)
|
|
2139
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2140
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2141
|
-
assert summary["verdict"] == "NEEDS_WORK", summary
|
|
2142
|
-
assert summary["source_verdicts"]["pair_judge"] == "NEEDS_WORK", summary
|
|
2143
|
-
assert "pair_timeout" not in summary, summary
|
|
2144
|
-
(devlyn / "verify.pair.timeout.json").unlink()
|
|
2145
|
-
(devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
|
|
2146
|
-
|
|
2147
|
-
# iter-0065 case 2b: stdout-only HIGH still blocks on emission contract;
|
|
2148
|
-
# timeout never converts an observed finding into a solo pass.
|
|
2149
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2150
|
-
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2151
|
-
encoding="utf-8",
|
|
2152
|
-
)
|
|
2153
|
-
(devlyn / "codex-judge.stdout").write_text(
|
|
2154
|
-
json.dumps({"id": "cj-timeout-high", "severity": "HIGH"}) + "\n",
|
|
2155
|
-
encoding="utf-8",
|
|
2156
|
-
)
|
|
2157
|
-
(devlyn / "verify.pair.timeout.json").write_text(
|
|
2158
|
-
json.dumps({"engine": "codex", "budget_seconds": 600}),
|
|
2159
|
-
encoding="utf-8",
|
|
2160
|
-
)
|
|
2161
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2162
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2163
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2164
|
-
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2165
|
-
assert any(
|
|
2166
|
-
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2167
|
-
for finding in findings
|
|
2168
|
-
), findings
|
|
2169
|
-
(devlyn / "codex-judge.stdout").unlink()
|
|
2170
|
-
(devlyn / "verify.pair.timeout.json").unlink()
|
|
2171
|
-
|
|
2172
|
-
# iter-0106: a budget abort can truncate the whole-message stream before
|
|
2173
|
-
# its terminal result. With a valid marker that capture stays TIMEOUT.
|
|
2174
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2175
|
-
json.dumps({
|
|
2176
|
-
"engine": "claude",
|
|
2177
|
-
"phases": {
|
|
2178
|
-
"verify": {
|
|
2179
|
-
"verdict": "PASS",
|
|
2180
|
-
"sub_verdicts": {},
|
|
2181
|
-
"pair_trigger": {
|
|
2182
|
-
"eligible": True,
|
|
2183
|
-
"reasons": ["judge.warning"],
|
|
2184
|
-
"skipped_reason": None,
|
|
2185
|
-
},
|
|
2186
|
-
}
|
|
2187
|
-
}
|
|
2188
|
-
}),
|
|
2189
|
-
encoding="utf-8",
|
|
2190
|
-
)
|
|
2191
|
-
partial_stream = (
|
|
2192
|
-
'{"type":"system","subtype":"init","session_id":"s106"}\n'
|
|
2193
|
-
'{"type":"assistant","session_id":"s106","message":{"stop_reason":null,'
|
|
2194
|
-
'"content":[{"type":"text","text":"reading the diff"}]}}\n'
|
|
2195
|
-
)
|
|
2196
|
-
(devlyn / "grok-judge.stdout").write_text(partial_stream, encoding="utf-8")
|
|
2197
|
-
(devlyn / "verify.pair.timeout.json").write_text(
|
|
2198
|
-
json.dumps({"engine": "grok", "budget_seconds": 600}),
|
|
2199
|
-
encoding="utf-8",
|
|
2200
|
-
)
|
|
2201
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2202
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2203
|
-
assert summary["verdict"] == "PASS", summary
|
|
2204
|
-
assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
|
|
2205
|
-
assert summary["pair_timeout"] == {"engine": "grok", "budget_seconds": 600}, summary
|
|
2206
|
-
assert not findings, findings
|
|
2207
|
-
|
|
2208
|
-
# A kill may cut the final JSON record mid-write; one malformed tail is
|
|
2209
|
-
# still a valid prefix, while malformed/unknown complete records are not.
|
|
2210
|
-
(devlyn / "grok-judge.stdout").write_text(
|
|
2211
|
-
partial_stream + '{"type":"assistant","session_id":"s106"',
|
|
2212
|
-
encoding="utf-8",
|
|
2213
|
-
)
|
|
2214
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2215
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2216
|
-
assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
|
|
2217
|
-
assert not findings, findings
|
|
2218
|
-
|
|
2219
|
-
for invalid_prefix in (
|
|
2220
|
-
'{"type":"system","subtype":"init"}\n',
|
|
2221
|
-
partial_stream + "trailing narration\n",
|
|
2222
|
-
partial_stream
|
|
2223
|
-
+ '{"type":"user","session_id":"other"}\n',
|
|
2224
|
-
partial_stream
|
|
2225
|
-
+ '{"type":"stream_event","session_id":"s106"}\n',
|
|
2226
|
-
partial_stream
|
|
2227
|
-
+ '{"type":"system","subtype":"compact_boundary","session_id":"s106"}\n',
|
|
2228
|
-
partial_stream
|
|
2229
|
-
+ '{"type":"assistant"\n'
|
|
2230
|
-
+ '{"type":"user","session_id":"s106"}\n',
|
|
2231
|
-
):
|
|
2232
|
-
(devlyn / "grok-judge.stdout").write_text(invalid_prefix, encoding="utf-8")
|
|
2233
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2234
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2235
|
-
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2236
|
-
assert any(
|
|
2237
|
-
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2238
|
-
for finding in findings
|
|
2239
|
-
), findings
|
|
2240
|
-
|
|
2241
|
-
# iter-0106 control: the marker conserves TIMEOUT only for that shape. A
|
|
2242
|
-
# stream that reached its result but welded narration into the terminal
|
|
2243
|
-
# message stays BLOCKED on the emission contract.
|
|
2244
|
-
(devlyn / "grok-judge.stdout").write_text(
|
|
2245
|
-
'{"type":"system","subtype":"init","session_id":"s106"}\n'
|
|
2246
|
-
+ json.dumps({
|
|
2247
|
-
"type": "assistant",
|
|
2248
|
-
"session_id": "s106",
|
|
2249
|
-
"message": {
|
|
2250
|
-
"stop_reason": "end_turn",
|
|
2251
|
-
"content": [{"type": "text", "text": "I reviewed the diff.\nPASS"}],
|
|
2252
|
-
},
|
|
2253
|
-
}) + "\n"
|
|
2254
|
-
+ json.dumps({
|
|
2255
|
-
"type": "result",
|
|
2256
|
-
"subtype": "success",
|
|
2257
|
-
"is_error": False,
|
|
2258
|
-
"stop_reason": "end_turn",
|
|
2259
|
-
"session_id": "s106",
|
|
2260
|
-
"result": "I reviewed the diff.\nPASS",
|
|
2261
|
-
}) + "\n",
|
|
2262
|
-
encoding="utf-8",
|
|
2263
|
-
)
|
|
2264
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2265
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2266
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2267
|
-
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2268
|
-
assert any(
|
|
2269
|
-
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2270
|
-
for finding in findings
|
|
2271
|
-
), findings
|
|
2272
|
-
(devlyn / "verify.pair.timeout.json").unlink()
|
|
2273
|
-
|
|
2274
|
-
# iter-0106 control: without a marker the same truncated stream is a
|
|
2275
|
-
# plain emission-contract violation.
|
|
2276
|
-
(devlyn / "grok-judge.stdout").write_text(partial_stream, encoding="utf-8")
|
|
2277
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2278
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2279
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2280
|
-
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2281
|
-
assert any(
|
|
2282
|
-
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2283
|
-
for finding in findings
|
|
2284
|
-
), findings
|
|
2285
|
-
(devlyn / "grok-judge.stdout").unlink()
|
|
2286
|
-
|
|
2287
|
-
# iter-0065 case 3: without a marker, required empty pair output remains BLOCKED.
|
|
2288
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2289
|
-
json.dumps({
|
|
2290
|
-
"engine": "codex",
|
|
2291
|
-
"phases": {
|
|
2292
|
-
"verify": {
|
|
2293
|
-
"verdict": "PASS",
|
|
2294
|
-
"sub_verdicts": {},
|
|
2295
|
-
"pair_trigger": {
|
|
2296
|
-
"eligible": True,
|
|
2297
|
-
"reasons": ["risk.high"],
|
|
2298
|
-
"skipped_reason": None,
|
|
2299
|
-
},
|
|
2300
|
-
}
|
|
2301
|
-
}
|
|
2302
|
-
}),
|
|
2303
|
-
encoding="utf-8",
|
|
2304
|
-
)
|
|
2305
|
-
(devlyn / "claude-judge.stdout").write_text("", encoding="utf-8")
|
|
2306
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2307
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2308
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2309
|
-
assert any(
|
|
2310
|
-
finding.get("id") == "verify-pair-empty-output"
|
|
2311
|
-
for finding in findings
|
|
2312
|
-
), findings
|
|
2313
|
-
(devlyn / "claude-judge.stdout").unlink()
|
|
2314
|
-
|
|
2315
|
-
# iter-0065 malformed timeout markers fail closed as a CRITICAL pair blocker.
|
|
2316
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2317
|
-
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2318
|
-
encoding="utf-8",
|
|
2319
|
-
)
|
|
2320
|
-
(devlyn / "verify.pair.timeout.json").write_text(
|
|
2321
|
-
json.dumps({"engine": "claude", "budget_seconds": 0}),
|
|
2322
|
-
encoding="utf-8",
|
|
2323
|
-
)
|
|
2324
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2325
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2326
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2327
|
-
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2328
|
-
assert any(
|
|
2329
|
-
finding.get("id") == "verify-pair-timeout-marker-malformed"
|
|
2330
|
-
and finding.get("severity") == "CRITICAL"
|
|
2331
|
-
for finding in findings
|
|
2332
|
-
), findings
|
|
2333
|
-
(devlyn / "verify.pair.timeout.json").unlink()
|
|
2334
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2335
|
-
json.dumps({
|
|
2336
|
-
"engine": "claude",
|
|
2337
|
-
"phases": {
|
|
2338
|
-
"verify": {
|
|
2339
|
-
"verdict": "PASS",
|
|
2340
|
-
"sub_verdicts": {},
|
|
2341
|
-
"pair_trigger": {
|
|
2342
|
-
"eligible": True,
|
|
2343
|
-
"reasons": ["risk.high"],
|
|
2344
|
-
"skipped_reason": None,
|
|
2345
|
-
},
|
|
2346
|
-
}
|
|
2347
|
-
}
|
|
2348
|
-
}),
|
|
2349
|
-
encoding="utf-8",
|
|
2350
|
-
)
|
|
2351
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2352
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2353
|
-
write_state(devlyn, summary)
|
|
2354
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
2355
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2356
|
-
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "BLOCKED", state
|
|
2357
|
-
assert any(
|
|
2358
|
-
finding.get("id") == "verify-pair-required-output-missing"
|
|
2359
|
-
for finding in findings
|
|
2360
|
-
), findings
|
|
2361
|
-
|
|
2362
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2363
|
-
json.dumps({
|
|
2364
|
-
"engine": "claude",
|
|
2365
|
-
"mode": "spec",
|
|
2366
|
-
"risk_profile": {
|
|
2367
|
-
"high_risk": True,
|
|
2368
|
-
"risk_probes_enabled": True,
|
|
2369
|
-
"pair_default_enabled": True,
|
|
2370
|
-
},
|
|
2371
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2372
|
-
}),
|
|
2373
|
-
encoding="utf-8",
|
|
2374
|
-
)
|
|
2375
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2376
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2377
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2378
|
-
assert any(
|
|
2379
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
2380
|
-
for finding in findings
|
|
2381
|
-
), findings
|
|
2382
|
-
|
|
2383
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2384
|
-
json.dumps({
|
|
2385
|
-
"engine": "claude",
|
|
2386
|
-
"mode": "spec",
|
|
2387
|
-
"risk_profile": "enabled",
|
|
2388
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2389
|
-
}),
|
|
2390
|
-
encoding="utf-8",
|
|
2391
|
-
)
|
|
2392
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2393
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2394
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2395
|
-
assert any(
|
|
2396
|
-
finding.get("id") == "verify-risk-profile-malformed"
|
|
2397
|
-
and "risk_profile must be an object" in str(finding.get("message"))
|
|
2398
|
-
for finding in findings
|
|
2399
|
-
), findings
|
|
2400
|
-
|
|
2401
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2402
|
-
json.dumps({
|
|
2403
|
-
"engine": "claude",
|
|
2404
|
-
"mode": "spec",
|
|
2405
|
-
"risk_profile": {
|
|
2406
|
-
"high_risk": True,
|
|
2407
|
-
"risk_probes_enabled": "true",
|
|
2408
|
-
"pair_default_enabled": True,
|
|
2409
|
-
},
|
|
2410
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2411
|
-
}),
|
|
2412
|
-
encoding="utf-8",
|
|
2413
|
-
)
|
|
2414
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2415
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2416
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2417
|
-
assert any(
|
|
2418
|
-
finding.get("id") == "verify-risk-profile-malformed"
|
|
2419
|
-
and "risk_profile.risk_probes_enabled must be a boolean" in str(finding.get("message"))
|
|
2420
|
-
for finding in findings
|
|
2421
|
-
), findings
|
|
2422
|
-
|
|
2423
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2424
|
-
json.dumps({
|
|
2425
|
-
"engine": "claude",
|
|
2426
|
-
"mode": "spec",
|
|
2427
|
-
"risk_profile": {
|
|
2428
|
-
"high_risk": True,
|
|
2429
|
-
"risk_probes_enabled": False,
|
|
2430
|
-
"pair_default_enabled": True,
|
|
2431
|
-
"reasons": ["explicit", 3],
|
|
2432
|
-
},
|
|
2433
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2434
|
-
}),
|
|
2435
|
-
encoding="utf-8",
|
|
2436
|
-
)
|
|
2437
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2438
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2439
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2440
|
-
assert any(
|
|
2441
|
-
finding.get("id") == "verify-risk-profile-malformed"
|
|
2442
|
-
and "risk_profile.reasons must be a list of strings" in str(finding.get("message"))
|
|
2443
|
-
for finding in findings
|
|
2444
|
-
), findings
|
|
2445
|
-
|
|
2446
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2447
|
-
json.dumps({
|
|
2448
|
-
"engine": "claude",
|
|
2449
|
-
"mode": "spec",
|
|
2450
|
-
"pair_verify": True,
|
|
2451
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2452
|
-
}),
|
|
2453
|
-
encoding="utf-8",
|
|
2454
|
-
)
|
|
2455
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2456
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2457
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2458
|
-
assert any(
|
|
2459
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
2460
|
-
and "mode.pair-verify" in finding.get("message", "")
|
|
2461
|
-
for finding in findings
|
|
2462
|
-
), findings
|
|
2463
|
-
|
|
2464
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2465
|
-
json.dumps({
|
|
2466
|
-
"engine": "claude",
|
|
2467
|
-
"mode": "spec",
|
|
2468
|
-
"complexity": "large",
|
|
2469
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2470
|
-
}),
|
|
2471
|
-
encoding="utf-8",
|
|
2472
|
-
)
|
|
2473
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2474
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2475
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2476
|
-
assert any(
|
|
2477
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
2478
|
-
and "complexity.large" in str(finding.get("message"))
|
|
2479
|
-
for finding in findings
|
|
2480
|
-
), findings
|
|
2481
|
-
|
|
2482
|
-
spec_path = devlyn / "spec.md"
|
|
2483
|
-
spec_path.write_text(
|
|
2484
|
-
'---\nid: "spec-high"\ncomplexity: high\n---\n\n# Spec\n',
|
|
2485
|
-
encoding="utf-8",
|
|
2486
|
-
)
|
|
2487
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2488
|
-
json.dumps({
|
|
2489
|
-
"engine": "claude",
|
|
2490
|
-
"mode": "spec",
|
|
2491
|
-
"source": {"spec_path": str(spec_path)},
|
|
2492
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2493
|
-
}),
|
|
2494
|
-
encoding="utf-8",
|
|
2495
|
-
)
|
|
2496
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2497
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2498
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2499
|
-
assert any(
|
|
2500
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
2501
|
-
and "spec.complexity.high" in str(finding.get("message"))
|
|
2502
|
-
for finding in findings
|
|
2503
|
-
), findings
|
|
2504
|
-
|
|
2505
|
-
spec_path.write_text(
|
|
2506
|
-
'---\nid: "spec-large"\ncomplexity: large\n---\n\n# Spec\n',
|
|
2507
|
-
encoding="utf-8",
|
|
2508
|
-
)
|
|
2509
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2510
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2511
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2512
|
-
assert any(
|
|
2513
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
2514
|
-
and "spec.complexity.large" in str(finding.get("message"))
|
|
2515
|
-
for finding in findings
|
|
2516
|
-
), findings
|
|
2517
|
-
|
|
2518
|
-
spec_path.write_text(
|
|
2519
|
-
"# Spec\n\n## Context\n\nsolo-headroom hypothesis: `SOLO_CLAUDE` should miss the priority rollback behavior; implementation token `rollback`.\n",
|
|
2520
|
-
encoding="utf-8",
|
|
2521
|
-
)
|
|
2522
|
-
assert spec_has_solo_headroom_hypothesis(
|
|
2523
|
-
{"source": {"spec_path": str(spec_path)}}
|
|
2524
|
-
) is False
|
|
2525
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2526
|
-
json.dumps({
|
|
2527
|
-
"engine": "claude",
|
|
2528
|
-
"mode": "spec",
|
|
2529
|
-
"source": {"spec_path": str(spec_path)},
|
|
2530
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2531
|
-
}),
|
|
2532
|
-
encoding="utf-8",
|
|
2533
|
-
)
|
|
2534
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2535
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2536
|
-
assert summary["verdict"] == "PASS", summary
|
|
2537
|
-
assert not any(
|
|
2538
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
2539
|
-
and "spec.solo_headroom_hypothesis" in str(finding.get("message"))
|
|
2540
|
-
for finding in findings
|
|
2541
|
-
), findings
|
|
2542
|
-
|
|
2543
|
-
spec_path.write_text(
|
|
2544
|
-
"# Spec\n\n## Context\n\nsolo-headroom hypothesis: solo_claude should miss the priority rollback behavior.\nObservable command: `node check.js` exposes behavior.\n",
|
|
2545
|
-
encoding="utf-8",
|
|
2546
|
-
)
|
|
2547
|
-
assert spec_has_solo_headroom_hypothesis(
|
|
2548
|
-
{"source": {"spec_path": str(spec_path)}}
|
|
2549
|
-
) is False
|
|
2550
|
-
|
|
2551
|
-
spec_path.write_text(
|
|
2552
|
-
"# Spec\n\n## Context\n\nsolo-headroom hypothesis: `SOLO_CLAUDE` should miss the priority rollback behavior; observable `SOLO_CLAUDE` exposes the miss.\n",
|
|
2553
|
-
encoding="utf-8",
|
|
2554
|
-
)
|
|
2555
|
-
assert spec_has_solo_headroom_hypothesis(
|
|
2556
|
-
{"source": {"spec_path": str(spec_path)}}
|
|
2557
|
-
) is False
|
|
2558
|
-
|
|
2559
|
-
spec_path.write_text(
|
|
2560
|
-
"# Spec\n\n## Context\n\nsolo-headroom hypothesis: solo_claude should miss behavior where observable `priority rollback` exposes the miss.\n",
|
|
2561
|
-
encoding="utf-8",
|
|
2562
|
-
)
|
|
2563
|
-
assert spec_has_solo_headroom_hypothesis(
|
|
2564
|
-
{"source": {"spec_path": str(spec_path)}}
|
|
2565
|
-
) is False
|
|
2566
|
-
|
|
2567
|
-
spec_path.write_text(
|
|
2568
|
-
"# Spec\n\n## Context\n\nsolo-headroom hypothesis: `SOLO_CLAUDE` should miss the priority rollback behavior exposed by `node check.js`.\n",
|
|
2569
|
-
encoding="utf-8",
|
|
2570
|
-
)
|
|
2571
|
-
assert spec_has_solo_headroom_hypothesis(
|
|
2572
|
-
{"source": {"spec_path": str(spec_path)}}
|
|
2573
|
-
) is True
|
|
2574
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2575
|
-
json.dumps({
|
|
2576
|
-
"engine": "claude",
|
|
2577
|
-
"mode": "spec",
|
|
2578
|
-
"source": {"spec_path": str(spec_path)},
|
|
2579
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2580
|
-
}),
|
|
2581
|
-
encoding="utf-8",
|
|
2582
|
-
)
|
|
2583
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2584
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2585
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2586
|
-
assert any(
|
|
2587
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
2588
|
-
and "spec.solo_headroom_hypothesis" in str(finding.get("message"))
|
|
2589
|
-
for finding in findings
|
|
2590
|
-
), findings
|
|
2591
|
-
|
|
2592
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2593
|
-
json.dumps({
|
|
2594
|
-
"engine": "claude",
|
|
2595
|
-
"mode": "spec",
|
|
2596
|
-
"source": {"spec_path": str(spec_path)},
|
|
2597
|
-
"risk_profile": {
|
|
2598
|
-
"high_risk": True,
|
|
2599
|
-
"risk_probes_enabled": False,
|
|
2600
|
-
"pair_default_enabled": True,
|
|
2601
|
-
},
|
|
2602
|
-
"phases": {
|
|
2603
|
-
"verify": {
|
|
2604
|
-
"verdict": "PASS",
|
|
2605
|
-
"sub_verdicts": {},
|
|
2606
|
-
"pair_trigger": {
|
|
2607
|
-
"eligible": True,
|
|
2608
|
-
"reasons": ["risk.high"],
|
|
2609
|
-
"skipped_reason": None,
|
|
2610
|
-
},
|
|
2611
|
-
}
|
|
2612
|
-
},
|
|
2613
|
-
}),
|
|
2614
|
-
encoding="utf-8",
|
|
2615
|
-
)
|
|
2616
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2617
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2618
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2619
|
-
assert any(
|
|
2620
|
-
finding.get("id") == "verify-pair-trigger-reasons-incomplete"
|
|
2621
|
-
and "spec.solo_headroom_hypothesis" in str(finding.get("message"))
|
|
2622
|
-
for finding in findings
|
|
2623
|
-
), findings
|
|
2624
|
-
|
|
2625
|
-
criteria_path = devlyn / "criteria.generated.md"
|
|
2626
|
-
criteria_path.write_text(
|
|
2627
|
-
"# Criteria\n\nsolo-headroom hypothesis: `SOLO_CLAUDE` should miss the priority rollback behavior exposed by `node check.js`.\n",
|
|
2628
|
-
encoding="utf-8",
|
|
2629
|
-
)
|
|
2630
|
-
assert spec_has_solo_headroom_hypothesis(
|
|
2631
|
-
{"source": {"criteria_path": str(criteria_path)}}
|
|
2632
|
-
) is True
|
|
2633
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2634
|
-
json.dumps({
|
|
2635
|
-
"engine": "claude",
|
|
2636
|
-
"mode": "free-form",
|
|
2637
|
-
"source": {"criteria_path": str(criteria_path)},
|
|
2638
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2639
|
-
}),
|
|
2640
|
-
encoding="utf-8",
|
|
2641
|
-
)
|
|
2642
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2643
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2644
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2645
|
-
assert any(
|
|
2646
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
2647
|
-
and "spec.solo_headroom_hypothesis" in str(finding.get("message"))
|
|
2648
|
-
for finding in findings
|
|
2649
|
-
), findings
|
|
2650
|
-
|
|
2651
|
-
(devlyn / "verify-mechanical.findings.jsonl").write_text(
|
|
2652
|
-
json.dumps({"id": "m0", "severity": "HIGH"}) + "\n",
|
|
2653
|
-
encoding="utf-8",
|
|
2654
|
-
)
|
|
2655
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2656
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2657
|
-
assert summary["verdict"] == "NEEDS_WORK", summary
|
|
2658
|
-
assert not any(
|
|
2659
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
2660
|
-
for finding in findings
|
|
2661
|
-
), findings
|
|
2662
|
-
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
2663
|
-
|
|
2664
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2665
|
-
json.dumps({
|
|
2666
|
-
"engine": "claude",
|
|
2667
|
-
"phases": {
|
|
2668
|
-
"verify": {
|
|
2669
|
-
"verdict": "PASS",
|
|
2670
|
-
"sub_verdicts": {},
|
|
2671
|
-
"pair_trigger": {
|
|
2672
|
-
"eligible": "true",
|
|
2673
|
-
"reasons": ["risk.high"],
|
|
2674
|
-
"skipped_reason": None,
|
|
2675
|
-
},
|
|
2676
|
-
}
|
|
2677
|
-
}
|
|
2678
|
-
}),
|
|
2679
|
-
encoding="utf-8",
|
|
2680
|
-
)
|
|
2681
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2682
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2683
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2684
|
-
assert any(
|
|
2685
|
-
finding.get("id") == "verify-pair-trigger-eligible-malformed"
|
|
2686
|
-
for finding in findings
|
|
2687
|
-
), findings
|
|
2688
|
-
|
|
2689
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2690
|
-
json.dumps({
|
|
2691
|
-
"engine": "claude",
|
|
2692
|
-
"phases": {
|
|
2693
|
-
"verify": {
|
|
2694
|
-
"verdict": "PASS",
|
|
2695
|
-
"sub_verdicts": {},
|
|
2696
|
-
"pair_trigger": {
|
|
2697
|
-
"eligible": True,
|
|
2698
|
-
"reasons": "risk.high",
|
|
2699
|
-
"skipped_reason": None,
|
|
2700
|
-
},
|
|
2701
|
-
}
|
|
2702
|
-
}
|
|
2703
|
-
}),
|
|
2704
|
-
encoding="utf-8",
|
|
2705
|
-
)
|
|
2706
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2707
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2708
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2709
|
-
assert any(
|
|
2710
|
-
finding.get("id") == "verify-pair-trigger-reasons-malformed"
|
|
2711
|
-
for finding in findings
|
|
2712
|
-
), findings
|
|
2713
|
-
|
|
2714
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2715
|
-
json.dumps({
|
|
2716
|
-
"engine": "claude",
|
|
2717
|
-
"phases": {
|
|
2718
|
-
"verify": {
|
|
2719
|
-
"verdict": "PASS",
|
|
2720
|
-
"sub_verdicts": {},
|
|
2721
|
-
"pair_trigger": {
|
|
2722
|
-
"eligible": True,
|
|
2723
|
-
"reasons": ["risk.high", "looks-hard"],
|
|
2724
|
-
"skipped_reason": None,
|
|
2725
|
-
},
|
|
2726
|
-
}
|
|
2727
|
-
}
|
|
2728
|
-
}),
|
|
2729
|
-
encoding="utf-8",
|
|
2730
|
-
)
|
|
2731
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2732
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2733
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2734
|
-
assert any(
|
|
2735
|
-
finding.get("id") == "verify-pair-trigger-reasons-unknown"
|
|
2736
|
-
and "only include known" in finding.get("message", "")
|
|
2737
|
-
for finding in findings
|
|
2738
|
-
), findings
|
|
2739
|
-
|
|
2740
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2741
|
-
json.dumps({
|
|
2742
|
-
"engine": "claude",
|
|
2743
|
-
"phases": {
|
|
2744
|
-
"verify": {
|
|
2745
|
-
"verdict": "PASS",
|
|
2746
|
-
"sub_verdicts": {},
|
|
2747
|
-
"pair_trigger": {
|
|
2748
|
-
"eligible": True,
|
|
2749
|
-
"reasons": ["risk high"],
|
|
2750
|
-
"skipped_reason": None,
|
|
2751
|
-
},
|
|
2752
|
-
}
|
|
2753
|
-
}
|
|
2754
|
-
}),
|
|
2755
|
-
encoding="utf-8",
|
|
2756
|
-
)
|
|
2757
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2758
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2759
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2760
|
-
assert any(
|
|
2761
|
-
finding.get("id") == "verify-pair-trigger-reasons-unknown"
|
|
2762
|
-
and "include a known" in finding.get("message", "")
|
|
2763
|
-
for finding in findings
|
|
2764
|
-
), findings
|
|
2765
|
-
|
|
2766
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2767
|
-
json.dumps({
|
|
2768
|
-
"engine": "claude",
|
|
2769
|
-
"phases": {
|
|
2770
|
-
"verify": {
|
|
2771
|
-
"verdict": "PASS",
|
|
2772
|
-
"sub_verdicts": {},
|
|
2773
|
-
"pair_trigger": {
|
|
2774
|
-
"eligible": True,
|
|
2775
|
-
"reasons": ["risk_profile.high_risk", "risk_probes_enabled"],
|
|
2776
|
-
"skipped_reason": None,
|
|
2777
|
-
},
|
|
2778
|
-
}
|
|
2779
|
-
}
|
|
2780
|
-
}),
|
|
2781
|
-
encoding="utf-8",
|
|
2782
|
-
)
|
|
2783
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2784
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2785
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2786
|
-
assert any(
|
|
2787
|
-
finding.get("id") == "verify-pair-trigger-reasons-unknown"
|
|
2788
|
-
and "include a known" in finding.get("message", "")
|
|
2789
|
-
for finding in findings
|
|
2790
|
-
), findings
|
|
2791
|
-
|
|
2792
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2793
|
-
json.dumps({
|
|
2794
|
-
"engine": "claude",
|
|
2795
|
-
"phases": {
|
|
2796
|
-
"verify": {
|
|
2797
|
-
"verdict": "PASS",
|
|
2798
|
-
"sub_verdicts": {},
|
|
2799
|
-
"pair_trigger": {
|
|
2800
|
-
"eligible": True,
|
|
2801
|
-
"reasons": ["risk.high", 3],
|
|
2802
|
-
"skipped_reason": None,
|
|
2803
|
-
},
|
|
2804
|
-
}
|
|
2805
|
-
}
|
|
2806
|
-
}),
|
|
2807
|
-
encoding="utf-8",
|
|
2808
|
-
)
|
|
2809
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2810
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2811
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2812
|
-
assert any(
|
|
2813
|
-
finding.get("id") == "verify-pair-trigger-reasons-malformed"
|
|
2814
|
-
for finding in findings
|
|
2815
|
-
), findings
|
|
2816
|
-
|
|
2817
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2818
|
-
json.dumps({
|
|
2819
|
-
"engine": "claude",
|
|
2820
|
-
"phases": {
|
|
2821
|
-
"verify": {
|
|
2822
|
-
"verdict": "PASS",
|
|
2823
|
-
"sub_verdicts": {},
|
|
2824
|
-
"pair_trigger": {
|
|
2825
|
-
"eligible": True,
|
|
2826
|
-
"reasons": [],
|
|
2827
|
-
"skipped_reason": None,
|
|
2828
|
-
},
|
|
2829
|
-
}
|
|
2830
|
-
}
|
|
2831
|
-
}),
|
|
2832
|
-
encoding="utf-8",
|
|
2833
|
-
)
|
|
2834
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2835
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2836
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2837
|
-
assert any(
|
|
2838
|
-
finding.get("id") == "verify-pair-trigger-reasons-empty"
|
|
2839
|
-
for finding in findings
|
|
2840
|
-
), findings
|
|
2841
|
-
|
|
2842
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2843
|
-
json.dumps({
|
|
2844
|
-
"engine": "claude",
|
|
2845
|
-
"phases": {
|
|
2846
|
-
"verify": {
|
|
2847
|
-
"verdict": "PASS",
|
|
2848
|
-
"sub_verdicts": {},
|
|
2849
|
-
"pair_trigger": {
|
|
2850
|
-
"eligible": True,
|
|
2851
|
-
"reasons": ["risk.high"],
|
|
2852
|
-
"skipped_reason": "user_no_pair",
|
|
2853
|
-
},
|
|
2854
|
-
}
|
|
2855
|
-
}
|
|
2856
|
-
}),
|
|
2857
|
-
encoding="utf-8",
|
|
2858
|
-
)
|
|
2859
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2860
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2861
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2862
|
-
assert any(
|
|
2863
|
-
finding.get("id") == "verify-pair-trigger-skip-contradiction"
|
|
2864
|
-
for finding in findings
|
|
2865
|
-
), findings
|
|
2866
|
-
|
|
2867
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2868
|
-
json.dumps({
|
|
2869
|
-
"engine": "claude",
|
|
2870
|
-
"phases": {
|
|
2871
|
-
"verify": {
|
|
2872
|
-
"verdict": "PASS",
|
|
2873
|
-
"sub_verdicts": {},
|
|
2874
|
-
"pair_trigger": {
|
|
2875
|
-
"eligible": False,
|
|
2876
|
-
"reasons": ["risk.high"],
|
|
2877
|
-
"skipped_reason": "user_no_pair",
|
|
2878
|
-
},
|
|
2879
|
-
}
|
|
2880
|
-
}
|
|
2881
|
-
}),
|
|
2882
|
-
encoding="utf-8",
|
|
2883
|
-
)
|
|
2884
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2885
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2886
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2887
|
-
assert any(
|
|
2888
|
-
finding.get("id") == "verify-pair-trigger-ineligible-reasons"
|
|
2889
|
-
for finding in findings
|
|
2890
|
-
), findings
|
|
2891
|
-
|
|
2892
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2893
|
-
json.dumps({
|
|
2894
|
-
"engine": "claude",
|
|
2895
|
-
"mode": "spec",
|
|
2896
|
-
"risk_profile": {
|
|
2897
|
-
"high_risk": True,
|
|
2898
|
-
"risk_probes_enabled": False,
|
|
2899
|
-
"pair_default_enabled": True,
|
|
2900
|
-
},
|
|
2901
|
-
"phases": {
|
|
2902
|
-
"verify": {
|
|
2903
|
-
"verdict": "PASS",
|
|
2904
|
-
"sub_verdicts": {},
|
|
2905
|
-
"pair_trigger": {
|
|
2906
|
-
"eligible": False,
|
|
2907
|
-
"reasons": [],
|
|
2908
|
-
"skipped_reason": None,
|
|
2909
|
-
},
|
|
2910
|
-
}
|
|
2911
|
-
},
|
|
2912
|
-
}),
|
|
2913
|
-
encoding="utf-8",
|
|
2914
|
-
)
|
|
2915
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2916
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2917
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2918
|
-
assert any(
|
|
2919
|
-
finding.get("id") == "verify-pair-trigger-ineligible-unjustified"
|
|
2920
|
-
and "risk.high" in str(finding.get("message"))
|
|
2921
|
-
for finding in findings
|
|
2922
|
-
), findings
|
|
2923
|
-
|
|
2924
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2925
|
-
json.dumps({
|
|
2926
|
-
"engine": "claude",
|
|
2927
|
-
"mode": "spec",
|
|
2928
|
-
"risk_profile": {
|
|
2929
|
-
"high_risk": True,
|
|
2930
|
-
"risk_probes_enabled": True,
|
|
2931
|
-
"pair_default_enabled": True,
|
|
2932
|
-
},
|
|
2933
|
-
"phases": {
|
|
2934
|
-
"verify": {
|
|
2935
|
-
"verdict": "PASS",
|
|
2936
|
-
"sub_verdicts": {},
|
|
2937
|
-
"pair_trigger": {
|
|
2938
|
-
"eligible": False,
|
|
2939
|
-
"reasons": [],
|
|
2940
|
-
"skipped_reason": "user_no_pair",
|
|
2941
|
-
},
|
|
2942
|
-
}
|
|
2943
|
-
},
|
|
2944
|
-
}),
|
|
2945
|
-
encoding="utf-8",
|
|
2946
|
-
)
|
|
2947
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2948
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2949
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2950
|
-
assert any(
|
|
2951
|
-
finding.get("id") == "verify-pair-trigger-user-no-pair-unsupported"
|
|
2952
|
-
and "pair_default_enabled false" in str(finding.get("message"))
|
|
2953
|
-
for finding in findings
|
|
2954
|
-
), findings
|
|
2955
|
-
|
|
2956
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2957
|
-
json.dumps({
|
|
2958
|
-
"engine": "claude",
|
|
2959
|
-
"pair_verify": True,
|
|
2960
|
-
"risk_profile": {
|
|
2961
|
-
"high_risk": True,
|
|
2962
|
-
"risk_probes_enabled": False,
|
|
2963
|
-
"pair_default_enabled": False,
|
|
2964
|
-
},
|
|
2965
|
-
"phases": {
|
|
2966
|
-
"verify": {
|
|
2967
|
-
"verdict": "PASS",
|
|
2968
|
-
"sub_verdicts": {},
|
|
2969
|
-
"pair_trigger": {
|
|
2970
|
-
"eligible": False,
|
|
2971
|
-
"reasons": [],
|
|
2972
|
-
"skipped_reason": "user_no_pair",
|
|
2973
|
-
},
|
|
2974
|
-
}
|
|
2975
|
-
},
|
|
2976
|
-
}),
|
|
2977
|
-
encoding="utf-8",
|
|
2978
|
-
)
|
|
2979
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
2980
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2981
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
2982
|
-
assert any(
|
|
2983
|
-
finding.get("id") == "verify-pair-trigger-conflicting-pair-flags"
|
|
2984
|
-
for finding in findings
|
|
2985
|
-
), findings
|
|
2986
|
-
|
|
2987
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
2988
|
-
json.dumps({
|
|
2989
|
-
"engine": "claude",
|
|
2990
|
-
"mode": "spec",
|
|
2991
|
-
"risk_profile": {
|
|
2992
|
-
"high_risk": True,
|
|
2993
|
-
"risk_probes_enabled": True,
|
|
2994
|
-
"pair_default_enabled": False,
|
|
2995
|
-
},
|
|
2996
|
-
"phases": {
|
|
2997
|
-
"verify": {
|
|
2998
|
-
"verdict": "PASS",
|
|
2999
|
-
"sub_verdicts": {},
|
|
3000
|
-
"pair_trigger": {
|
|
3001
|
-
"eligible": False,
|
|
3002
|
-
"reasons": [],
|
|
3003
|
-
"skipped_reason": "user_no_pair",
|
|
3004
|
-
},
|
|
3005
|
-
}
|
|
3006
|
-
},
|
|
3007
|
-
}),
|
|
3008
|
-
encoding="utf-8",
|
|
3009
|
-
)
|
|
3010
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3011
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3012
|
-
assert summary["verdict"] == "PASS", summary
|
|
3013
|
-
assert not any(
|
|
3014
|
-
finding.get("id") == "verify-pair-trigger-required-missing"
|
|
3015
|
-
for finding in findings
|
|
3016
|
-
), findings
|
|
3017
|
-
|
|
3018
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3019
|
-
json.dumps({
|
|
3020
|
-
"engine": "claude",
|
|
3021
|
-
"phases": {
|
|
3022
|
-
"verify": {
|
|
3023
|
-
"verdict": "PASS",
|
|
3024
|
-
"sub_verdicts": {},
|
|
3025
|
-
"pair_trigger": {
|
|
3026
|
-
"eligible": False,
|
|
3027
|
-
"reasons": [],
|
|
3028
|
-
"skipped_reason": ["user_no_pair"],
|
|
3029
|
-
},
|
|
3030
|
-
}
|
|
3031
|
-
}
|
|
3032
|
-
}),
|
|
3033
|
-
encoding="utf-8",
|
|
3034
|
-
)
|
|
3035
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3036
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3037
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
3038
|
-
assert any(
|
|
3039
|
-
finding.get("id") == "verify-pair-trigger-skipped-reason-malformed"
|
|
3040
|
-
for finding in findings
|
|
3041
|
-
), findings
|
|
3042
|
-
|
|
3043
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3044
|
-
json.dumps({
|
|
3045
|
-
"engine": "claude",
|
|
3046
|
-
"phases": {
|
|
3047
|
-
"verify": {
|
|
3048
|
-
"verdict": "PASS",
|
|
3049
|
-
"sub_verdicts": {},
|
|
3050
|
-
"pair_trigger": {
|
|
3051
|
-
"eligible": False,
|
|
3052
|
-
"reasons": [],
|
|
3053
|
-
"skipped_reason": "codex_unavailable",
|
|
3054
|
-
},
|
|
3055
|
-
}
|
|
3056
|
-
}
|
|
3057
|
-
}),
|
|
3058
|
-
encoding="utf-8",
|
|
3059
|
-
)
|
|
3060
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3061
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3062
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
3063
|
-
assert any(
|
|
3064
|
-
finding.get("id") == "verify-pair-trigger-skipped-reason-unsupported"
|
|
3065
|
-
for finding in findings
|
|
3066
|
-
), findings
|
|
3067
|
-
|
|
3068
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3069
|
-
json.dumps({
|
|
3070
|
-
"engine": "claude",
|
|
3071
|
-
"phases": {
|
|
3072
|
-
"verify": {
|
|
3073
|
-
"verdict": "PASS",
|
|
3074
|
-
"sub_verdicts": {},
|
|
3075
|
-
"pair_trigger": {
|
|
3076
|
-
"eligible": False,
|
|
3077
|
-
"reasons": [],
|
|
3078
|
-
"skipped_reason": "mechanical_blocker",
|
|
3079
|
-
},
|
|
3080
|
-
}
|
|
3081
|
-
}
|
|
3082
|
-
}),
|
|
3083
|
-
encoding="utf-8",
|
|
3084
|
-
)
|
|
3085
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3086
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3087
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
3088
|
-
assert any(
|
|
3089
|
-
finding.get("id") == "verify-pair-trigger-mechanical-blocker-unsupported"
|
|
3090
|
-
for finding in findings
|
|
3091
|
-
), findings
|
|
3092
|
-
|
|
3093
|
-
(devlyn / "verify-mechanical.findings.jsonl").write_text(
|
|
3094
|
-
json.dumps({"id": "m1", "severity": "HIGH"}) + "\n",
|
|
3095
|
-
encoding="utf-8",
|
|
3096
|
-
)
|
|
3097
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3098
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3099
|
-
assert summary["verdict"] == "NEEDS_WORK", summary
|
|
3100
|
-
assert not any(
|
|
3101
|
-
finding.get("id") == "verify-pair-trigger-mechanical-blocker-unsupported"
|
|
3102
|
-
for finding in findings
|
|
3103
|
-
), findings
|
|
3104
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3105
|
-
json.dumps({
|
|
3106
|
-
"engine": "claude",
|
|
3107
|
-
"phases": {
|
|
3108
|
-
"verify": {
|
|
3109
|
-
"verdict": None,
|
|
3110
|
-
"sub_verdicts": None,
|
|
3111
|
-
"judge_durations_ms": {"judge": None, "pair_judge": None},
|
|
3112
|
-
}
|
|
3113
|
-
}
|
|
3114
|
-
}),
|
|
3115
|
-
encoding="utf-8",
|
|
3116
|
-
)
|
|
3117
|
-
(devlyn / "verify.findings.jsonl").unlink()
|
|
3118
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3119
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3120
|
-
write_state(devlyn, summary)
|
|
3121
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
3122
|
-
assert summary["verdict"] == "NEEDS_WORK", summary
|
|
3123
|
-
assert summary["source_verdicts"]["judge"] is None, summary
|
|
3124
|
-
assert state["phases"]["verify"]["verdict"] == "NEEDS_WORK", state
|
|
3125
|
-
assert state["phases"]["verify"]["sub_verdicts"]["judge"] is None, state
|
|
3126
|
-
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
3127
|
-
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
3128
|
-
|
|
3129
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3130
|
-
json.dumps({
|
|
3131
|
-
"engine": "claude",
|
|
3132
|
-
"phases": {
|
|
3133
|
-
"verify": {
|
|
3134
|
-
"verdict": "PASS",
|
|
3135
|
-
"sub_verdicts": {},
|
|
3136
|
-
"pair_trigger": {
|
|
3137
|
-
"eligible": False,
|
|
3138
|
-
"reasons": [],
|
|
3139
|
-
"skipped_reason": "primary_judge_blocker",
|
|
3140
|
-
},
|
|
3141
|
-
}
|
|
3142
|
-
}
|
|
3143
|
-
}),
|
|
3144
|
-
encoding="utf-8",
|
|
3145
|
-
)
|
|
3146
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3147
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3148
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
3149
|
-
assert any(
|
|
3150
|
-
finding.get("id") == "verify-pair-trigger-primary-judge-blocker-unsupported"
|
|
3151
|
-
for finding in findings
|
|
3152
|
-
), findings
|
|
3153
|
-
|
|
3154
|
-
# Self-test: preknown_primary_blocker_requires_pair.
|
|
3155
|
-
(devlyn / "verify.pair.findings.jsonl").unlink(missing_ok=True)
|
|
3156
|
-
(devlyn / "verify.findings.jsonl").write_text(
|
|
3157
|
-
json.dumps({"id": "j-preknown", "severity": "HIGH"}) + "\n",
|
|
3158
|
-
encoding="utf-8",
|
|
3159
|
-
)
|
|
3160
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3161
|
-
json.dumps({
|
|
3162
|
-
"version": "2.0",
|
|
3163
|
-
"engine": "claude",
|
|
3164
|
-
"mode": "spec",
|
|
3165
|
-
"pair_verify": True,
|
|
3166
|
-
"phases": {
|
|
3167
|
-
"verify": {
|
|
3168
|
-
"verdict": "PASS",
|
|
3169
|
-
"sub_verdicts": {},
|
|
3170
|
-
"pair_trigger": {
|
|
3171
|
-
"eligible": False,
|
|
3172
|
-
"reasons": [],
|
|
3173
|
-
"skipped_reason": "primary_judge_blocker",
|
|
3174
|
-
},
|
|
3175
|
-
}
|
|
3176
|
-
},
|
|
3177
|
-
}),
|
|
3178
|
-
encoding="utf-8",
|
|
3179
|
-
)
|
|
3180
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3181
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3182
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
3183
|
-
assert any(
|
|
3184
|
-
finding.get("id") == "verify-pair-trigger-primary-judge-blocker-preknown"
|
|
3185
|
-
and "mode.pair-verify" in str(finding.get("message"))
|
|
3186
|
-
for finding in findings
|
|
3187
|
-
), findings
|
|
3188
|
-
|
|
3189
|
-
# Self-test: schema-v3 default pair reason is required before merge.
|
|
3190
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3191
|
-
json.dumps({
|
|
3192
|
-
"version": "3.0",
|
|
3193
|
-
"engine": "claude",
|
|
3194
|
-
"mode": "spec",
|
|
3195
|
-
"pair_verify": True,
|
|
3196
|
-
"phases": {
|
|
3197
|
-
"verify": {
|
|
3198
|
-
"verdict": "PASS",
|
|
3199
|
-
"sub_verdicts": {},
|
|
3200
|
-
"pair_trigger": {
|
|
3201
|
-
"eligible": True,
|
|
3202
|
-
"reasons": ["mode.pair-verify"],
|
|
3203
|
-
"skipped_reason": None,
|
|
3204
|
-
},
|
|
3205
|
-
}
|
|
3206
|
-
},
|
|
3207
|
-
}),
|
|
3208
|
-
encoding="utf-8",
|
|
3209
|
-
)
|
|
3210
|
-
(devlyn / "verify.pair.findings.jsonl").write_text(
|
|
3211
|
-
json.dumps({"id": "p-preknown", "severity": "LOW"}) + "\n",
|
|
3212
|
-
encoding="utf-8",
|
|
3213
|
-
)
|
|
3214
|
-
(devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
3215
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3216
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3217
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
3218
|
-
assert any(
|
|
3219
|
-
finding.get("id") == "verify-pair-trigger-reasons-incomplete"
|
|
3220
|
-
and "pair.default" in str(finding.get("message"))
|
|
3221
|
-
for finding in findings
|
|
3222
|
-
), findings
|
|
3223
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
3224
|
-
state["phases"]["verify"]["pair_trigger"]["reasons"].insert(0, "pair.default")
|
|
3225
|
-
(devlyn / "pipeline.state.json").write_text(json.dumps(state), encoding="utf-8")
|
|
3226
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3227
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3228
|
-
assert summary["verdict"] == "NEEDS_WORK", summary
|
|
3229
|
-
assert summary["source_verdicts"]["judge"] == "NEEDS_WORK", summary
|
|
3230
|
-
assert summary["source_verdicts"]["pair_judge"] == "PASS_WITH_ISSUES", summary
|
|
3231
|
-
assert '"id":"p-preknown"' in (
|
|
3232
|
-
devlyn / "verify-merged.findings.jsonl"
|
|
3233
|
-
).read_text(encoding="utf-8"), findings
|
|
3234
|
-
(devlyn / "codex-judge.stdout").unlink()
|
|
3235
|
-
|
|
3236
|
-
# Self-test: archived-v2 sequential primary blocker remains legal.
|
|
3237
|
-
(devlyn / "verify.pair.findings.jsonl").unlink()
|
|
3238
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3239
|
-
json.dumps({
|
|
3240
|
-
"version": "2.0",
|
|
3241
|
-
"engine": "claude",
|
|
3242
|
-
"mode": "spec",
|
|
3243
|
-
"phases": {
|
|
3244
|
-
"verify": {
|
|
3245
|
-
"verdict": "PASS",
|
|
3246
|
-
"sub_verdicts": {},
|
|
3247
|
-
"pair_trigger": {
|
|
3248
|
-
"eligible": False,
|
|
3249
|
-
"reasons": [],
|
|
3250
|
-
"skipped_reason": "primary_judge_blocker",
|
|
3251
|
-
},
|
|
3252
|
-
}
|
|
3253
|
-
},
|
|
3254
|
-
}),
|
|
3255
|
-
encoding="utf-8",
|
|
3256
|
-
)
|
|
3257
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3258
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3259
|
-
assert summary["verdict"] == "NEEDS_WORK", summary
|
|
3260
|
-
assert summary["source_verdicts"]["pair_judge"] is None, summary
|
|
3261
|
-
assert not any(
|
|
3262
|
-
finding.get("id") in {
|
|
3263
|
-
"verify-pair-trigger-primary-judge-blocker-unsupported",
|
|
3264
|
-
"verify-pair-trigger-primary-judge-blocker-preknown",
|
|
3265
|
-
}
|
|
3266
|
-
for finding in findings
|
|
3267
|
-
), findings
|
|
3268
|
-
# Replay the same state as schema v3 and require the retired-skip blocker.
|
|
3269
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
3270
|
-
state["version"] = "3.0"
|
|
3271
|
-
(devlyn / "pipeline.state.json").write_text(json.dumps(state), encoding="utf-8")
|
|
3272
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3273
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3274
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
3275
|
-
assert any(
|
|
3276
|
-
finding.get("id") == "verify-pair-trigger-primary-judge-blocker-retired"
|
|
3277
|
-
for finding in findings
|
|
3278
|
-
), findings
|
|
3279
|
-
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
3280
|
-
(devlyn / "verify.pair.findings.jsonl").write_text("", encoding="utf-8")
|
|
3281
|
-
|
|
3282
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3283
|
-
json.dumps({
|
|
3284
|
-
"engine": "claude",
|
|
3285
|
-
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
3286
|
-
"verify": {
|
|
3287
|
-
"pair_trigger": {
|
|
3288
|
-
"eligible": True,
|
|
3289
|
-
"reasons": ["looks-hard"],
|
|
3290
|
-
"skipped_reason": None,
|
|
3291
|
-
}
|
|
3292
|
-
},
|
|
3293
|
-
}),
|
|
3294
|
-
encoding="utf-8",
|
|
3295
|
-
)
|
|
3296
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3297
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3298
|
-
assert summary["verdict"] == "BLOCKED", summary
|
|
3299
|
-
assert any(
|
|
3300
|
-
finding.get("id") == "verify-pair-trigger-reasons-unknown"
|
|
3301
|
-
for finding in findings
|
|
3302
|
-
), findings
|
|
3303
|
-
|
|
3304
|
-
# stdout-only spawn evidence: no pair findings file at all; a clean
|
|
3305
|
-
# PASS stdout from a claude judge promotes pair_judge null -> PASS.
|
|
3306
|
-
(devlyn / "verify.pair.findings.jsonl").unlink()
|
|
3307
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3308
|
-
json.dumps({"engine": "codex", "phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
|
|
3309
|
-
encoding="utf-8",
|
|
3310
|
-
)
|
|
3311
|
-
(devlyn / "claude-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
3312
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3313
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3314
|
-
write_state(devlyn, summary)
|
|
3315
|
-
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
3316
|
-
assert summary["verdict"] == "PASS", summary
|
|
3317
|
-
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "PASS", state
|
|
3318
|
-
(devlyn / "claude-judge.stdout").unlink()
|
|
3319
|
-
|
|
3320
|
-
# The primary capture must not be misattributed to the OTHER-engine
|
|
3321
|
-
# pair seat merely because both adapters use *-judge.stdout names.
|
|
3322
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3323
|
-
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
|
|
3324
|
-
encoding="utf-8",
|
|
3325
|
-
)
|
|
3326
|
-
(devlyn / "claude-judge.stdout").write_text("primary prose\n", encoding="utf-8")
|
|
3327
|
-
(devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
3328
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3329
|
-
assert source_verdicts["pair_judge"] == "PASS", source_verdicts
|
|
3330
|
-
assert not any(finding["source"] == "pair_judge" for finding in findings), findings
|
|
3331
|
-
(devlyn / "codex-judge.stdout").write_text("pair prose\n", encoding="utf-8")
|
|
3332
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3333
|
-
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3334
|
-
assert any(finding["id"] == "verify-pair-emission-contract-violated" for finding in findings)
|
|
3335
|
-
(devlyn / "claude-judge.stdout").unlink()
|
|
3336
|
-
(devlyn / "codex-judge.stdout").unlink()
|
|
3337
|
-
|
|
3338
|
-
# A primary capture cannot become pair evidence when the state needed
|
|
3339
|
-
# to identify the primary seat is absent, incomplete, or malformed.
|
|
3340
|
-
(devlyn / "claude-judge.stdout").write_text("primary prose\n", encoding="utf-8")
|
|
3341
|
-
(devlyn / "pipeline.state.json").unlink()
|
|
3342
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3343
|
-
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3344
|
-
assert any(finding["id"] == "verify-state-missing" for finding in findings), findings
|
|
3345
|
-
assert not any(finding.get("file") == "claude-judge.stdout" for finding in findings), findings
|
|
3346
|
-
|
|
3347
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3348
|
-
json.dumps({"version": "3.0", "phases": {"verify": {}}}), encoding="utf-8",
|
|
3349
|
-
)
|
|
3350
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3351
|
-
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3352
|
-
assert any(finding.get("rule_id") == "verify.state.engine-malformed" for finding in findings), findings
|
|
3353
|
-
|
|
3354
|
-
(devlyn / "pipeline.state.json").write_text("{", encoding="utf-8")
|
|
3355
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3356
|
-
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3357
|
-
assert any(finding["id"] == "verify-pair-trigger-state-malformed" for finding in findings), findings
|
|
3358
|
-
(devlyn / "claude-judge.stdout").unlink()
|
|
3359
|
-
|
|
3360
|
-
# iter-0083: canonical summary verdict conservation.
|
|
3361
|
-
iter_0083_paths = (
|
|
3362
|
-
"pipeline.state.json",
|
|
3363
|
-
"verify-mechanical.findings.jsonl",
|
|
3364
|
-
"verify.findings.jsonl",
|
|
3365
|
-
"verify.pair.findings.jsonl",
|
|
3366
|
-
"verify.pair-judge.findings.jsonl",
|
|
3367
|
-
"pair-judge.summary.json",
|
|
3368
|
-
"codex-primary-judge.summary.json",
|
|
3369
|
-
"grok-judge.summary.json",
|
|
3370
|
-
"verify.pair.timeout.json",
|
|
3371
|
-
"codex-judge.stdout",
|
|
3372
|
-
"claude-judge.stdout",
|
|
3373
|
-
)
|
|
3374
|
-
|
|
3375
|
-
def iter_0083_reset() -> None:
|
|
3376
|
-
for name in iter_0083_paths:
|
|
3377
|
-
(devlyn / name).unlink(missing_ok=True)
|
|
3378
|
-
|
|
3379
|
-
def iter_0083_case(
|
|
3380
|
-
summary_payload: object | None,
|
|
3381
|
-
severity: str | None,
|
|
3382
|
-
*,
|
|
3383
|
-
carrier: str | None = "verify.pair.findings.jsonl",
|
|
3384
|
-
other_summary: str | None = None,
|
|
3385
|
-
timeout: bool = False,
|
|
3386
|
-
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
|
3387
|
-
iter_0083_reset()
|
|
3388
|
-
(devlyn / "pipeline.state.json").write_text(
|
|
3389
|
-
json.dumps({"engine": "claude", "phases": {"verify": {}}}), encoding="utf-8",
|
|
3390
|
-
)
|
|
3391
|
-
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
3392
|
-
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
3393
|
-
if carrier is not None:
|
|
3394
|
-
content = (
|
|
3395
|
-
json.dumps({"id": "iter-0083", "severity": severity}) + "\n"
|
|
3396
|
-
if severity
|
|
3397
|
-
else ""
|
|
3398
|
-
)
|
|
3399
|
-
(devlyn / carrier).write_text(content, encoding="utf-8")
|
|
3400
|
-
if summary_payload is not None:
|
|
3401
|
-
content = (
|
|
3402
|
-
summary_payload
|
|
3403
|
-
if isinstance(summary_payload, str)
|
|
3404
|
-
else json.dumps(summary_payload)
|
|
3405
|
-
)
|
|
3406
|
-
(devlyn / "pair-judge.summary.json").write_text(content, encoding="utf-8")
|
|
3407
|
-
if other_summary is not None:
|
|
3408
|
-
(devlyn / other_summary).write_text(
|
|
3409
|
-
json.dumps({"verdict": "BLOCKED"}), encoding="utf-8"
|
|
3410
|
-
)
|
|
3411
|
-
if timeout:
|
|
3412
|
-
(devlyn / "verify.pair.timeout.json").write_text(
|
|
3413
|
-
json.dumps({"engine": "codex", "budget_seconds": 600}),
|
|
3414
|
-
encoding="utf-8",
|
|
3415
|
-
)
|
|
3416
|
-
case_findings, case_source_verdicts = read_findings(devlyn)
|
|
3417
|
-
return case_findings, write_outputs(devlyn, case_findings, case_source_verdicts)
|
|
3418
|
-
|
|
3419
|
-
for case_id, verdict, severity, expected in (
|
|
3420
|
-
("P1", "NEEDS_WORK", "INFO", "NEEDS_WORK"),
|
|
3421
|
-
("P2", "NEEDS_WORK", "LOW", "NEEDS_WORK"),
|
|
3422
|
-
("P3", "NEEDS_WORK", "MEDIUM", "NEEDS_WORK"),
|
|
3423
|
-
("P4", "PASS", "HIGH", "NEEDS_WORK"),
|
|
3424
|
-
("P5", "BLOCKED", "INFO", "BLOCKED"),
|
|
3425
|
-
("P6", "FAIL", "INFO", "NEEDS_WORK"),
|
|
3426
|
-
("P7", "PASS", None, "PASS"),
|
|
3427
|
-
):
|
|
3428
|
-
_, summary = iter_0083_case({"verdict": verdict}, severity)
|
|
3429
|
-
assert summary["source_verdicts"]["pair_judge"] == expected, case_id
|
|
3430
|
-
assert summary["verdict"] == expected, case_id
|
|
3431
|
-
|
|
3432
|
-
for case_id, severity, expected in (
|
|
3433
|
-
("P8-INFO", "INFO", "PASS"),
|
|
3434
|
-
("P8-LOW", "LOW", "PASS_WITH_ISSUES"),
|
|
3435
|
-
):
|
|
3436
|
-
_, summary = iter_0083_case(None, severity)
|
|
3437
|
-
assert summary["source_verdicts"]["pair_judge"] == expected, case_id
|
|
3438
|
-
assert summary["verdict"] == expected, case_id
|
|
3439
|
-
|
|
3440
|
-
def assert_iter_0083_blocked(
|
|
3441
|
-
case_id: str,
|
|
3442
|
-
case_findings: list[dict[str, Any]],
|
|
3443
|
-
summary: dict[str, Any],
|
|
3444
|
-
) -> None:
|
|
3445
|
-
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", case_id
|
|
3446
|
-
assert summary["verdict"] == "BLOCKED", case_id
|
|
3447
|
-
assert any(
|
|
3448
|
-
finding.get("source") == "pair_judge"
|
|
3449
|
-
and finding.get("severity") == "CRITICAL"
|
|
3450
|
-
and finding.get("file") == "pair-judge.summary.json"
|
|
3451
|
-
for finding in case_findings
|
|
3452
|
-
), case_id
|
|
3453
|
-
|
|
3454
|
-
for case_id, payload in (
|
|
3455
|
-
("N1-malformed", "{"),
|
|
3456
|
-
("N1-non-object", []),
|
|
3457
|
-
("N1-unknown", {"verdict": "UNKNOWN"}),
|
|
3458
|
-
("N2", {}),
|
|
3459
|
-
("N7", {"verdict": "TIMEOUT"}),
|
|
3460
|
-
):
|
|
3461
|
-
case_findings, summary = iter_0083_case(payload, "INFO")
|
|
3462
|
-
assert_iter_0083_blocked(case_id, case_findings, summary)
|
|
3463
|
-
|
|
3464
|
-
_, summary = iter_0083_case({"verdict": "BLOCKED"}, None, carrier=None)
|
|
3465
|
-
assert summary["source_verdicts"]["pair_judge"] is None, summary
|
|
3466
|
-
assert summary["verdict"] == "PASS", summary
|
|
3467
|
-
|
|
3468
|
-
_, summary = iter_0083_case(
|
|
3469
|
-
{"verdict": "PASS"},
|
|
3470
|
-
"INFO",
|
|
3471
|
-
other_summary="codex-primary-judge.summary.json",
|
|
3472
|
-
)
|
|
3473
|
-
assert summary["source_verdicts"]["pair_judge"] == "PASS", "N4"
|
|
3474
|
-
assert summary["verdict"] == "PASS", "N4"
|
|
3475
|
-
|
|
3476
|
-
_, summary = iter_0083_case(
|
|
3477
|
-
None,
|
|
3478
|
-
"INFO",
|
|
3479
|
-
other_summary="grok-judge.summary.json",
|
|
3480
|
-
)
|
|
3481
|
-
assert summary["source_verdicts"]["pair_judge"] == "PASS", "N5"
|
|
3482
|
-
assert summary["verdict"] == "PASS", "N5"
|
|
3483
|
-
|
|
3484
|
-
_, summary = iter_0083_case(
|
|
3485
|
-
{"verdict": "NEEDS_WORK"},
|
|
3486
|
-
"INFO",
|
|
3487
|
-
carrier="verify.pair-judge.findings.jsonl",
|
|
3488
|
-
)
|
|
3489
|
-
assert summary["source_verdicts"]["pair_judge"] == "NEEDS_WORK", "N6"
|
|
3490
|
-
assert summary["verdict"] == "NEEDS_WORK", "N6"
|
|
3491
|
-
|
|
3492
|
-
case_findings, summary = iter_0083_case({"verdict": "BLOCKED"}, None, timeout=True)
|
|
3493
|
-
assert case_findings == [], "VERIFY-JUDGE-001"
|
|
3494
|
-
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", "VERIFY-JUDGE-001"
|
|
3495
|
-
assert summary["verdict"] == "BLOCKED", "VERIFY-JUDGE-001"
|
|
3496
|
-
return 0
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
def main() -> int:
|
|
3500
|
-
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
3501
|
-
parser.add_argument("--devlyn-dir", default=".devlyn")
|
|
3502
|
-
parser.add_argument("--write-state", action="store_true")
|
|
3503
|
-
parser.add_argument("--self-test", action="store_true")
|
|
3504
|
-
args = parser.parse_args()
|
|
3505
|
-
if args.self_test:
|
|
3506
|
-
return self_test()
|
|
3507
|
-
|
|
3508
|
-
devlyn = pathlib.Path(args.devlyn_dir)
|
|
3509
|
-
if not devlyn.is_dir():
|
|
3510
|
-
sys.stderr.write(f"error: {devlyn} is not a directory\n")
|
|
3511
|
-
return 1
|
|
3512
|
-
if args.write_state:
|
|
3513
|
-
lock = runpy.run_path(str(pathlib.Path(__file__).with_name("platform-support.py")))["file_lock"]
|
|
3514
|
-
with lock(devlyn.resolve() / "pipeline.state.lock", blocking=True):
|
|
3515
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3516
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3517
|
-
write_state(devlyn, summary)
|
|
3518
|
-
print(json.dumps(summary, sort_keys=True))
|
|
3519
|
-
return 0
|
|
3520
|
-
findings, source_verdicts = read_findings(devlyn)
|
|
3521
|
-
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3522
|
-
print(json.dumps(summary, sort_keys=True))
|
|
3523
|
-
return 0
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
if __name__ == "__main__":
|
|
3527
|
-
runpy.run_path(str(pathlib.Path(__file__).with_name("platform-support.py")))["configure_utf8"]()
|
|
3528
|
-
raise SystemExit(main())
|