devlyn-cli 2.10.1 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -9
- package/CLAUDE.md +17 -17
- package/README.md +33 -9
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +25 -0
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +10 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +30 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +32 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +44 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +16 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +15 -0
- package/bin/devlyn.js +9 -2
- package/config/skills/_shared/.ruff.toml +2 -0
- package/config/skills/_shared/adapters/claude.md +9 -1
- package/config/skills/_shared/adapters/codex.md +5 -0
- package/config/skills/_shared/adapters/grok.md +4 -2
- package/config/skills/_shared/archive_run.py +738 -56
- package/config/skills/_shared/codex-config.md +19 -4
- package/config/skills/_shared/codex-monitored.sh +81 -12
- package/config/skills/_shared/collect-codex-findings.py +311 -128
- package/config/skills/_shared/engine-preflight.md +8 -7
- package/config/skills/_shared/expected.schema.json +61 -0
- package/config/skills/_shared/finish-gate.py +12 -11
- package/config/skills/_shared/grok-anchor-guard.py +18 -1
- package/config/skills/_shared/invocation-receipt.py +1041 -0
- package/config/skills/_shared/judge-output-parser.py +200 -0
- package/config/skills/_shared/judge-role-evidence.py +273 -0
- package/config/skills/_shared/process-evidence.py +909 -0
- package/config/skills/_shared/resolve-bootstrap.py +776 -84
- package/config/skills/_shared/resolve-stop-hook.py +33 -1
- package/config/skills/_shared/role-config.py +552 -0
- package/config/skills/_shared/run-bounded.py +1 -1
- package/config/skills/_shared/spec-verify-check.py +749 -181
- package/config/skills/_shared/state-phase-write.py +1838 -58
- package/config/skills/_shared/terminal-claim-check.py +15 -0
- package/config/skills/_shared/verify-merge-findings.py +958 -51
- package/config/skills/devlyn:engines/SKILL.md +8 -16
- package/config/skills/devlyn:ideate/SKILL.md +2 -2
- package/config/skills/devlyn:ideate/references/spec-template.md +6 -5
- package/config/skills/devlyn:queue/SKILL.md +7 -4
- package/config/skills/devlyn:resolve/SKILL.md +93 -35
- package/config/skills/devlyn:resolve/references/free-form-mode.md +1 -0
- package/config/skills/devlyn:resolve/references/outer-loop.md +15 -0
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +31 -2
- package/config/skills/devlyn:resolve/references/phases/implement.md +3 -2
- package/config/skills/devlyn:resolve/references/phases/verify.md +120 -84
- package/config/skills/devlyn:resolve/references/state-schema.md +22 -8
- package/package.json +5 -1
- package/scripts/lint-skills.sh +314 -285
|
@@ -15,15 +15,24 @@ import pathlib
|
|
|
15
15
|
import re
|
|
16
16
|
import sys
|
|
17
17
|
import tempfile
|
|
18
|
+
import runpy
|
|
18
19
|
from typing import Any
|
|
19
20
|
|
|
20
21
|
|
|
22
|
+
JUDGE_OUTPUT_PARSER = runpy.run_path(pathlib.Path(__file__).with_name("judge-output-parser.py"))
|
|
23
|
+
PROCESS_EVIDENCE = runpy.run_path(pathlib.Path(__file__).with_name("process-evidence.py"))
|
|
24
|
+
|
|
25
|
+
|
|
21
26
|
SOURCE_FILES = (
|
|
22
27
|
("mechanical", "verify-mechanical.findings.jsonl"),
|
|
23
28
|
("judge", "verify.findings.jsonl"),
|
|
24
29
|
("pair_judge", "verify.pair.findings.jsonl"),
|
|
25
30
|
("pair_judge", "verify.pair-judge.findings.jsonl"),
|
|
26
31
|
)
|
|
32
|
+
REQUIRED_SOURCE_FILES = {
|
|
33
|
+
"mechanical": "verify-mechanical.findings.jsonl",
|
|
34
|
+
"judge": "verify.findings.jsonl",
|
|
35
|
+
}
|
|
27
36
|
|
|
28
37
|
VERDICT_RANK = {
|
|
29
38
|
"PASS": 0,
|
|
@@ -86,8 +95,21 @@ def reject_json_constant(token: str) -> None:
|
|
|
86
95
|
raise ValueError(f"invalid JSON numeric constant: {token}")
|
|
87
96
|
|
|
88
97
|
|
|
98
|
+
def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict:
|
|
99
|
+
result = {}
|
|
100
|
+
for key, value in pairs:
|
|
101
|
+
if key in result:
|
|
102
|
+
raise ValueError(f"duplicate JSON key: {key}")
|
|
103
|
+
result[key] = value
|
|
104
|
+
return result
|
|
105
|
+
|
|
106
|
+
|
|
89
107
|
def loads_strict_json(text: str) -> Any:
|
|
90
|
-
return json.loads(
|
|
108
|
+
return json.loads(
|
|
109
|
+
text,
|
|
110
|
+
parse_constant=reject_json_constant,
|
|
111
|
+
object_pairs_hook=reject_duplicate_keys,
|
|
112
|
+
)
|
|
91
113
|
|
|
92
114
|
|
|
93
115
|
def rank(verdict: str | None) -> int:
|
|
@@ -125,9 +147,187 @@ def finding_rank(finding: dict[str, Any]) -> int:
|
|
|
125
147
|
return 0
|
|
126
148
|
|
|
127
149
|
|
|
150
|
+
def mechanical_evidence_required(devlyn: pathlib.Path, state: dict[str, Any]) -> bool:
|
|
151
|
+
return PROCESS_EVIDENCE["mechanical_evidence_required"](devlyn.parent.resolve(), state)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def mechanical_evidence_carrier(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
155
|
+
state_path = devlyn / "pipeline.state.json"
|
|
156
|
+
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
157
|
+
if not isinstance(state, dict):
|
|
158
|
+
raise ValueError("pipeline.state.json must contain a JSON object")
|
|
159
|
+
results_path = devlyn / "spec-verify.results.json"
|
|
160
|
+
if not results_path.is_file():
|
|
161
|
+
if state.get("version") == "3.0" and mechanical_evidence_required(devlyn, state):
|
|
162
|
+
raise ValueError("spec-verify.results.json is required for VERIFY MECHANICAL evidence")
|
|
163
|
+
return None
|
|
164
|
+
results = loads_strict_json(results_path.read_text(encoding="utf-8"))
|
|
165
|
+
if not isinstance(results, dict):
|
|
166
|
+
raise ValueError("spec-verify.results.json must contain a JSON object")
|
|
167
|
+
commands = results.get("commands")
|
|
168
|
+
if not isinstance(commands, list):
|
|
169
|
+
raise ValueError("spec-verify.results.json commands must be an array")
|
|
170
|
+
carrier = results.get("process_evidence")
|
|
171
|
+
if carrier is None:
|
|
172
|
+
if commands:
|
|
173
|
+
raise ValueError("MECHANICAL commands exist without a process-evidence carrier")
|
|
174
|
+
if mechanical_evidence_required(devlyn, state):
|
|
175
|
+
raise ValueError("required VERIFY MECHANICAL evidence has no process-evidence carrier")
|
|
176
|
+
run_id = state.get("run_id")
|
|
177
|
+
phases = state.get("phases")
|
|
178
|
+
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
179
|
+
round_ = verify_phase.get("round") if isinstance(verify_phase, dict) else None
|
|
180
|
+
if isinstance(run_id, str) and isinstance(round_, int) and not isinstance(round_, bool):
|
|
181
|
+
manifest_path = (
|
|
182
|
+
devlyn / "process-evidence" / run_id / "verify"
|
|
183
|
+
/ f"round-{round_}" / "manifest.json"
|
|
184
|
+
)
|
|
185
|
+
if manifest_path.exists():
|
|
186
|
+
raise ValueError("VERIFY manifest exists without a process-evidence carrier")
|
|
187
|
+
return None
|
|
188
|
+
run_id = state.get("run_id")
|
|
189
|
+
phases = state.get("phases")
|
|
190
|
+
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
191
|
+
round_ = verify_phase.get("round") if isinstance(verify_phase, dict) else None
|
|
192
|
+
if not isinstance(run_id, str) or not run_id:
|
|
193
|
+
raise ValueError("state.run_id is required for sealed MECHANICAL evidence")
|
|
194
|
+
if isinstance(round_, bool) or not isinstance(round_, int) or round_ < 0:
|
|
195
|
+
raise ValueError("phases.verify.round is required for sealed MECHANICAL evidence")
|
|
196
|
+
if not isinstance(carrier, dict) or carrier.get("phase") != "verify" or carrier.get("round") != round_:
|
|
197
|
+
raise ValueError("MECHANICAL process-evidence carrier does not match the VERIFY round")
|
|
198
|
+
PROCESS_EVIDENCE["validate_summary_commands"](
|
|
199
|
+
devlyn.parent.resolve(), commands, carrier,
|
|
200
|
+
)
|
|
201
|
+
expected_prefix = f".devlyn/process-evidence/{run_id}/verify/round-{round_}/"
|
|
202
|
+
manifest = carrier.get("manifest")
|
|
203
|
+
if not isinstance(manifest, dict) or not str(manifest.get("path", "")).startswith(expected_prefix):
|
|
204
|
+
raise ValueError("MECHANICAL process-evidence carrier does not match state.run_id")
|
|
205
|
+
|
|
206
|
+
bound = state.get("process_evidence")
|
|
207
|
+
if bound is not None:
|
|
208
|
+
if not isinstance(bound, list):
|
|
209
|
+
raise ValueError("state.process_evidence must be null or an array")
|
|
210
|
+
same_round = [
|
|
211
|
+
item for item in bound
|
|
212
|
+
if isinstance(item, dict)
|
|
213
|
+
and item.get("phase") == "verify"
|
|
214
|
+
and item.get("round") == round_
|
|
215
|
+
]
|
|
216
|
+
if same_round and same_round != [carrier]:
|
|
217
|
+
raise ValueError("state-bound VERIFY evidence disagrees with MECHANICAL results")
|
|
218
|
+
return carrier
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def mechanical_evidence_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
222
|
+
try:
|
|
223
|
+
mechanical_evidence_carrier(devlyn)
|
|
224
|
+
except (PROCESS_EVIDENCE["EvidenceError"], OSError, UnicodeError, ValueError) as exc:
|
|
225
|
+
return {
|
|
226
|
+
"id": "verify-mechanical-evidence-invalid",
|
|
227
|
+
"rule_id": "invariant.process-evidence-invalid",
|
|
228
|
+
"severity": "CRITICAL",
|
|
229
|
+
"confidence": "high",
|
|
230
|
+
"file": "spec-verify.results.json",
|
|
231
|
+
"line": 1,
|
|
232
|
+
"message": f"Sealed VERIFY MECHANICAL evidence failed rehash before merge: {exc}",
|
|
233
|
+
"criterion_ref": "process-evidence://mechanical",
|
|
234
|
+
"source": "mechanical",
|
|
235
|
+
}
|
|
236
|
+
return None
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def mechanical_evidence_outcome(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
240
|
+
carrier = mechanical_evidence_carrier(devlyn)
|
|
241
|
+
if carrier is None:
|
|
242
|
+
return None
|
|
243
|
+
return PROCESS_EVIDENCE["bound_carrier_outcome"](devlyn.parent.resolve(), carrier)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
ROLE_CONFIG = runpy.run_path(pathlib.Path(__file__).with_name("role-config.py"))
|
|
247
|
+
JUDGE_ROLE_EVIDENCE = runpy.run_path(pathlib.Path(__file__).with_name("judge-role-evidence.py"))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def resolved_primary_engine(state):
|
|
251
|
+
return ROLE_CONFIG["primary_engine"](state)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def primary_timeout_blocker(id_: str, message: str) -> dict[str, Any]:
|
|
255
|
+
return {
|
|
256
|
+
"id": id_,
|
|
257
|
+
"rule_id": "verify.primary.timeout-contract",
|
|
258
|
+
"severity": "CRITICAL",
|
|
259
|
+
"confidence": "high",
|
|
260
|
+
"file": "verify.primary.timeout.json",
|
|
261
|
+
"line": 1,
|
|
262
|
+
"message": message,
|
|
263
|
+
"criterion_ref": "verify.primary.timeout",
|
|
264
|
+
"source": "judge",
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def read_primary_timeout_marker(
|
|
269
|
+
devlyn: pathlib.Path,
|
|
270
|
+
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
271
|
+
path = devlyn / "verify.primary.timeout.json"
|
|
272
|
+
if not path.is_file():
|
|
273
|
+
return None, None
|
|
274
|
+
try:
|
|
275
|
+
marker = loads_strict_json(path.read_text(encoding="utf-8"))
|
|
276
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
277
|
+
return None, primary_timeout_blocker(
|
|
278
|
+
"verify-primary-timeout-marker-malformed",
|
|
279
|
+
f"verify.primary.timeout.json is malformed: {exc}",
|
|
280
|
+
)
|
|
281
|
+
if not isinstance(marker, dict) or set(marker) != {"engine", "budget_seconds"}:
|
|
282
|
+
return None, primary_timeout_blocker(
|
|
283
|
+
"verify-primary-timeout-marker-malformed",
|
|
284
|
+
"verify.primary.timeout.json must contain exactly engine and budget_seconds.",
|
|
285
|
+
)
|
|
286
|
+
engine = marker["engine"]
|
|
287
|
+
budget_seconds = marker["budget_seconds"]
|
|
288
|
+
if not isinstance(engine, str) or not engine:
|
|
289
|
+
return None, primary_timeout_blocker(
|
|
290
|
+
"verify-primary-timeout-marker-malformed",
|
|
291
|
+
"verify.primary.timeout.json engine must be a non-empty string.",
|
|
292
|
+
)
|
|
293
|
+
if (
|
|
294
|
+
not isinstance(budget_seconds, int)
|
|
295
|
+
or isinstance(budget_seconds, bool)
|
|
296
|
+
or budget_seconds != 600
|
|
297
|
+
):
|
|
298
|
+
return None, primary_timeout_blocker(
|
|
299
|
+
"verify-primary-timeout-budget-mismatch",
|
|
300
|
+
"verify.primary.timeout.json budget_seconds must equal 600.",
|
|
301
|
+
)
|
|
302
|
+
try:
|
|
303
|
+
state = loads_strict_json(
|
|
304
|
+
(devlyn / "pipeline.state.json").read_text(encoding="utf-8")
|
|
305
|
+
)
|
|
306
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
307
|
+
return None, primary_timeout_blocker(
|
|
308
|
+
"verify-primary-timeout-engine-mismatch",
|
|
309
|
+
f"Cannot authenticate the primary timeout engine from pipeline.state.json: {exc}",
|
|
310
|
+
)
|
|
311
|
+
try:
|
|
312
|
+
expected_engine = resolved_primary_engine(state) if isinstance(state, dict) else None
|
|
313
|
+
except ValueError:
|
|
314
|
+
expected_engine = None
|
|
315
|
+
if (
|
|
316
|
+
not isinstance(expected_engine, str)
|
|
317
|
+
or not expected_engine
|
|
318
|
+
or engine != expected_engine
|
|
319
|
+
):
|
|
320
|
+
return None, primary_timeout_blocker(
|
|
321
|
+
"verify-primary-timeout-engine-mismatch",
|
|
322
|
+
"Primary timeout engine does not match the resolved primary judge.",
|
|
323
|
+
)
|
|
324
|
+
return {"engine": engine, "budget_seconds": budget_seconds}, None
|
|
325
|
+
|
|
326
|
+
|
|
128
327
|
def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str, str | None]]:
|
|
129
328
|
findings: list[dict[str, Any]] = []
|
|
130
329
|
source_verdicts: dict[str, str | None] = {source: "PASS" for source, _ in SOURCE_FILES}
|
|
330
|
+
primary_timeout_marker, primary_timeout_violation = read_primary_timeout_marker(devlyn)
|
|
131
331
|
# A verdict must come from a judge that ran: pair_judge stays null until
|
|
132
332
|
# spawn evidence exists (a pair findings file or pair stdout). verify.md's
|
|
133
333
|
# pair contract records null when no second agent is spawned.
|
|
@@ -135,6 +335,27 @@ def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str,
|
|
|
135
335
|
for source, name in SOURCE_FILES:
|
|
136
336
|
path = devlyn / name
|
|
137
337
|
if not path.is_file():
|
|
338
|
+
# A verdict-binding mechanical result contractually skips both
|
|
339
|
+
# judges. Its absent primary carrier is therefore not evidence a
|
|
340
|
+
# dispatched primary judge failed to produce.
|
|
341
|
+
if source == "judge" and rank(source_verdicts.get("mechanical")) >= 2:
|
|
342
|
+
source_verdicts[source] = None
|
|
343
|
+
continue
|
|
344
|
+
if source == "judge" and primary_timeout_marker is not None:
|
|
345
|
+
continue
|
|
346
|
+
if REQUIRED_SOURCE_FILES.get(source) == name:
|
|
347
|
+
findings.append({
|
|
348
|
+
"id": f"verify-merge-required-source-missing-{source}",
|
|
349
|
+
"rule_id": "verify.findings.required-source-missing",
|
|
350
|
+
"severity": "CRITICAL",
|
|
351
|
+
"confidence": "high",
|
|
352
|
+
"file": name,
|
|
353
|
+
"line": 1,
|
|
354
|
+
"message": f"Required VERIFY {source} findings file is missing: {name}",
|
|
355
|
+
"criterion_ref": "verify-merge",
|
|
356
|
+
"source": source,
|
|
357
|
+
})
|
|
358
|
+
source_verdicts[source] = "BLOCKED"
|
|
138
359
|
continue
|
|
139
360
|
if source_verdicts[source] is None:
|
|
140
361
|
source_verdicts[source] = "PASS"
|
|
@@ -168,6 +389,50 @@ def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str,
|
|
|
168
389
|
source_verdicts[source] = worse(
|
|
169
390
|
source_verdicts[source], RANK_VERDICT[finding_rank(item)]
|
|
170
391
|
)
|
|
392
|
+
evidence_violation = mechanical_evidence_violation(devlyn)
|
|
393
|
+
if evidence_violation is not None:
|
|
394
|
+
findings.append(evidence_violation)
|
|
395
|
+
source_verdicts["mechanical"] = "BLOCKED"
|
|
396
|
+
else:
|
|
397
|
+
outcome = mechanical_evidence_outcome(devlyn)
|
|
398
|
+
if outcome is not None and outcome["verdict"] != "PASS":
|
|
399
|
+
blocked = outcome["verdict"] == "BLOCKED"
|
|
400
|
+
ids = (
|
|
401
|
+
[item["id"] for item in outcome["capability_denials"]]
|
|
402
|
+
if blocked else outcome["failed_ids"]
|
|
403
|
+
)
|
|
404
|
+
findings.append({
|
|
405
|
+
"id": (
|
|
406
|
+
"verify-mechanical-capability-denied"
|
|
407
|
+
if blocked else "verify-mechanical-expectation-mismatch"
|
|
408
|
+
),
|
|
409
|
+
"rule_id": (
|
|
410
|
+
"invariant.build-env-underprovisioned"
|
|
411
|
+
if blocked else "invariant.mechanical-expectation-mismatch"
|
|
412
|
+
),
|
|
413
|
+
"severity": "CRITICAL" if blocked else "HIGH",
|
|
414
|
+
"confidence": "high",
|
|
415
|
+
"file": "spec-verify.results.json",
|
|
416
|
+
"line": 1,
|
|
417
|
+
"message": (
|
|
418
|
+
"Sealed VERIFY MECHANICAL evidence records a capability denial: "
|
|
419
|
+
if blocked else
|
|
420
|
+
"Sealed VERIFY MECHANICAL evidence records failed expectations: "
|
|
421
|
+
) + ",".join(ids),
|
|
422
|
+
"criterion_ref": "process-evidence://mechanical",
|
|
423
|
+
"source": "mechanical",
|
|
424
|
+
"verdict_binding": True,
|
|
425
|
+
})
|
|
426
|
+
source_verdicts["mechanical"] = outcome["verdict"]
|
|
427
|
+
if primary_timeout_violation is not None:
|
|
428
|
+
findings.append(primary_timeout_violation)
|
|
429
|
+
source_verdicts["judge"] = "BLOCKED"
|
|
430
|
+
elif primary_timeout_marker is not None:
|
|
431
|
+
findings.append(primary_timeout_blocker(
|
|
432
|
+
"verify-primary-timeout",
|
|
433
|
+
"Primary JUDGE exceeded its authenticated 600-second wall budget.",
|
|
434
|
+
))
|
|
435
|
+
source_verdicts["judge"] = "BLOCKED"
|
|
171
436
|
findings.extend(detect_pair_stdout_contract_violations(devlyn, source_verdicts))
|
|
172
437
|
pair_summary_path = devlyn / "pair-judge.summary.json"
|
|
173
438
|
pair_carrier_exists = any(
|
|
@@ -207,6 +472,26 @@ def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str,
|
|
|
207
472
|
source_verdicts["pair_judge"] = worse(
|
|
208
473
|
source_verdicts["pair_judge"], pair_verdict
|
|
209
474
|
)
|
|
475
|
+
if (devlyn / "pipeline.state.json").is_file():
|
|
476
|
+
try:
|
|
477
|
+
state = loads_strict_json((devlyn / "pipeline.state.json").read_text())
|
|
478
|
+
required = JUDGE_ROLE_EVIDENCE["required_roles"](state)
|
|
479
|
+
if rank(source_verdicts.get("mechanical")) >= 2:
|
|
480
|
+
required = []
|
|
481
|
+
for role in required:
|
|
482
|
+
source = "judge" if role == "primary_judge" else "pair_judge"
|
|
483
|
+
if source_verdicts.get(source) in {"BLOCKED", "TIMEOUT"}:
|
|
484
|
+
continue
|
|
485
|
+
try:
|
|
486
|
+
JUDGE_ROLE_EVIDENCE["authenticate"](devlyn, state, role)
|
|
487
|
+
except (ValueError, OSError, TypeError, KeyError) as exc:
|
|
488
|
+
source_verdicts[source] = "BLOCKED"
|
|
489
|
+
findings.append({**primary_timeout_blocker("verify-role-evidence-invalid", str(exc)),
|
|
490
|
+
"source": source, "rule_id": "verify.role-evidence"})
|
|
491
|
+
except (ValueError, OSError, TypeError, KeyError) as exc:
|
|
492
|
+
source_verdicts["judge"] = "BLOCKED"
|
|
493
|
+
findings.append({**primary_timeout_blocker("verify-role-resolution-invalid", str(exc)),
|
|
494
|
+
"rule_id": "verify.role-resolution"})
|
|
210
495
|
return findings, source_verdicts
|
|
211
496
|
|
|
212
497
|
|
|
@@ -218,6 +503,29 @@ def has_pair_findings(devlyn: pathlib.Path) -> bool:
|
|
|
218
503
|
return False
|
|
219
504
|
|
|
220
505
|
|
|
506
|
+
def canonical_other_judge_stdout(devlyn: pathlib.Path, primary_engine: str) -> list[pathlib.Path]:
|
|
507
|
+
adapters = pathlib.Path(__file__).with_name("adapters")
|
|
508
|
+
engine_names = {
|
|
509
|
+
path.stem for path in adapters.glob("*.md")
|
|
510
|
+
if path.stem != "README"
|
|
511
|
+
}
|
|
512
|
+
return sorted(
|
|
513
|
+
devlyn / f"{engine}-judge.stdout"
|
|
514
|
+
for engine in engine_names
|
|
515
|
+
if engine != primary_engine and (devlyn / f"{engine}-judge.stdout").is_file()
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def pair_capture_exists(devlyn: pathlib.Path, stdout_paths: list[pathlib.Path]) -> bool:
|
|
520
|
+
return bool(stdout_paths) or has_pair_findings(devlyn) or any(
|
|
521
|
+
(devlyn / name).is_file()
|
|
522
|
+
for name in (
|
|
523
|
+
"verify.pair.timeout.json",
|
|
524
|
+
"pair-judge.summary.json",
|
|
525
|
+
)
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
|
|
221
529
|
def pair_trigger_status(devlyn: pathlib.Path) -> tuple[bool, dict[str, Any] | None]:
|
|
222
530
|
state_path = devlyn / "pipeline.state.json"
|
|
223
531
|
if not state_path.is_file():
|
|
@@ -384,12 +692,41 @@ def risk_profile_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | No
|
|
|
384
692
|
def verify_state_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
385
693
|
state_path = devlyn / "pipeline.state.json"
|
|
386
694
|
if not state_path.is_file():
|
|
387
|
-
return
|
|
695
|
+
return {
|
|
696
|
+
"id": "verify-state-missing",
|
|
697
|
+
"rule_id": "verify.state.missing",
|
|
698
|
+
"message": "pipeline.state.json is required before VERIFY merge.",
|
|
699
|
+
"file": "pipeline.state.json",
|
|
700
|
+
}
|
|
388
701
|
try:
|
|
389
702
|
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
390
703
|
except ValueError:
|
|
391
|
-
return
|
|
392
|
-
|
|
704
|
+
return {
|
|
705
|
+
"id": "verify-pair-trigger-state-malformed",
|
|
706
|
+
"rule_id": "verify.pair.emission-contract",
|
|
707
|
+
"message": "pipeline.state.json is malformed; cannot verify pair_trigger contract.",
|
|
708
|
+
"file": "pipeline.state.json",
|
|
709
|
+
}
|
|
710
|
+
if not isinstance(state, dict):
|
|
711
|
+
return {
|
|
712
|
+
"id": "verify-state-malformed",
|
|
713
|
+
"rule_id": "verify.state.malformed",
|
|
714
|
+
"message": "pipeline.state.json must be a JSON object before VERIFY merge.",
|
|
715
|
+
"file": "pipeline.state.json",
|
|
716
|
+
}
|
|
717
|
+
try:
|
|
718
|
+
engine = resolved_primary_engine(state)
|
|
719
|
+
except ValueError:
|
|
720
|
+
engine = None
|
|
721
|
+
if not isinstance(engine, str) or not engine.strip():
|
|
722
|
+
rule = "verify.state.engine-malformed"
|
|
723
|
+
return {
|
|
724
|
+
"id": rule,
|
|
725
|
+
"rule_id": rule,
|
|
726
|
+
"message": "pipeline.state.json requires a valid primary engine before VERIFY merge.",
|
|
727
|
+
"file": "pipeline.state.json",
|
|
728
|
+
}
|
|
729
|
+
if not state_uses_default_pair_contract(state):
|
|
393
730
|
return None
|
|
394
731
|
source = state.get("source")
|
|
395
732
|
if not isinstance(source, dict) or source.get("type") != "generated":
|
|
@@ -806,14 +1143,47 @@ def read_pair_timeout_marker(devlyn: pathlib.Path) -> tuple[dict[str, Any] | Non
|
|
|
806
1143
|
return {"engine": engine, "budget_seconds": budget_seconds}, None
|
|
807
1144
|
|
|
808
1145
|
|
|
1146
|
+
def is_result_less_message_stream(stdout_text: str) -> bool:
|
|
1147
|
+
"""True when a declared whole-message stream stops before its terminal result.
|
|
1148
|
+
|
|
1149
|
+
The timeout marker attests the killed process. This classifier admits only a
|
|
1150
|
+
valid stream prefix, optionally ending in one truncated JSON record.
|
|
1151
|
+
"""
|
|
1152
|
+
lines = [raw for line in stdout_text.splitlines() if (raw := line.strip())]
|
|
1153
|
+
if not lines:
|
|
1154
|
+
return False
|
|
1155
|
+
try:
|
|
1156
|
+
first = JUDGE_OUTPUT_PARSER["loads_strict_json"](lines[0])
|
|
1157
|
+
except ValueError:
|
|
1158
|
+
return False
|
|
1159
|
+
if (
|
|
1160
|
+
not isinstance(first, dict)
|
|
1161
|
+
or first.get("type") != "system"
|
|
1162
|
+
or first.get("subtype") != "init"
|
|
1163
|
+
or not isinstance(first.get("session_id"), str)
|
|
1164
|
+
):
|
|
1165
|
+
return False
|
|
1166
|
+
session_id = first["session_id"]
|
|
1167
|
+
for index, raw in enumerate(lines[1:], 1):
|
|
1168
|
+
try:
|
|
1169
|
+
item = JUDGE_OUTPUT_PARSER["loads_strict_json"](raw)
|
|
1170
|
+
except ValueError:
|
|
1171
|
+
return index == len(lines) - 1 and raw.startswith("{") and not raw.endswith("}")
|
|
1172
|
+
if (
|
|
1173
|
+
not isinstance(item, dict)
|
|
1174
|
+
or item.get("type") not in {"assistant", "user"}
|
|
1175
|
+
or item.get("session_id") != session_id
|
|
1176
|
+
):
|
|
1177
|
+
return False
|
|
1178
|
+
return True
|
|
1179
|
+
|
|
1180
|
+
|
|
809
1181
|
def detect_pair_stdout_contract_violations(
|
|
810
1182
|
devlyn: pathlib.Path,
|
|
811
1183
|
source_verdicts: dict[str, str | None],
|
|
812
1184
|
) -> list[dict[str, Any]]:
|
|
813
|
-
#
|
|
814
|
-
#
|
|
815
|
-
# contract is engine-neutral.
|
|
816
|
-
stdout_paths = sorted(devlyn.glob("*-judge.stdout"))
|
|
1185
|
+
# The primary uses its explicit phase engine, or the legacy executor; only the OTHER engine's
|
|
1186
|
+
# capture is pair evidence.
|
|
817
1187
|
timeout_marker, timeout_violation = read_pair_timeout_marker(devlyn)
|
|
818
1188
|
if timeout_violation is not None:
|
|
819
1189
|
source_verdicts["pair_judge"] = "BLOCKED"
|
|
@@ -865,6 +1235,11 @@ def detect_pair_stdout_contract_violations(
|
|
|
865
1235
|
state_violation["rule_id"],
|
|
866
1236
|
)
|
|
867
1237
|
]
|
|
1238
|
+
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
1239
|
+
assert isinstance(state, dict)
|
|
1240
|
+
engine = resolved_primary_engine(state)
|
|
1241
|
+
assert isinstance(engine, str) and engine.strip()
|
|
1242
|
+
stdout_paths = canonical_other_judge_stdout(devlyn, engine)
|
|
868
1243
|
if not required and not pair_trigger_present(devlyn):
|
|
869
1244
|
missing_violation = pair_trigger_missing_contract_violation(devlyn, source_verdicts)
|
|
870
1245
|
if missing_violation is not None:
|
|
@@ -896,27 +1271,65 @@ def detect_pair_stdout_contract_violations(
|
|
|
896
1271
|
reason_violation["file"],
|
|
897
1272
|
)
|
|
898
1273
|
]
|
|
899
|
-
|
|
1274
|
+
trigger_present = pair_trigger_present(devlyn)
|
|
1275
|
+
if trigger_present and not required:
|
|
1276
|
+
if pair_capture_exists(devlyn, stdout_paths):
|
|
1277
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1278
|
+
return [
|
|
1279
|
+
pair_blocker(
|
|
1280
|
+
"verify-pair-skipped-output-present",
|
|
1281
|
+
"Pair state is skipped or ineligible, but a canonical OTHER-judge carrier exists.",
|
|
1282
|
+
stdout_paths[0].name if stdout_paths else "verify.pair.findings.jsonl",
|
|
1283
|
+
"verify.pair.state-capture-contradiction",
|
|
1284
|
+
)
|
|
1285
|
+
]
|
|
1286
|
+
source_verdicts["pair_judge"] = None
|
|
900
1287
|
return []
|
|
1288
|
+
if len(stdout_paths) > 1:
|
|
1289
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1290
|
+
return [
|
|
1291
|
+
pair_blocker(
|
|
1292
|
+
"verify-pair-output-ambiguous",
|
|
1293
|
+
"Multiple canonical OTHER-engine stdout carriers exist: "
|
|
1294
|
+
+ ", ".join(path.name for path in stdout_paths),
|
|
1295
|
+
stdout_paths[0].name,
|
|
1296
|
+
)
|
|
1297
|
+
]
|
|
1298
|
+
if timeout_marker is not None:
|
|
1299
|
+
expected_timeout_stdout = f"{timeout_marker['engine']}-judge.stdout"
|
|
1300
|
+
if timeout_marker["engine"] == engine or (
|
|
1301
|
+
stdout_paths and stdout_paths[0].name != expected_timeout_stdout
|
|
1302
|
+
):
|
|
1303
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1304
|
+
return [
|
|
1305
|
+
pair_blocker(
|
|
1306
|
+
"verify-pair-timeout-engine-mismatch",
|
|
1307
|
+
"Pair timeout engine does not match the canonical OTHER-engine carrier.",
|
|
1308
|
+
"verify.pair.timeout.json",
|
|
1309
|
+
)
|
|
1310
|
+
]
|
|
901
1311
|
if not stdout_paths:
|
|
902
1312
|
if timeout_marker is not None:
|
|
903
|
-
source_verdicts["pair_judge"]
|
|
1313
|
+
if rank(source_verdicts["pair_judge"]) <= rank("TIMEOUT"):
|
|
1314
|
+
source_verdicts["pair_judge"] = "TIMEOUT"
|
|
904
1315
|
return []
|
|
905
1316
|
if required:
|
|
906
1317
|
source_verdicts["pair_judge"] = "BLOCKED"
|
|
907
1318
|
return [
|
|
908
1319
|
pair_blocker(
|
|
909
1320
|
"verify-pair-required-output-missing",
|
|
910
|
-
"Pair-mode was required, but the pair-JUDGE produced no
|
|
1321
|
+
"Pair-mode was required, but the pair-JUDGE produced no canonical OTHER-engine stdout.",
|
|
911
1322
|
"verify.pair.findings.jsonl",
|
|
912
1323
|
)
|
|
913
1324
|
]
|
|
914
1325
|
return []
|
|
1326
|
+
if has_pair_findings(devlyn):
|
|
1327
|
+
return []
|
|
915
1328
|
if source_verdicts["pair_judge"] is None and timeout_marker is None:
|
|
916
1329
|
source_verdicts["pair_judge"] = "PASS"
|
|
917
1330
|
for stdout_path in stdout_paths:
|
|
918
|
-
|
|
919
|
-
if not
|
|
1331
|
+
stdout_text = stdout_path.read_text(encoding="utf-8")
|
|
1332
|
+
if not stdout_text.strip():
|
|
920
1333
|
if timeout_marker is not None:
|
|
921
1334
|
continue
|
|
922
1335
|
source_verdicts["pair_judge"] = "BLOCKED"
|
|
@@ -927,41 +1340,27 @@ def detect_pair_stdout_contract_violations(
|
|
|
927
1340
|
stdout_path.name,
|
|
928
1341
|
)
|
|
929
1342
|
]
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
if not raw:
|
|
935
|
-
continue
|
|
936
|
-
if raw.startswith("# SUMMARY "):
|
|
937
|
-
try:
|
|
938
|
-
summary = loads_strict_json(raw.removeprefix("# SUMMARY ").strip())
|
|
939
|
-
except ValueError:
|
|
940
|
-
continue
|
|
941
|
-
if summary.get("verdict") in {"NEEDS_WORK", "FAIL", "BLOCKED"}:
|
|
942
|
-
has_nonpass_summary = True
|
|
943
|
-
continue
|
|
944
|
-
if raw.startswith("#"):
|
|
945
|
-
continue
|
|
946
|
-
try:
|
|
947
|
-
item = loads_strict_json(raw)
|
|
948
|
-
except ValueError:
|
|
1343
|
+
try:
|
|
1344
|
+
stdout_findings, stdout_summary = JUDGE_OUTPUT_PARSER["collect_stdout"](stdout_path)
|
|
1345
|
+
except SystemExit as exc:
|
|
1346
|
+
if timeout_marker is not None and is_result_less_message_stream(stdout_text):
|
|
949
1347
|
continue
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
1348
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1349
|
+
return [
|
|
1350
|
+
pair_blocker(
|
|
1351
|
+
"verify-pair-emission-contract-violated",
|
|
1352
|
+
f"pair-JUDGE stdout {stdout_path.name} violates the shared emission parser: {exc}",
|
|
1353
|
+
stdout_path.name,
|
|
1354
|
+
)
|
|
1355
|
+
]
|
|
1356
|
+
if stdout_findings or stdout_summary is None or stdout_summary["verdict"] != "PASS":
|
|
958
1357
|
source_verdicts["pair_judge"] = "BLOCKED"
|
|
959
1358
|
return [
|
|
960
1359
|
pair_blocker(
|
|
961
1360
|
"verify-pair-emission-contract-violated",
|
|
962
1361
|
(
|
|
963
1362
|
f"pair-JUDGE stdout {stdout_path.name} contained findings or a non-PASS "
|
|
964
|
-
"
|
|
1363
|
+
"verdict, but the canonical pair findings JSONL file was empty."
|
|
965
1364
|
),
|
|
966
1365
|
stdout_path.name,
|
|
967
1366
|
)
|
|
@@ -1004,11 +1403,47 @@ def write_state(devlyn: pathlib.Path, summary: dict[str, Any]) -> None:
|
|
|
1004
1403
|
if not state_path.is_file():
|
|
1005
1404
|
raise SystemExit(f"error: {state_path} not found")
|
|
1006
1405
|
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
1406
|
+
try:
|
|
1407
|
+
carrier = mechanical_evidence_carrier(devlyn)
|
|
1408
|
+
except (PROCESS_EVIDENCE["EvidenceError"], OSError, UnicodeError, ValueError) as exc:
|
|
1409
|
+
if summary.get("source_verdicts", {}).get("mechanical") != "BLOCKED":
|
|
1410
|
+
raise SystemExit(f"error: invalid MECHANICAL process evidence: {exc}") from exc
|
|
1411
|
+
carrier = None
|
|
1412
|
+
if carrier is not None:
|
|
1413
|
+
bound = state.get("process_evidence")
|
|
1414
|
+
if bound is None:
|
|
1415
|
+
bound = []
|
|
1416
|
+
elif not isinstance(bound, list):
|
|
1417
|
+
raise SystemExit("error: state.process_evidence must be null or an array")
|
|
1418
|
+
same_round = [
|
|
1419
|
+
item for item in bound
|
|
1420
|
+
if isinstance(item, dict)
|
|
1421
|
+
and item.get("phase") == carrier["phase"]
|
|
1422
|
+
and item.get("round") == carrier["round"]
|
|
1423
|
+
]
|
|
1424
|
+
if same_round and same_round != [carrier]:
|
|
1425
|
+
raise SystemExit("error: state-bound VERIFY evidence disagrees with MECHANICAL results")
|
|
1426
|
+
if not same_round:
|
|
1427
|
+
bound.append(carrier)
|
|
1428
|
+
state["process_evidence"] = bound
|
|
1007
1429
|
phases = state.setdefault("phases", {})
|
|
1008
1430
|
verify = phases.get("verify")
|
|
1009
1431
|
if not isinstance(verify, dict):
|
|
1010
1432
|
verify = {}
|
|
1011
1433
|
phases["verify"] = verify
|
|
1434
|
+
role_evidence = {}
|
|
1435
|
+
try:
|
|
1436
|
+
required_roles = JUDGE_ROLE_EVIDENCE["required_roles"](state)
|
|
1437
|
+
except ValueError as exc:
|
|
1438
|
+
if summary.get("source_verdicts", {}).get("judge") != "BLOCKED":
|
|
1439
|
+
raise SystemExit(str(exc)) from exc
|
|
1440
|
+
required_roles = [] # Persist the invalid-resolution failure; no legacy dispatch.
|
|
1441
|
+
for role in required_roles:
|
|
1442
|
+
source = "judge" if role == "primary_judge" else "pair_judge"
|
|
1443
|
+
if summary.get("source_verdicts", {}).get(source) in {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK"}:
|
|
1444
|
+
role_evidence[role] = JUDGE_ROLE_EVIDENCE["authenticate"](devlyn, state, role)
|
|
1445
|
+
if role_evidence:
|
|
1446
|
+
verify["role_evidence"] = role_evidence
|
|
1012
1447
|
verify["verdict"] = summary["verdict"]
|
|
1013
1448
|
sub = verify.get("sub_verdicts")
|
|
1014
1449
|
if sub is None:
|
|
@@ -1034,15 +1469,28 @@ def write_state(devlyn: pathlib.Path, summary: dict[str, Any]) -> None:
|
|
|
1034
1469
|
|
|
1035
1470
|
|
|
1036
1471
|
def self_test() -> int:
|
|
1472
|
+
import subprocess
|
|
1473
|
+
|
|
1474
|
+
try:
|
|
1475
|
+
loads_strict_json('{"verdict":"PASS","verdict":"BLOCKED"}')
|
|
1476
|
+
except ValueError as exc:
|
|
1477
|
+
assert "duplicate JSON key" in str(exc)
|
|
1478
|
+
else:
|
|
1479
|
+
raise AssertionError("duplicate VERIFY authority key was accepted")
|
|
1037
1480
|
with tempfile.TemporaryDirectory() as tmp:
|
|
1038
1481
|
devlyn = pathlib.Path(tmp)
|
|
1039
1482
|
|
|
1483
|
+
# Every completed VERIFY has deterministic mechanical and primary
|
|
1484
|
+
# judge carriers. Missing either one is an evidence failure, not PASS.
|
|
1485
|
+
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1486
|
+
|
|
1040
1487
|
# state-phase-write.py's spawn always writes sub_verdicts: null (the
|
|
1041
1488
|
# per-round reset contract, state-schema.md#write-protocol) — this is
|
|
1042
1489
|
# the real shape write_state() sees on every VERIFY completion, not
|
|
1043
1490
|
# the pre-populated {} the other scenarios below seed for brevity.
|
|
1044
1491
|
(devlyn / "pipeline.state.json").write_text(
|
|
1045
1492
|
json.dumps({
|
|
1493
|
+
"engine": "claude",
|
|
1046
1494
|
"phases": {
|
|
1047
1495
|
"verify": {
|
|
1048
1496
|
"verdict": None,
|
|
@@ -1054,6 +1502,152 @@ def self_test() -> int:
|
|
|
1054
1502
|
encoding="utf-8",
|
|
1055
1503
|
)
|
|
1056
1504
|
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1505
|
+
|
|
1506
|
+
sealed_work = pathlib.Path(tmp) / "sealed-work"
|
|
1507
|
+
sealed_devlyn = sealed_work / ".devlyn"
|
|
1508
|
+
sealed_devlyn.mkdir(parents=True)
|
|
1509
|
+
sealed_state = {
|
|
1510
|
+
"run_id": "rs-sealed-mechanical",
|
|
1511
|
+
"engine": "claude",
|
|
1512
|
+
"process_evidence": None,
|
|
1513
|
+
"phases": {"verify": {"round": 2, "verdict": None, "sub_verdicts": None}},
|
|
1514
|
+
}
|
|
1515
|
+
obligation = PROCESS_EVIDENCE["normalize_obligation"]({
|
|
1516
|
+
"id": "mechanical-0",
|
|
1517
|
+
"phase": "verify",
|
|
1518
|
+
"argv": [sys.executable, "-c", "print('sealed')"],
|
|
1519
|
+
})
|
|
1520
|
+
manifest_rel = PROCESS_EVIDENCE["manifest_relative_path"](sealed_state, "verify")
|
|
1521
|
+
PROCESS_EVIDENCE["capture_process"](
|
|
1522
|
+
sealed_work, sealed_work / manifest_rel, sealed_state["run_id"],
|
|
1523
|
+
"verify", 2, obligation,
|
|
1524
|
+
)
|
|
1525
|
+
carrier = PROCESS_EVIDENCE["validate_manifest"](
|
|
1526
|
+
sealed_work, manifest_rel, sealed_state["run_id"], "verify", 2,
|
|
1527
|
+
[obligation], require_expectations=False,
|
|
1528
|
+
)
|
|
1529
|
+
(sealed_devlyn / "pipeline.state.json").write_text(
|
|
1530
|
+
json.dumps(sealed_state), encoding="utf-8",
|
|
1531
|
+
)
|
|
1532
|
+
sealed_commands = PROCESS_EVIDENCE["bound_carrier_summary_commands"](
|
|
1533
|
+
sealed_work, carrier,
|
|
1534
|
+
)
|
|
1535
|
+
(sealed_devlyn / "spec-verify.results.json").write_text(
|
|
1536
|
+
json.dumps({"commands": sealed_commands, "process_evidence": carrier}),
|
|
1537
|
+
encoding="utf-8",
|
|
1538
|
+
)
|
|
1539
|
+
(sealed_devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1540
|
+
(sealed_devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1541
|
+
sealed_findings, sealed_verdicts = read_findings(sealed_devlyn)
|
|
1542
|
+
assert sealed_verdicts["mechanical"] == "PASS", sealed_findings
|
|
1543
|
+
sealed_summary = write_outputs(sealed_devlyn, sealed_findings, sealed_verdicts)
|
|
1544
|
+
write_state(sealed_devlyn, sealed_summary)
|
|
1545
|
+
bound_state = loads_strict_json(
|
|
1546
|
+
(sealed_devlyn / "pipeline.state.json").read_text(encoding="utf-8")
|
|
1547
|
+
)
|
|
1548
|
+
assert bound_state["process_evidence"] == [carrier], bound_state
|
|
1549
|
+
|
|
1550
|
+
failed_work = pathlib.Path(tmp) / "failed-work"
|
|
1551
|
+
failed_devlyn = failed_work / ".devlyn"
|
|
1552
|
+
failed_devlyn.mkdir(parents=True)
|
|
1553
|
+
failed_state = {
|
|
1554
|
+
"version": "3.0",
|
|
1555
|
+
"run_id": "rs-failed-mechanical",
|
|
1556
|
+
"engine": "claude",
|
|
1557
|
+
"source": {"type": "spec", "spec_path": "docs/failed/spec.md"},
|
|
1558
|
+
"process_evidence": None,
|
|
1559
|
+
"phases": {"verify": {"round": 0, "verdict": None, "sub_verdicts": None}},
|
|
1560
|
+
}
|
|
1561
|
+
failed_spec = failed_work / "docs" / "failed"
|
|
1562
|
+
failed_spec.mkdir(parents=True)
|
|
1563
|
+
(failed_spec / "spec.md").write_text("# failed fixture\n", encoding="utf-8")
|
|
1564
|
+
(failed_spec / "spec.expected.json").write_text(json.dumps({
|
|
1565
|
+
"verification_commands": [{"cmd": "exit 7", "exit_code": 0}],
|
|
1566
|
+
}) + "\n", encoding="utf-8")
|
|
1567
|
+
failed_obligation = PROCESS_EVIDENCE["normalize_obligation"]({
|
|
1568
|
+
"id": "verification-command-0001",
|
|
1569
|
+
"phase": "verify",
|
|
1570
|
+
"cmd": "exit 7",
|
|
1571
|
+
})
|
|
1572
|
+
failed_manifest = PROCESS_EVIDENCE["manifest_relative_path"](failed_state, "verify")
|
|
1573
|
+
PROCESS_EVIDENCE["capture_process"](
|
|
1574
|
+
failed_work, failed_work / failed_manifest, failed_state["run_id"],
|
|
1575
|
+
"verify", 0, failed_obligation,
|
|
1576
|
+
)
|
|
1577
|
+
failed_carrier = PROCESS_EVIDENCE["validate_manifest"](
|
|
1578
|
+
failed_work, failed_manifest, failed_state["run_id"], "verify", 0,
|
|
1579
|
+
[failed_obligation], require_expectations=False,
|
|
1580
|
+
)
|
|
1581
|
+
(failed_devlyn / "pipeline.state.json").write_text(
|
|
1582
|
+
json.dumps(failed_state), encoding="utf-8",
|
|
1583
|
+
)
|
|
1584
|
+
# Exact laundering attempt: mutable derivatives say PASS/empty while
|
|
1585
|
+
# the state-identical sealed manifest still records the failed command.
|
|
1586
|
+
laundered_commands = PROCESS_EVIDENCE["bound_carrier_summary_commands"](
|
|
1587
|
+
failed_work, failed_carrier,
|
|
1588
|
+
)
|
|
1589
|
+
laundered_commands[0]["pass"] = True
|
|
1590
|
+
(failed_devlyn / "spec-verify.results.json").write_text(json.dumps({
|
|
1591
|
+
"commands": laundered_commands, "process_evidence": failed_carrier,
|
|
1592
|
+
}), encoding="utf-8")
|
|
1593
|
+
(failed_devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1594
|
+
(failed_devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1595
|
+
failed_findings, failed_verdicts = read_findings(failed_devlyn)
|
|
1596
|
+
assert failed_verdicts["mechanical"] == "BLOCKED", failed_verdicts
|
|
1597
|
+
assert any(
|
|
1598
|
+
item.get("id") == "verify-mechanical-evidence-invalid"
|
|
1599
|
+
for item in failed_findings
|
|
1600
|
+
), failed_findings
|
|
1601
|
+
assert write_outputs(failed_devlyn, failed_findings, failed_verdicts)["verdict"] == "BLOCKED"
|
|
1602
|
+
|
|
1603
|
+
(failed_devlyn / "spec-verify.results.json").write_text(json.dumps({
|
|
1604
|
+
"commands": [], "process_evidence": None,
|
|
1605
|
+
}), encoding="utf-8")
|
|
1606
|
+
removed_findings, removed_verdicts = read_findings(failed_devlyn)
|
|
1607
|
+
assert removed_verdicts["mechanical"] == "BLOCKED", removed_verdicts
|
|
1608
|
+
assert any(
|
|
1609
|
+
item.get("id") == "verify-mechanical-evidence-invalid"
|
|
1610
|
+
for item in removed_findings
|
|
1611
|
+
), removed_findings
|
|
1612
|
+
print("PASS iter-0112 sealed VERIFY outcome resists mutable-derivative laundering")
|
|
1613
|
+
|
|
1614
|
+
sealed_stdout = sealed_work / carrier["streams"][0]["stdout"]["path"]
|
|
1615
|
+
sealed_stdout.write_bytes(sealed_stdout.read_bytes() + b"altered")
|
|
1616
|
+
altered_findings, altered_verdicts = read_findings(sealed_devlyn)
|
|
1617
|
+
assert altered_verdicts["mechanical"] == "BLOCKED", altered_verdicts
|
|
1618
|
+
assert any(
|
|
1619
|
+
finding.get("id") == "verify-mechanical-evidence-invalid"
|
|
1620
|
+
for finding in altered_findings
|
|
1621
|
+
), altered_findings
|
|
1622
|
+
|
|
1623
|
+
skipped_work = pathlib.Path(tmp) / "skipped-work"
|
|
1624
|
+
skipped_work.mkdir()
|
|
1625
|
+
(skipped_work / "pipeline.state.json").write_text(json.dumps({
|
|
1626
|
+
"engine": "claude",
|
|
1627
|
+
"risk_profile": {"pair_default_enabled": False},
|
|
1628
|
+
"phases": {"verify": {"pair_trigger": {
|
|
1629
|
+
"eligible": False,
|
|
1630
|
+
"reasons": [],
|
|
1631
|
+
"skipped_reason": "user_no_pair",
|
|
1632
|
+
}}},
|
|
1633
|
+
}), encoding="utf-8")
|
|
1634
|
+
(skipped_work / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1635
|
+
(skipped_work / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1636
|
+
generic_primary = skipped_work / "verify-judge.stdout"
|
|
1637
|
+
for generic_output in ("PASS\n", "NEEDS_WORK\n"):
|
|
1638
|
+
generic_primary.write_text(generic_output, encoding="utf-8")
|
|
1639
|
+
generic_findings, generic_verdicts = read_findings(skipped_work)
|
|
1640
|
+
assert generic_verdicts["pair_judge"] is None, generic_verdicts
|
|
1641
|
+
assert not any(
|
|
1642
|
+
finding.get("file") == generic_primary.name for finding in generic_findings
|
|
1643
|
+
), generic_findings
|
|
1644
|
+
(skipped_work / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
1645
|
+
contradiction_findings, contradiction_verdicts = read_findings(skipped_work)
|
|
1646
|
+
assert contradiction_verdicts["pair_judge"] == "BLOCKED", contradiction_verdicts
|
|
1647
|
+
assert any(
|
|
1648
|
+
finding.get("id") == "verify-pair-skipped-output-present"
|
|
1649
|
+
for finding in contradiction_findings
|
|
1650
|
+
), contradiction_findings
|
|
1057
1651
|
findings, source_verdicts = read_findings(devlyn)
|
|
1058
1652
|
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1059
1653
|
write_state(devlyn, summary)
|
|
@@ -1065,12 +1659,92 @@ def self_test() -> int:
|
|
|
1065
1659
|
assert state["phases"]["verify"]["judge_durations_ms"] == {
|
|
1066
1660
|
"judge": 23, "pair_judge": 31,
|
|
1067
1661
|
}, state
|
|
1662
|
+
original_state = (devlyn / "pipeline.state.json").read_bytes()
|
|
1663
|
+
for malformed in (None, {}, {"roles": {}}):
|
|
1664
|
+
bad_state = loads_strict_json(original_state)
|
|
1665
|
+
bad_state["role_resolution"] = malformed
|
|
1666
|
+
(devlyn / "pipeline.state.json").write_text(json.dumps(bad_state))
|
|
1667
|
+
result = subprocess.run([sys.executable, str(pathlib.Path(__file__).resolve()),
|
|
1668
|
+
"--devlyn-dir", str(devlyn), "--write-state"], capture_output=True, text=True)
|
|
1669
|
+
assert result.returncode == 0 and "Traceback" not in result.stderr, result.stderr
|
|
1670
|
+
persisted = loads_strict_json((devlyn / "pipeline.state.json").read_text())
|
|
1671
|
+
assert persisted["phases"]["verify"]["verdict"] == "BLOCKED"
|
|
1672
|
+
assert "verify-role-resolution-invalid" in (devlyn / "verify-merged.findings.jsonl").read_text()
|
|
1673
|
+
(devlyn / "pipeline.state.json").write_bytes(original_state)
|
|
1674
|
+
different_primary = {"engine": "codex", "phases": {"verify": {"engine": "claude"}}}
|
|
1675
|
+
(devlyn / "pipeline.state.json").write_text(json.dumps(different_primary))
|
|
1676
|
+
(devlyn / "verify.primary.timeout.json").write_text(json.dumps({"engine": "claude", "budget_seconds": 600}))
|
|
1677
|
+
marker, violation = read_primary_timeout_marker(devlyn)
|
|
1678
|
+
assert marker is not None and violation is None
|
|
1679
|
+
different_primary["phases"]["verify"]["engine"] = None
|
|
1680
|
+
(devlyn / "pipeline.state.json").write_text(json.dumps(different_primary))
|
|
1681
|
+
assert read_primary_timeout_marker(devlyn)[1] is not None
|
|
1682
|
+
(devlyn / "verify.primary.timeout.json").unlink()
|
|
1683
|
+
(devlyn / "pipeline.state.json").write_bytes(original_state)
|
|
1684
|
+
|
|
1685
|
+
(devlyn / "verify.findings.jsonl").unlink()
|
|
1686
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
1687
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1688
|
+
assert summary["verdict"] == "BLOCKED", summary
|
|
1689
|
+
assert summary["source_verdicts"]["judge"] == "BLOCKED", summary
|
|
1690
|
+
assert any(
|
|
1691
|
+
finding["id"] == "verify-merge-required-source-missing-judge"
|
|
1692
|
+
for finding in findings
|
|
1693
|
+
), findings
|
|
1694
|
+
|
|
1695
|
+
# iter-0113: a valid primary timeout is an explicit BLOCKED source,
|
|
1696
|
+
# never the generic missing-source path or pair-style TIMEOUT.
|
|
1697
|
+
(devlyn / "verify.primary.timeout.json").write_text(
|
|
1698
|
+
json.dumps({"engine": "claude", "budget_seconds": 600}) + "\n",
|
|
1699
|
+
encoding="utf-8",
|
|
1700
|
+
)
|
|
1701
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
1702
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1703
|
+
assert summary["verdict"] == "BLOCKED", summary
|
|
1704
|
+
assert summary["source_verdicts"]["judge"] == "BLOCKED", summary
|
|
1705
|
+
assert any(
|
|
1706
|
+
finding["id"] == "verify-primary-timeout"
|
|
1707
|
+
for finding in findings
|
|
1708
|
+
), findings
|
|
1709
|
+
assert not any(
|
|
1710
|
+
finding["id"] == "verify-merge-required-source-missing-judge"
|
|
1711
|
+
for finding in findings
|
|
1712
|
+
), findings
|
|
1713
|
+
|
|
1714
|
+
(devlyn / "verify.findings.jsonl").write_text(
|
|
1715
|
+
json.dumps({"id": "primary-timeout-high", "severity": "HIGH"}) + "\n",
|
|
1716
|
+
encoding="utf-8",
|
|
1717
|
+
)
|
|
1718
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
1719
|
+
assert source_verdicts["judge"] == "BLOCKED", source_verdicts
|
|
1720
|
+
assert {finding["id"] for finding in findings} >= {
|
|
1721
|
+
"primary-timeout-high", "verify-primary-timeout",
|
|
1722
|
+
}, findings
|
|
1723
|
+
|
|
1724
|
+
for marker, expected_id in (
|
|
1725
|
+
("{\n", "verify-primary-timeout-marker-malformed"),
|
|
1726
|
+
(
|
|
1727
|
+
json.dumps({"engine": "codex", "budget_seconds": 600}) + "\n",
|
|
1728
|
+
"verify-primary-timeout-engine-mismatch",
|
|
1729
|
+
),
|
|
1730
|
+
(
|
|
1731
|
+
json.dumps({"engine": "claude", "budget_seconds": 601}) + "\n",
|
|
1732
|
+
"verify-primary-timeout-budget-mismatch",
|
|
1733
|
+
),
|
|
1734
|
+
):
|
|
1735
|
+
(devlyn / "verify.primary.timeout.json").write_text(marker, encoding="utf-8")
|
|
1736
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
1737
|
+
assert source_verdicts["judge"] == "BLOCKED", source_verdicts
|
|
1738
|
+
assert any(finding["id"] == expected_id for finding in findings), findings
|
|
1739
|
+
(devlyn / "verify.primary.timeout.json").unlink()
|
|
1740
|
+
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1068
1741
|
|
|
1069
1742
|
# iter-0072 Amendment 3: generated schema-v3 runs mechanically prove
|
|
1070
1743
|
# raw-goal persistence and required SURFACE_CLOSE dispatch.
|
|
1071
1744
|
(devlyn / "pipeline.state.json").write_text(
|
|
1072
1745
|
json.dumps({
|
|
1073
1746
|
"version": "3.0",
|
|
1747
|
+
"engine": "claude",
|
|
1074
1748
|
"complexity": "large",
|
|
1075
1749
|
"source": {"type": "generated"},
|
|
1076
1750
|
"phases": {"verify": {"verdict": None, "sub_verdicts": None}},
|
|
@@ -1089,6 +1763,7 @@ def self_test() -> int:
|
|
|
1089
1763
|
(devlyn / "pipeline.state.json").write_text(
|
|
1090
1764
|
json.dumps({
|
|
1091
1765
|
"version": "3.0",
|
|
1766
|
+
"engine": "claude",
|
|
1092
1767
|
"complexity": "large",
|
|
1093
1768
|
"source": {"type": "generated", "goal_path": "", "goal_sha256": "A" * 64},
|
|
1094
1769
|
}),
|
|
@@ -1106,6 +1781,7 @@ def self_test() -> int:
|
|
|
1106
1781
|
(devlyn / "pipeline.state.json").write_text(
|
|
1107
1782
|
json.dumps({
|
|
1108
1783
|
"version": "3.0",
|
|
1784
|
+
"engine": "claude",
|
|
1109
1785
|
"complexity": "medium",
|
|
1110
1786
|
"source": generated_source,
|
|
1111
1787
|
"phases": {
|
|
@@ -1134,6 +1810,7 @@ def self_test() -> int:
|
|
|
1134
1810
|
(devlyn / "pipeline.state.json").write_text(
|
|
1135
1811
|
json.dumps({
|
|
1136
1812
|
"version": "3.0",
|
|
1813
|
+
"engine": "claude",
|
|
1137
1814
|
"complexity": "medium",
|
|
1138
1815
|
"source": generated_source,
|
|
1139
1816
|
"phases": {"surface_close": surface_entry},
|
|
@@ -1145,6 +1822,7 @@ def self_test() -> int:
|
|
|
1145
1822
|
(devlyn / "pipeline.state.json").write_text(
|
|
1146
1823
|
json.dumps({
|
|
1147
1824
|
"version": "3.0",
|
|
1825
|
+
"engine": "claude",
|
|
1148
1826
|
"complexity": "medium",
|
|
1149
1827
|
"source": generated_source,
|
|
1150
1828
|
"phases": {
|
|
@@ -1161,15 +1839,15 @@ def self_test() -> int:
|
|
|
1161
1839
|
|
|
1162
1840
|
for state_shape in (
|
|
1163
1841
|
{
|
|
1164
|
-
"version": "3.0", "complexity": "medium",
|
|
1842
|
+
"version": "3.0", "engine": "claude", "complexity": "medium",
|
|
1165
1843
|
"source": {"type": "spec"}, "phases": {"surface_close": None},
|
|
1166
1844
|
},
|
|
1167
1845
|
{
|
|
1168
|
-
"version": "3.0", "complexity": "large",
|
|
1846
|
+
"version": "3.0", "engine": "claude", "complexity": "large",
|
|
1169
1847
|
"source": generated_source, "phases": {"surface_close": None},
|
|
1170
1848
|
},
|
|
1171
1849
|
{
|
|
1172
|
-
"version": "2.0", "complexity": "medium",
|
|
1850
|
+
"version": "2.0", "engine": "claude", "complexity": "medium",
|
|
1173
1851
|
"source": {"type": "generated"}, "phases": {"surface_close": None},
|
|
1174
1852
|
},
|
|
1175
1853
|
):
|
|
@@ -1184,6 +1862,7 @@ def self_test() -> int:
|
|
|
1184
1862
|
(devlyn / "pipeline.state.json").write_text(
|
|
1185
1863
|
json.dumps({
|
|
1186
1864
|
"version": "3.0",
|
|
1865
|
+
"engine": "claude",
|
|
1187
1866
|
"mode": "spec",
|
|
1188
1867
|
"risk_profile": {"pair_default_enabled": True},
|
|
1189
1868
|
"phases": {
|
|
@@ -1211,7 +1890,7 @@ def self_test() -> int:
|
|
|
1211
1890
|
# A non-null, non-dict sub_verdicts is corrupted state, not a legal
|
|
1212
1891
|
# placeholder — write_state() must fail loud, not silently coerce it.
|
|
1213
1892
|
(devlyn / "pipeline.state.json").write_text(
|
|
1214
|
-
json.dumps({"phases": {"verify": {"verdict": None, "sub_verdicts": "corrupt"}}}),
|
|
1893
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": None, "sub_verdicts": "corrupt"}}}),
|
|
1215
1894
|
encoding="utf-8",
|
|
1216
1895
|
)
|
|
1217
1896
|
findings, source_verdicts = read_findings(devlyn)
|
|
@@ -1225,6 +1904,7 @@ def self_test() -> int:
|
|
|
1225
1904
|
|
|
1226
1905
|
(devlyn / "pipeline.state.json").write_text(
|
|
1227
1906
|
json.dumps({
|
|
1907
|
+
"engine": "claude",
|
|
1228
1908
|
"phases": {
|
|
1229
1909
|
"verify": {
|
|
1230
1910
|
"verdict": "PASS",
|
|
@@ -1247,6 +1927,7 @@ def self_test() -> int:
|
|
|
1247
1927
|
json.dumps({"id": "p1", "severity": "HIGH"}) + "\n",
|
|
1248
1928
|
encoding="utf-8",
|
|
1249
1929
|
)
|
|
1930
|
+
(devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
1250
1931
|
findings, source_verdicts = read_findings(devlyn)
|
|
1251
1932
|
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1252
1933
|
write_state(devlyn, summary)
|
|
@@ -1255,6 +1936,7 @@ def self_test() -> int:
|
|
|
1255
1936
|
assert state["phases"]["verify"]["verdict"] == "NEEDS_WORK", state
|
|
1256
1937
|
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "NEEDS_WORK", state
|
|
1257
1938
|
assert (devlyn / "verify-merged.findings.jsonl").read_text(encoding="utf-8")
|
|
1939
|
+
(devlyn / "codex-judge.stdout").unlink()
|
|
1258
1940
|
(devlyn / "verify.findings.jsonl").write_text(
|
|
1259
1941
|
'{"id":"nan","severity":NaN}\n',
|
|
1260
1942
|
encoding="utf-8",
|
|
@@ -1268,7 +1950,7 @@ def self_test() -> int:
|
|
|
1268
1950
|
for finding in findings
|
|
1269
1951
|
), findings
|
|
1270
1952
|
(devlyn / "pipeline.state.json").write_text(
|
|
1271
|
-
json.dumps({"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1953
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1272
1954
|
encoding="utf-8",
|
|
1273
1955
|
)
|
|
1274
1956
|
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
@@ -1296,6 +1978,10 @@ def self_test() -> int:
|
|
|
1296
1978
|
# Engine-neutral stdout contract (iter-0060): a Claude pair-judge
|
|
1297
1979
|
# capture (claude-judge.stdout, adapters/claude.md ## Invocation)
|
|
1298
1980
|
# binds the same emission contract as the Codex one.
|
|
1981
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
1982
|
+
json.dumps({"engine": "codex", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1983
|
+
encoding="utf-8",
|
|
1984
|
+
)
|
|
1299
1985
|
(devlyn / "claude-judge.stdout").write_text(
|
|
1300
1986
|
json.dumps({"id": "clj1", "severity": "HIGH"}) + "\n",
|
|
1301
1987
|
encoding="utf-8",
|
|
@@ -1314,6 +2000,7 @@ def self_test() -> int:
|
|
|
1314
2000
|
# records TIMEOUT and leaves the merged verdict to mechanical+primary.
|
|
1315
2001
|
(devlyn / "pipeline.state.json").write_text(
|
|
1316
2002
|
json.dumps({
|
|
2003
|
+
"engine": "codex",
|
|
1317
2004
|
"phases": {
|
|
1318
2005
|
"verify": {
|
|
1319
2006
|
"verdict": "PASS",
|
|
@@ -1353,7 +2040,7 @@ def self_test() -> int:
|
|
|
1353
2040
|
|
|
1354
2041
|
# iter-0065 case 2: canonical pair findings still bind after timeout.
|
|
1355
2042
|
(devlyn / "pipeline.state.json").write_text(
|
|
1356
|
-
json.dumps({"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2043
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1357
2044
|
encoding="utf-8",
|
|
1358
2045
|
)
|
|
1359
2046
|
(devlyn / "verify.pair.findings.jsonl").write_text(
|
|
@@ -1375,7 +2062,7 @@ def self_test() -> int:
|
|
|
1375
2062
|
# iter-0065 case 2b: stdout-only HIGH still blocks on emission contract;
|
|
1376
2063
|
# timeout never converts an observed finding into a solo pass.
|
|
1377
2064
|
(devlyn / "pipeline.state.json").write_text(
|
|
1378
|
-
json.dumps({"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2065
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1379
2066
|
encoding="utf-8",
|
|
1380
2067
|
)
|
|
1381
2068
|
(devlyn / "codex-judge.stdout").write_text(
|
|
@@ -1397,9 +2084,125 @@ def self_test() -> int:
|
|
|
1397
2084
|
(devlyn / "codex-judge.stdout").unlink()
|
|
1398
2085
|
(devlyn / "verify.pair.timeout.json").unlink()
|
|
1399
2086
|
|
|
2087
|
+
# iter-0106: a budget abort can truncate the whole-message stream before
|
|
2088
|
+
# its terminal result. With a valid marker that capture stays TIMEOUT.
|
|
2089
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
2090
|
+
json.dumps({
|
|
2091
|
+
"engine": "claude",
|
|
2092
|
+
"phases": {
|
|
2093
|
+
"verify": {
|
|
2094
|
+
"verdict": "PASS",
|
|
2095
|
+
"sub_verdicts": {},
|
|
2096
|
+
"pair_trigger": {
|
|
2097
|
+
"eligible": True,
|
|
2098
|
+
"reasons": ["judge.warning"],
|
|
2099
|
+
"skipped_reason": None,
|
|
2100
|
+
},
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
}),
|
|
2104
|
+
encoding="utf-8",
|
|
2105
|
+
)
|
|
2106
|
+
partial_stream = (
|
|
2107
|
+
'{"type":"system","subtype":"init","session_id":"s106"}\n'
|
|
2108
|
+
'{"type":"assistant","session_id":"s106","message":{"stop_reason":null,'
|
|
2109
|
+
'"content":[{"type":"text","text":"reading the diff"}]}}\n'
|
|
2110
|
+
)
|
|
2111
|
+
(devlyn / "grok-judge.stdout").write_text(partial_stream, encoding="utf-8")
|
|
2112
|
+
(devlyn / "verify.pair.timeout.json").write_text(
|
|
2113
|
+
json.dumps({"engine": "grok", "budget_seconds": 600}),
|
|
2114
|
+
encoding="utf-8",
|
|
2115
|
+
)
|
|
2116
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2117
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2118
|
+
assert summary["verdict"] == "PASS", summary
|
|
2119
|
+
assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
|
|
2120
|
+
assert summary["pair_timeout"] == {"engine": "grok", "budget_seconds": 600}, summary
|
|
2121
|
+
assert not findings, findings
|
|
2122
|
+
|
|
2123
|
+
# A kill may cut the final JSON record mid-write; one malformed tail is
|
|
2124
|
+
# still a valid prefix, while malformed/unknown complete records are not.
|
|
2125
|
+
(devlyn / "grok-judge.stdout").write_text(
|
|
2126
|
+
partial_stream + '{"type":"assistant","session_id":"s106"',
|
|
2127
|
+
encoding="utf-8",
|
|
2128
|
+
)
|
|
2129
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2130
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2131
|
+
assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
|
|
2132
|
+
assert not findings, findings
|
|
2133
|
+
|
|
2134
|
+
for invalid_prefix in (
|
|
2135
|
+
'{"type":"system","subtype":"init"}\n',
|
|
2136
|
+
partial_stream + "trailing narration\n",
|
|
2137
|
+
partial_stream
|
|
2138
|
+
+ '{"type":"user","session_id":"other"}\n',
|
|
2139
|
+
partial_stream
|
|
2140
|
+
+ '{"type":"stream_event","session_id":"s106"}\n',
|
|
2141
|
+
partial_stream
|
|
2142
|
+
+ '{"type":"system","subtype":"compact_boundary","session_id":"s106"}\n',
|
|
2143
|
+
partial_stream
|
|
2144
|
+
+ '{"type":"assistant"\n'
|
|
2145
|
+
+ '{"type":"user","session_id":"s106"}\n',
|
|
2146
|
+
):
|
|
2147
|
+
(devlyn / "grok-judge.stdout").write_text(invalid_prefix, encoding="utf-8")
|
|
2148
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2149
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2150
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2151
|
+
assert any(
|
|
2152
|
+
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2153
|
+
for finding in findings
|
|
2154
|
+
), findings
|
|
2155
|
+
|
|
2156
|
+
# iter-0106 control: the marker conserves TIMEOUT only for that shape. A
|
|
2157
|
+
# stream that reached its result but welded narration into the terminal
|
|
2158
|
+
# message stays BLOCKED on the emission contract.
|
|
2159
|
+
(devlyn / "grok-judge.stdout").write_text(
|
|
2160
|
+
'{"type":"system","subtype":"init","session_id":"s106"}\n'
|
|
2161
|
+
+ json.dumps({
|
|
2162
|
+
"type": "assistant",
|
|
2163
|
+
"session_id": "s106",
|
|
2164
|
+
"message": {
|
|
2165
|
+
"stop_reason": "end_turn",
|
|
2166
|
+
"content": [{"type": "text", "text": "I reviewed the diff.\nPASS"}],
|
|
2167
|
+
},
|
|
2168
|
+
}) + "\n"
|
|
2169
|
+
+ json.dumps({
|
|
2170
|
+
"type": "result",
|
|
2171
|
+
"subtype": "success",
|
|
2172
|
+
"is_error": False,
|
|
2173
|
+
"stop_reason": "end_turn",
|
|
2174
|
+
"session_id": "s106",
|
|
2175
|
+
"result": "I reviewed the diff.\nPASS",
|
|
2176
|
+
}) + "\n",
|
|
2177
|
+
encoding="utf-8",
|
|
2178
|
+
)
|
|
2179
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2180
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2181
|
+
assert summary["verdict"] == "BLOCKED", summary
|
|
2182
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2183
|
+
assert any(
|
|
2184
|
+
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2185
|
+
for finding in findings
|
|
2186
|
+
), findings
|
|
2187
|
+
(devlyn / "verify.pair.timeout.json").unlink()
|
|
2188
|
+
|
|
2189
|
+
# iter-0106 control: without a marker the same truncated stream is a
|
|
2190
|
+
# plain emission-contract violation.
|
|
2191
|
+
(devlyn / "grok-judge.stdout").write_text(partial_stream, encoding="utf-8")
|
|
2192
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2193
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2194
|
+
assert summary["verdict"] == "BLOCKED", summary
|
|
2195
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2196
|
+
assert any(
|
|
2197
|
+
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2198
|
+
for finding in findings
|
|
2199
|
+
), findings
|
|
2200
|
+
(devlyn / "grok-judge.stdout").unlink()
|
|
2201
|
+
|
|
1400
2202
|
# iter-0065 case 3: without a marker, required empty pair output remains BLOCKED.
|
|
1401
2203
|
(devlyn / "pipeline.state.json").write_text(
|
|
1402
2204
|
json.dumps({
|
|
2205
|
+
"engine": "codex",
|
|
1403
2206
|
"phases": {
|
|
1404
2207
|
"verify": {
|
|
1405
2208
|
"verdict": "PASS",
|
|
@@ -1426,7 +2229,7 @@ def self_test() -> int:
|
|
|
1426
2229
|
|
|
1427
2230
|
# iter-0065 malformed timeout markers fail closed as a CRITICAL pair blocker.
|
|
1428
2231
|
(devlyn / "pipeline.state.json").write_text(
|
|
1429
|
-
json.dumps({"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2232
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1430
2233
|
encoding="utf-8",
|
|
1431
2234
|
)
|
|
1432
2235
|
(devlyn / "verify.pair.timeout.json").write_text(
|
|
@@ -1445,6 +2248,7 @@ def self_test() -> int:
|
|
|
1445
2248
|
(devlyn / "verify.pair.timeout.json").unlink()
|
|
1446
2249
|
(devlyn / "pipeline.state.json").write_text(
|
|
1447
2250
|
json.dumps({
|
|
2251
|
+
"engine": "claude",
|
|
1448
2252
|
"phases": {
|
|
1449
2253
|
"verify": {
|
|
1450
2254
|
"verdict": "PASS",
|
|
@@ -1472,6 +2276,7 @@ def self_test() -> int:
|
|
|
1472
2276
|
|
|
1473
2277
|
(devlyn / "pipeline.state.json").write_text(
|
|
1474
2278
|
json.dumps({
|
|
2279
|
+
"engine": "claude",
|
|
1475
2280
|
"mode": "spec",
|
|
1476
2281
|
"risk_profile": {
|
|
1477
2282
|
"high_risk": True,
|
|
@@ -1492,6 +2297,7 @@ def self_test() -> int:
|
|
|
1492
2297
|
|
|
1493
2298
|
(devlyn / "pipeline.state.json").write_text(
|
|
1494
2299
|
json.dumps({
|
|
2300
|
+
"engine": "claude",
|
|
1495
2301
|
"mode": "spec",
|
|
1496
2302
|
"risk_profile": "enabled",
|
|
1497
2303
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1509,6 +2315,7 @@ def self_test() -> int:
|
|
|
1509
2315
|
|
|
1510
2316
|
(devlyn / "pipeline.state.json").write_text(
|
|
1511
2317
|
json.dumps({
|
|
2318
|
+
"engine": "claude",
|
|
1512
2319
|
"mode": "spec",
|
|
1513
2320
|
"risk_profile": {
|
|
1514
2321
|
"high_risk": True,
|
|
@@ -1530,6 +2337,7 @@ def self_test() -> int:
|
|
|
1530
2337
|
|
|
1531
2338
|
(devlyn / "pipeline.state.json").write_text(
|
|
1532
2339
|
json.dumps({
|
|
2340
|
+
"engine": "claude",
|
|
1533
2341
|
"mode": "spec",
|
|
1534
2342
|
"risk_profile": {
|
|
1535
2343
|
"high_risk": True,
|
|
@@ -1552,6 +2360,7 @@ def self_test() -> int:
|
|
|
1552
2360
|
|
|
1553
2361
|
(devlyn / "pipeline.state.json").write_text(
|
|
1554
2362
|
json.dumps({
|
|
2363
|
+
"engine": "claude",
|
|
1555
2364
|
"mode": "spec",
|
|
1556
2365
|
"pair_verify": True,
|
|
1557
2366
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1569,6 +2378,7 @@ def self_test() -> int:
|
|
|
1569
2378
|
|
|
1570
2379
|
(devlyn / "pipeline.state.json").write_text(
|
|
1571
2380
|
json.dumps({
|
|
2381
|
+
"engine": "claude",
|
|
1572
2382
|
"mode": "spec",
|
|
1573
2383
|
"complexity": "large",
|
|
1574
2384
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1591,6 +2401,7 @@ def self_test() -> int:
|
|
|
1591
2401
|
)
|
|
1592
2402
|
(devlyn / "pipeline.state.json").write_text(
|
|
1593
2403
|
json.dumps({
|
|
2404
|
+
"engine": "claude",
|
|
1594
2405
|
"mode": "spec",
|
|
1595
2406
|
"source": {"spec_path": str(spec_path)},
|
|
1596
2407
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1628,6 +2439,7 @@ def self_test() -> int:
|
|
|
1628
2439
|
) is False
|
|
1629
2440
|
(devlyn / "pipeline.state.json").write_text(
|
|
1630
2441
|
json.dumps({
|
|
2442
|
+
"engine": "claude",
|
|
1631
2443
|
"mode": "spec",
|
|
1632
2444
|
"source": {"spec_path": str(spec_path)},
|
|
1633
2445
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1676,6 +2488,7 @@ def self_test() -> int:
|
|
|
1676
2488
|
) is True
|
|
1677
2489
|
(devlyn / "pipeline.state.json").write_text(
|
|
1678
2490
|
json.dumps({
|
|
2491
|
+
"engine": "claude",
|
|
1679
2492
|
"mode": "spec",
|
|
1680
2493
|
"source": {"spec_path": str(spec_path)},
|
|
1681
2494
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1693,6 +2506,7 @@ def self_test() -> int:
|
|
|
1693
2506
|
|
|
1694
2507
|
(devlyn / "pipeline.state.json").write_text(
|
|
1695
2508
|
json.dumps({
|
|
2509
|
+
"engine": "claude",
|
|
1696
2510
|
"mode": "spec",
|
|
1697
2511
|
"source": {"spec_path": str(spec_path)},
|
|
1698
2512
|
"risk_profile": {
|
|
@@ -1733,6 +2547,7 @@ def self_test() -> int:
|
|
|
1733
2547
|
) is True
|
|
1734
2548
|
(devlyn / "pipeline.state.json").write_text(
|
|
1735
2549
|
json.dumps({
|
|
2550
|
+
"engine": "claude",
|
|
1736
2551
|
"mode": "free-form",
|
|
1737
2552
|
"source": {"criteria_path": str(criteria_path)},
|
|
1738
2553
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1763,6 +2578,7 @@ def self_test() -> int:
|
|
|
1763
2578
|
|
|
1764
2579
|
(devlyn / "pipeline.state.json").write_text(
|
|
1765
2580
|
json.dumps({
|
|
2581
|
+
"engine": "claude",
|
|
1766
2582
|
"phases": {
|
|
1767
2583
|
"verify": {
|
|
1768
2584
|
"verdict": "PASS",
|
|
@@ -1787,6 +2603,7 @@ def self_test() -> int:
|
|
|
1787
2603
|
|
|
1788
2604
|
(devlyn / "pipeline.state.json").write_text(
|
|
1789
2605
|
json.dumps({
|
|
2606
|
+
"engine": "claude",
|
|
1790
2607
|
"phases": {
|
|
1791
2608
|
"verify": {
|
|
1792
2609
|
"verdict": "PASS",
|
|
@@ -1811,6 +2628,7 @@ def self_test() -> int:
|
|
|
1811
2628
|
|
|
1812
2629
|
(devlyn / "pipeline.state.json").write_text(
|
|
1813
2630
|
json.dumps({
|
|
2631
|
+
"engine": "claude",
|
|
1814
2632
|
"phases": {
|
|
1815
2633
|
"verify": {
|
|
1816
2634
|
"verdict": "PASS",
|
|
@@ -1836,6 +2654,7 @@ def self_test() -> int:
|
|
|
1836
2654
|
|
|
1837
2655
|
(devlyn / "pipeline.state.json").write_text(
|
|
1838
2656
|
json.dumps({
|
|
2657
|
+
"engine": "claude",
|
|
1839
2658
|
"phases": {
|
|
1840
2659
|
"verify": {
|
|
1841
2660
|
"verdict": "PASS",
|
|
@@ -1861,6 +2680,7 @@ def self_test() -> int:
|
|
|
1861
2680
|
|
|
1862
2681
|
(devlyn / "pipeline.state.json").write_text(
|
|
1863
2682
|
json.dumps({
|
|
2683
|
+
"engine": "claude",
|
|
1864
2684
|
"phases": {
|
|
1865
2685
|
"verify": {
|
|
1866
2686
|
"verdict": "PASS",
|
|
@@ -1886,6 +2706,7 @@ def self_test() -> int:
|
|
|
1886
2706
|
|
|
1887
2707
|
(devlyn / "pipeline.state.json").write_text(
|
|
1888
2708
|
json.dumps({
|
|
2709
|
+
"engine": "claude",
|
|
1889
2710
|
"phases": {
|
|
1890
2711
|
"verify": {
|
|
1891
2712
|
"verdict": "PASS",
|
|
@@ -1910,6 +2731,7 @@ def self_test() -> int:
|
|
|
1910
2731
|
|
|
1911
2732
|
(devlyn / "pipeline.state.json").write_text(
|
|
1912
2733
|
json.dumps({
|
|
2734
|
+
"engine": "claude",
|
|
1913
2735
|
"phases": {
|
|
1914
2736
|
"verify": {
|
|
1915
2737
|
"verdict": "PASS",
|
|
@@ -1934,6 +2756,7 @@ def self_test() -> int:
|
|
|
1934
2756
|
|
|
1935
2757
|
(devlyn / "pipeline.state.json").write_text(
|
|
1936
2758
|
json.dumps({
|
|
2759
|
+
"engine": "claude",
|
|
1937
2760
|
"phases": {
|
|
1938
2761
|
"verify": {
|
|
1939
2762
|
"verdict": "PASS",
|
|
@@ -1958,6 +2781,7 @@ def self_test() -> int:
|
|
|
1958
2781
|
|
|
1959
2782
|
(devlyn / "pipeline.state.json").write_text(
|
|
1960
2783
|
json.dumps({
|
|
2784
|
+
"engine": "claude",
|
|
1961
2785
|
"phases": {
|
|
1962
2786
|
"verify": {
|
|
1963
2787
|
"verdict": "PASS",
|
|
@@ -1982,6 +2806,7 @@ def self_test() -> int:
|
|
|
1982
2806
|
|
|
1983
2807
|
(devlyn / "pipeline.state.json").write_text(
|
|
1984
2808
|
json.dumps({
|
|
2809
|
+
"engine": "claude",
|
|
1985
2810
|
"mode": "spec",
|
|
1986
2811
|
"risk_profile": {
|
|
1987
2812
|
"high_risk": True,
|
|
@@ -2013,6 +2838,7 @@ def self_test() -> int:
|
|
|
2013
2838
|
|
|
2014
2839
|
(devlyn / "pipeline.state.json").write_text(
|
|
2015
2840
|
json.dumps({
|
|
2841
|
+
"engine": "claude",
|
|
2016
2842
|
"mode": "spec",
|
|
2017
2843
|
"risk_profile": {
|
|
2018
2844
|
"high_risk": True,
|
|
@@ -2044,6 +2870,7 @@ def self_test() -> int:
|
|
|
2044
2870
|
|
|
2045
2871
|
(devlyn / "pipeline.state.json").write_text(
|
|
2046
2872
|
json.dumps({
|
|
2873
|
+
"engine": "claude",
|
|
2047
2874
|
"pair_verify": True,
|
|
2048
2875
|
"risk_profile": {
|
|
2049
2876
|
"high_risk": True,
|
|
@@ -2074,6 +2901,7 @@ def self_test() -> int:
|
|
|
2074
2901
|
|
|
2075
2902
|
(devlyn / "pipeline.state.json").write_text(
|
|
2076
2903
|
json.dumps({
|
|
2904
|
+
"engine": "claude",
|
|
2077
2905
|
"mode": "spec",
|
|
2078
2906
|
"risk_profile": {
|
|
2079
2907
|
"high_risk": True,
|
|
@@ -2104,6 +2932,7 @@ def self_test() -> int:
|
|
|
2104
2932
|
|
|
2105
2933
|
(devlyn / "pipeline.state.json").write_text(
|
|
2106
2934
|
json.dumps({
|
|
2935
|
+
"engine": "claude",
|
|
2107
2936
|
"phases": {
|
|
2108
2937
|
"verify": {
|
|
2109
2938
|
"verdict": "PASS",
|
|
@@ -2128,6 +2957,7 @@ def self_test() -> int:
|
|
|
2128
2957
|
|
|
2129
2958
|
(devlyn / "pipeline.state.json").write_text(
|
|
2130
2959
|
json.dumps({
|
|
2960
|
+
"engine": "claude",
|
|
2131
2961
|
"phases": {
|
|
2132
2962
|
"verify": {
|
|
2133
2963
|
"verdict": "PASS",
|
|
@@ -2152,6 +2982,7 @@ def self_test() -> int:
|
|
|
2152
2982
|
|
|
2153
2983
|
(devlyn / "pipeline.state.json").write_text(
|
|
2154
2984
|
json.dumps({
|
|
2985
|
+
"engine": "claude",
|
|
2155
2986
|
"phases": {
|
|
2156
2987
|
"verify": {
|
|
2157
2988
|
"verdict": "PASS",
|
|
@@ -2185,10 +3016,34 @@ def self_test() -> int:
|
|
|
2185
3016
|
finding.get("id") == "verify-pair-trigger-mechanical-blocker-unsupported"
|
|
2186
3017
|
for finding in findings
|
|
2187
3018
|
), findings
|
|
3019
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
3020
|
+
json.dumps({
|
|
3021
|
+
"engine": "claude",
|
|
3022
|
+
"phases": {
|
|
3023
|
+
"verify": {
|
|
3024
|
+
"verdict": None,
|
|
3025
|
+
"sub_verdicts": None,
|
|
3026
|
+
"judge_durations_ms": {"judge": None, "pair_judge": None},
|
|
3027
|
+
}
|
|
3028
|
+
}
|
|
3029
|
+
}),
|
|
3030
|
+
encoding="utf-8",
|
|
3031
|
+
)
|
|
3032
|
+
(devlyn / "verify.findings.jsonl").unlink()
|
|
3033
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3034
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3035
|
+
write_state(devlyn, summary)
|
|
3036
|
+
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
3037
|
+
assert summary["verdict"] == "NEEDS_WORK", summary
|
|
3038
|
+
assert summary["source_verdicts"]["judge"] is None, summary
|
|
3039
|
+
assert state["phases"]["verify"]["verdict"] == "NEEDS_WORK", state
|
|
3040
|
+
assert state["phases"]["verify"]["sub_verdicts"]["judge"] is None, state
|
|
3041
|
+
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
2188
3042
|
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
2189
3043
|
|
|
2190
3044
|
(devlyn / "pipeline.state.json").write_text(
|
|
2191
3045
|
json.dumps({
|
|
3046
|
+
"engine": "claude",
|
|
2192
3047
|
"phases": {
|
|
2193
3048
|
"verify": {
|
|
2194
3049
|
"verdict": "PASS",
|
|
@@ -2220,6 +3075,7 @@ def self_test() -> int:
|
|
|
2220
3075
|
(devlyn / "pipeline.state.json").write_text(
|
|
2221
3076
|
json.dumps({
|
|
2222
3077
|
"version": "2.0",
|
|
3078
|
+
"engine": "claude",
|
|
2223
3079
|
"mode": "spec",
|
|
2224
3080
|
"pair_verify": True,
|
|
2225
3081
|
"phases": {
|
|
@@ -2249,6 +3105,7 @@ def self_test() -> int:
|
|
|
2249
3105
|
(devlyn / "pipeline.state.json").write_text(
|
|
2250
3106
|
json.dumps({
|
|
2251
3107
|
"version": "3.0",
|
|
3108
|
+
"engine": "claude",
|
|
2252
3109
|
"mode": "spec",
|
|
2253
3110
|
"pair_verify": True,
|
|
2254
3111
|
"phases": {
|
|
@@ -2269,6 +3126,7 @@ def self_test() -> int:
|
|
|
2269
3126
|
json.dumps({"id": "p-preknown", "severity": "LOW"}) + "\n",
|
|
2270
3127
|
encoding="utf-8",
|
|
2271
3128
|
)
|
|
3129
|
+
(devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
2272
3130
|
findings, source_verdicts = read_findings(devlyn)
|
|
2273
3131
|
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2274
3132
|
assert summary["verdict"] == "BLOCKED", summary
|
|
@@ -2288,12 +3146,14 @@ def self_test() -> int:
|
|
|
2288
3146
|
assert '"id":"p-preknown"' in (
|
|
2289
3147
|
devlyn / "verify-merged.findings.jsonl"
|
|
2290
3148
|
).read_text(encoding="utf-8"), findings
|
|
3149
|
+
(devlyn / "codex-judge.stdout").unlink()
|
|
2291
3150
|
|
|
2292
3151
|
# Self-test: archived-v2 sequential primary blocker remains legal.
|
|
2293
3152
|
(devlyn / "verify.pair.findings.jsonl").unlink()
|
|
2294
3153
|
(devlyn / "pipeline.state.json").write_text(
|
|
2295
3154
|
json.dumps({
|
|
2296
3155
|
"version": "2.0",
|
|
3156
|
+
"engine": "claude",
|
|
2297
3157
|
"mode": "spec",
|
|
2298
3158
|
"phases": {
|
|
2299
3159
|
"verify": {
|
|
@@ -2336,6 +3196,7 @@ def self_test() -> int:
|
|
|
2336
3196
|
|
|
2337
3197
|
(devlyn / "pipeline.state.json").write_text(
|
|
2338
3198
|
json.dumps({
|
|
3199
|
+
"engine": "claude",
|
|
2339
3200
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2340
3201
|
"verify": {
|
|
2341
3202
|
"pair_trigger": {
|
|
@@ -2359,7 +3220,7 @@ def self_test() -> int:
|
|
|
2359
3220
|
# PASS stdout from a claude judge promotes pair_judge null -> PASS.
|
|
2360
3221
|
(devlyn / "verify.pair.findings.jsonl").unlink()
|
|
2361
3222
|
(devlyn / "pipeline.state.json").write_text(
|
|
2362
|
-
json.dumps({"phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
|
|
3223
|
+
json.dumps({"engine": "codex", "phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
|
|
2363
3224
|
encoding="utf-8",
|
|
2364
3225
|
)
|
|
2365
3226
|
(devlyn / "claude-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
@@ -2369,6 +3230,47 @@ def self_test() -> int:
|
|
|
2369
3230
|
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
2370
3231
|
assert summary["verdict"] == "PASS", summary
|
|
2371
3232
|
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "PASS", state
|
|
3233
|
+
(devlyn / "claude-judge.stdout").unlink()
|
|
3234
|
+
|
|
3235
|
+
# The primary capture must not be misattributed to the OTHER-engine
|
|
3236
|
+
# pair seat merely because both adapters use *-judge.stdout names.
|
|
3237
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
3238
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
|
|
3239
|
+
encoding="utf-8",
|
|
3240
|
+
)
|
|
3241
|
+
(devlyn / "claude-judge.stdout").write_text("primary prose\n", encoding="utf-8")
|
|
3242
|
+
(devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
3243
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3244
|
+
assert source_verdicts["pair_judge"] == "PASS", source_verdicts
|
|
3245
|
+
assert not any(finding["source"] == "pair_judge" for finding in findings), findings
|
|
3246
|
+
(devlyn / "codex-judge.stdout").write_text("pair prose\n", encoding="utf-8")
|
|
3247
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3248
|
+
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3249
|
+
assert any(finding["id"] == "verify-pair-emission-contract-violated" for finding in findings)
|
|
3250
|
+
(devlyn / "claude-judge.stdout").unlink()
|
|
3251
|
+
(devlyn / "codex-judge.stdout").unlink()
|
|
3252
|
+
|
|
3253
|
+
# A primary capture cannot become pair evidence when the state needed
|
|
3254
|
+
# to identify the primary seat is absent, incomplete, or malformed.
|
|
3255
|
+
(devlyn / "claude-judge.stdout").write_text("primary prose\n", encoding="utf-8")
|
|
3256
|
+
(devlyn / "pipeline.state.json").unlink()
|
|
3257
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3258
|
+
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3259
|
+
assert any(finding["id"] == "verify-state-missing" for finding in findings), findings
|
|
3260
|
+
assert not any(finding.get("file") == "claude-judge.stdout" for finding in findings), findings
|
|
3261
|
+
|
|
3262
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
3263
|
+
json.dumps({"version": "3.0", "phases": {"verify": {}}}), encoding="utf-8",
|
|
3264
|
+
)
|
|
3265
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3266
|
+
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3267
|
+
assert any(finding.get("rule_id") == "verify.state.engine-malformed" for finding in findings), findings
|
|
3268
|
+
|
|
3269
|
+
(devlyn / "pipeline.state.json").write_text("{", encoding="utf-8")
|
|
3270
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3271
|
+
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3272
|
+
assert any(finding["id"] == "verify-pair-trigger-state-malformed" for finding in findings), findings
|
|
3273
|
+
(devlyn / "claude-judge.stdout").unlink()
|
|
2372
3274
|
|
|
2373
3275
|
# iter-0083: canonical summary verdict conservation.
|
|
2374
3276
|
iter_0083_paths = (
|
|
@@ -2398,6 +3300,11 @@ def self_test() -> int:
|
|
|
2398
3300
|
timeout: bool = False,
|
|
2399
3301
|
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
|
2400
3302
|
iter_0083_reset()
|
|
3303
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
3304
|
+
json.dumps({"engine": "claude", "phases": {"verify": {}}}), encoding="utf-8",
|
|
3305
|
+
)
|
|
3306
|
+
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
3307
|
+
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
2401
3308
|
if carrier is not None:
|
|
2402
3309
|
content = (
|
|
2403
3310
|
json.dumps({"id": "iter-0083", "severity": severity}) + "\n"
|
|
@@ -2418,7 +3325,7 @@ def self_test() -> int:
|
|
|
2418
3325
|
)
|
|
2419
3326
|
if timeout:
|
|
2420
3327
|
(devlyn / "verify.pair.timeout.json").write_text(
|
|
2421
|
-
json.dumps({"engine": "
|
|
3328
|
+
json.dumps({"engine": "codex", "budget_seconds": 600}),
|
|
2422
3329
|
encoding="utf-8",
|
|
2423
3330
|
)
|
|
2424
3331
|
case_findings, case_source_verdicts = read_findings(devlyn)
|