devlyn-cli 2.10.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -9
- package/CLAUDE.md +11 -15
- package/README.md +31 -7
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +25 -0
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +10 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +30 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +32 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +44 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +16 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/scripts/run-fixture.sh +11 -1
- package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +83 -1
- package/bin/devlyn.js +59 -9
- package/config/skills/_shared/.ruff.toml +2 -0
- package/config/skills/_shared/adapters/claude.md +9 -1
- package/config/skills/_shared/adapters/codex.md +5 -0
- package/config/skills/_shared/adapters/grok.md +37 -13
- package/config/skills/_shared/archive_run.py +738 -56
- package/config/skills/_shared/codex-config.md +19 -4
- package/config/skills/_shared/codex-monitored.sh +81 -12
- package/config/skills/_shared/collect-codex-findings.py +419 -60
- package/config/skills/_shared/engine-preflight.md +8 -7
- package/config/skills/_shared/expected.schema.json +68 -0
- package/config/skills/_shared/finish-gate.py +12 -11
- package/config/skills/_shared/grok-anchor-guard.py +78 -0
- package/config/skills/_shared/invocation-receipt.py +1041 -0
- package/config/skills/_shared/judge-output-parser.py +200 -0
- package/config/skills/_shared/judge-role-evidence.py +273 -0
- package/config/skills/_shared/phase-prompt-render.py +181 -0
- package/config/skills/_shared/process-evidence.py +909 -0
- package/config/skills/_shared/resolve-bootstrap.py +776 -84
- package/config/skills/_shared/resolve-stop-hook.py +33 -1
- package/config/skills/_shared/role-config.py +552 -0
- package/config/skills/_shared/run-bounded.py +1 -1
- package/config/skills/_shared/spec-verify-check.py +1037 -192
- package/config/skills/_shared/state-phase-write.py +2551 -83
- package/config/skills/_shared/terminal-claim-check.py +15 -0
- package/config/skills/_shared/verify-merge-findings.py +1127 -50
- package/config/skills/devlyn:engines/SKILL.md +8 -16
- package/config/skills/devlyn:ideate/SKILL.md +1 -1
- package/config/skills/devlyn:ideate/references/spec-template.md +1 -0
- package/config/skills/devlyn:queue/SKILL.md +7 -4
- package/config/skills/devlyn:resolve/SKILL.md +94 -35
- package/config/skills/devlyn:resolve/references/outer-loop.md +15 -0
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +33 -4
- package/config/skills/devlyn:resolve/references/phases/implement.md +4 -3
- package/config/skills/devlyn:resolve/references/phases/plan.md +1 -5
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +4 -2
- package/config/skills/devlyn:resolve/references/phases/verify.md +120 -80
- package/config/skills/devlyn:resolve/references/state-schema.md +23 -9
- package/package.json +5 -1
- package/scripts/lint-fixtures.sh +4 -1
- package/scripts/lint-skills.sh +447 -290
|
@@ -15,15 +15,24 @@ import pathlib
|
|
|
15
15
|
import re
|
|
16
16
|
import sys
|
|
17
17
|
import tempfile
|
|
18
|
+
import runpy
|
|
18
19
|
from typing import Any
|
|
19
20
|
|
|
20
21
|
|
|
22
|
+
JUDGE_OUTPUT_PARSER = runpy.run_path(pathlib.Path(__file__).with_name("judge-output-parser.py"))
|
|
23
|
+
PROCESS_EVIDENCE = runpy.run_path(pathlib.Path(__file__).with_name("process-evidence.py"))
|
|
24
|
+
|
|
25
|
+
|
|
21
26
|
SOURCE_FILES = (
|
|
22
27
|
("mechanical", "verify-mechanical.findings.jsonl"),
|
|
23
28
|
("judge", "verify.findings.jsonl"),
|
|
24
29
|
("pair_judge", "verify.pair.findings.jsonl"),
|
|
25
30
|
("pair_judge", "verify.pair-judge.findings.jsonl"),
|
|
26
31
|
)
|
|
32
|
+
REQUIRED_SOURCE_FILES = {
|
|
33
|
+
"mechanical": "verify-mechanical.findings.jsonl",
|
|
34
|
+
"judge": "verify.findings.jsonl",
|
|
35
|
+
}
|
|
27
36
|
|
|
28
37
|
VERDICT_RANK = {
|
|
29
38
|
"PASS": 0,
|
|
@@ -86,8 +95,21 @@ def reject_json_constant(token: str) -> None:
|
|
|
86
95
|
raise ValueError(f"invalid JSON numeric constant: {token}")
|
|
87
96
|
|
|
88
97
|
|
|
98
|
+
def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict:
|
|
99
|
+
result = {}
|
|
100
|
+
for key, value in pairs:
|
|
101
|
+
if key in result:
|
|
102
|
+
raise ValueError(f"duplicate JSON key: {key}")
|
|
103
|
+
result[key] = value
|
|
104
|
+
return result
|
|
105
|
+
|
|
106
|
+
|
|
89
107
|
def loads_strict_json(text: str) -> Any:
|
|
90
|
-
return json.loads(
|
|
108
|
+
return json.loads(
|
|
109
|
+
text,
|
|
110
|
+
parse_constant=reject_json_constant,
|
|
111
|
+
object_pairs_hook=reject_duplicate_keys,
|
|
112
|
+
)
|
|
91
113
|
|
|
92
114
|
|
|
93
115
|
def rank(verdict: str | None) -> int:
|
|
@@ -125,9 +147,187 @@ def finding_rank(finding: dict[str, Any]) -> int:
|
|
|
125
147
|
return 0
|
|
126
148
|
|
|
127
149
|
|
|
150
|
+
def mechanical_evidence_required(devlyn: pathlib.Path, state: dict[str, Any]) -> bool:
|
|
151
|
+
return PROCESS_EVIDENCE["mechanical_evidence_required"](devlyn.parent.resolve(), state)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def mechanical_evidence_carrier(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
155
|
+
state_path = devlyn / "pipeline.state.json"
|
|
156
|
+
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
157
|
+
if not isinstance(state, dict):
|
|
158
|
+
raise ValueError("pipeline.state.json must contain a JSON object")
|
|
159
|
+
results_path = devlyn / "spec-verify.results.json"
|
|
160
|
+
if not results_path.is_file():
|
|
161
|
+
if state.get("version") == "3.0" and mechanical_evidence_required(devlyn, state):
|
|
162
|
+
raise ValueError("spec-verify.results.json is required for VERIFY MECHANICAL evidence")
|
|
163
|
+
return None
|
|
164
|
+
results = loads_strict_json(results_path.read_text(encoding="utf-8"))
|
|
165
|
+
if not isinstance(results, dict):
|
|
166
|
+
raise ValueError("spec-verify.results.json must contain a JSON object")
|
|
167
|
+
commands = results.get("commands")
|
|
168
|
+
if not isinstance(commands, list):
|
|
169
|
+
raise ValueError("spec-verify.results.json commands must be an array")
|
|
170
|
+
carrier = results.get("process_evidence")
|
|
171
|
+
if carrier is None:
|
|
172
|
+
if commands:
|
|
173
|
+
raise ValueError("MECHANICAL commands exist without a process-evidence carrier")
|
|
174
|
+
if mechanical_evidence_required(devlyn, state):
|
|
175
|
+
raise ValueError("required VERIFY MECHANICAL evidence has no process-evidence carrier")
|
|
176
|
+
run_id = state.get("run_id")
|
|
177
|
+
phases = state.get("phases")
|
|
178
|
+
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
179
|
+
round_ = verify_phase.get("round") if isinstance(verify_phase, dict) else None
|
|
180
|
+
if isinstance(run_id, str) and isinstance(round_, int) and not isinstance(round_, bool):
|
|
181
|
+
manifest_path = (
|
|
182
|
+
devlyn / "process-evidence" / run_id / "verify"
|
|
183
|
+
/ f"round-{round_}" / "manifest.json"
|
|
184
|
+
)
|
|
185
|
+
if manifest_path.exists():
|
|
186
|
+
raise ValueError("VERIFY manifest exists without a process-evidence carrier")
|
|
187
|
+
return None
|
|
188
|
+
run_id = state.get("run_id")
|
|
189
|
+
phases = state.get("phases")
|
|
190
|
+
verify_phase = phases.get("verify") if isinstance(phases, dict) else None
|
|
191
|
+
round_ = verify_phase.get("round") if isinstance(verify_phase, dict) else None
|
|
192
|
+
if not isinstance(run_id, str) or not run_id:
|
|
193
|
+
raise ValueError("state.run_id is required for sealed MECHANICAL evidence")
|
|
194
|
+
if isinstance(round_, bool) or not isinstance(round_, int) or round_ < 0:
|
|
195
|
+
raise ValueError("phases.verify.round is required for sealed MECHANICAL evidence")
|
|
196
|
+
if not isinstance(carrier, dict) or carrier.get("phase") != "verify" or carrier.get("round") != round_:
|
|
197
|
+
raise ValueError("MECHANICAL process-evidence carrier does not match the VERIFY round")
|
|
198
|
+
PROCESS_EVIDENCE["validate_summary_commands"](
|
|
199
|
+
devlyn.parent.resolve(), commands, carrier,
|
|
200
|
+
)
|
|
201
|
+
expected_prefix = f".devlyn/process-evidence/{run_id}/verify/round-{round_}/"
|
|
202
|
+
manifest = carrier.get("manifest")
|
|
203
|
+
if not isinstance(manifest, dict) or not str(manifest.get("path", "")).startswith(expected_prefix):
|
|
204
|
+
raise ValueError("MECHANICAL process-evidence carrier does not match state.run_id")
|
|
205
|
+
|
|
206
|
+
bound = state.get("process_evidence")
|
|
207
|
+
if bound is not None:
|
|
208
|
+
if not isinstance(bound, list):
|
|
209
|
+
raise ValueError("state.process_evidence must be null or an array")
|
|
210
|
+
same_round = [
|
|
211
|
+
item for item in bound
|
|
212
|
+
if isinstance(item, dict)
|
|
213
|
+
and item.get("phase") == "verify"
|
|
214
|
+
and item.get("round") == round_
|
|
215
|
+
]
|
|
216
|
+
if same_round and same_round != [carrier]:
|
|
217
|
+
raise ValueError("state-bound VERIFY evidence disagrees with MECHANICAL results")
|
|
218
|
+
return carrier
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def mechanical_evidence_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
222
|
+
try:
|
|
223
|
+
mechanical_evidence_carrier(devlyn)
|
|
224
|
+
except (PROCESS_EVIDENCE["EvidenceError"], OSError, UnicodeError, ValueError) as exc:
|
|
225
|
+
return {
|
|
226
|
+
"id": "verify-mechanical-evidence-invalid",
|
|
227
|
+
"rule_id": "invariant.process-evidence-invalid",
|
|
228
|
+
"severity": "CRITICAL",
|
|
229
|
+
"confidence": "high",
|
|
230
|
+
"file": "spec-verify.results.json",
|
|
231
|
+
"line": 1,
|
|
232
|
+
"message": f"Sealed VERIFY MECHANICAL evidence failed rehash before merge: {exc}",
|
|
233
|
+
"criterion_ref": "process-evidence://mechanical",
|
|
234
|
+
"source": "mechanical",
|
|
235
|
+
}
|
|
236
|
+
return None
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def mechanical_evidence_outcome(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
240
|
+
carrier = mechanical_evidence_carrier(devlyn)
|
|
241
|
+
if carrier is None:
|
|
242
|
+
return None
|
|
243
|
+
return PROCESS_EVIDENCE["bound_carrier_outcome"](devlyn.parent.resolve(), carrier)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
ROLE_CONFIG = runpy.run_path(pathlib.Path(__file__).with_name("role-config.py"))
|
|
247
|
+
JUDGE_ROLE_EVIDENCE = runpy.run_path(pathlib.Path(__file__).with_name("judge-role-evidence.py"))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def resolved_primary_engine(state):
|
|
251
|
+
return ROLE_CONFIG["primary_engine"](state)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def primary_timeout_blocker(id_: str, message: str) -> dict[str, Any]:
|
|
255
|
+
return {
|
|
256
|
+
"id": id_,
|
|
257
|
+
"rule_id": "verify.primary.timeout-contract",
|
|
258
|
+
"severity": "CRITICAL",
|
|
259
|
+
"confidence": "high",
|
|
260
|
+
"file": "verify.primary.timeout.json",
|
|
261
|
+
"line": 1,
|
|
262
|
+
"message": message,
|
|
263
|
+
"criterion_ref": "verify.primary.timeout",
|
|
264
|
+
"source": "judge",
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def read_primary_timeout_marker(
|
|
269
|
+
devlyn: pathlib.Path,
|
|
270
|
+
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
271
|
+
path = devlyn / "verify.primary.timeout.json"
|
|
272
|
+
if not path.is_file():
|
|
273
|
+
return None, None
|
|
274
|
+
try:
|
|
275
|
+
marker = loads_strict_json(path.read_text(encoding="utf-8"))
|
|
276
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
277
|
+
return None, primary_timeout_blocker(
|
|
278
|
+
"verify-primary-timeout-marker-malformed",
|
|
279
|
+
f"verify.primary.timeout.json is malformed: {exc}",
|
|
280
|
+
)
|
|
281
|
+
if not isinstance(marker, dict) or set(marker) != {"engine", "budget_seconds"}:
|
|
282
|
+
return None, primary_timeout_blocker(
|
|
283
|
+
"verify-primary-timeout-marker-malformed",
|
|
284
|
+
"verify.primary.timeout.json must contain exactly engine and budget_seconds.",
|
|
285
|
+
)
|
|
286
|
+
engine = marker["engine"]
|
|
287
|
+
budget_seconds = marker["budget_seconds"]
|
|
288
|
+
if not isinstance(engine, str) or not engine:
|
|
289
|
+
return None, primary_timeout_blocker(
|
|
290
|
+
"verify-primary-timeout-marker-malformed",
|
|
291
|
+
"verify.primary.timeout.json engine must be a non-empty string.",
|
|
292
|
+
)
|
|
293
|
+
if (
|
|
294
|
+
not isinstance(budget_seconds, int)
|
|
295
|
+
or isinstance(budget_seconds, bool)
|
|
296
|
+
or budget_seconds != 600
|
|
297
|
+
):
|
|
298
|
+
return None, primary_timeout_blocker(
|
|
299
|
+
"verify-primary-timeout-budget-mismatch",
|
|
300
|
+
"verify.primary.timeout.json budget_seconds must equal 600.",
|
|
301
|
+
)
|
|
302
|
+
try:
|
|
303
|
+
state = loads_strict_json(
|
|
304
|
+
(devlyn / "pipeline.state.json").read_text(encoding="utf-8")
|
|
305
|
+
)
|
|
306
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
307
|
+
return None, primary_timeout_blocker(
|
|
308
|
+
"verify-primary-timeout-engine-mismatch",
|
|
309
|
+
f"Cannot authenticate the primary timeout engine from pipeline.state.json: {exc}",
|
|
310
|
+
)
|
|
311
|
+
try:
|
|
312
|
+
expected_engine = resolved_primary_engine(state) if isinstance(state, dict) else None
|
|
313
|
+
except ValueError:
|
|
314
|
+
expected_engine = None
|
|
315
|
+
if (
|
|
316
|
+
not isinstance(expected_engine, str)
|
|
317
|
+
or not expected_engine
|
|
318
|
+
or engine != expected_engine
|
|
319
|
+
):
|
|
320
|
+
return None, primary_timeout_blocker(
|
|
321
|
+
"verify-primary-timeout-engine-mismatch",
|
|
322
|
+
"Primary timeout engine does not match the resolved primary judge.",
|
|
323
|
+
)
|
|
324
|
+
return {"engine": engine, "budget_seconds": budget_seconds}, None
|
|
325
|
+
|
|
326
|
+
|
|
128
327
|
def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str, str | None]]:
|
|
129
328
|
findings: list[dict[str, Any]] = []
|
|
130
329
|
source_verdicts: dict[str, str | None] = {source: "PASS" for source, _ in SOURCE_FILES}
|
|
330
|
+
primary_timeout_marker, primary_timeout_violation = read_primary_timeout_marker(devlyn)
|
|
131
331
|
# A verdict must come from a judge that ran: pair_judge stays null until
|
|
132
332
|
# spawn evidence exists (a pair findings file or pair stdout). verify.md's
|
|
133
333
|
# pair contract records null when no second agent is spawned.
|
|
@@ -135,6 +335,27 @@ def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str,
|
|
|
135
335
|
for source, name in SOURCE_FILES:
|
|
136
336
|
path = devlyn / name
|
|
137
337
|
if not path.is_file():
|
|
338
|
+
# A verdict-binding mechanical result contractually skips both
|
|
339
|
+
# judges. Its absent primary carrier is therefore not evidence a
|
|
340
|
+
# dispatched primary judge failed to produce.
|
|
341
|
+
if source == "judge" and rank(source_verdicts.get("mechanical")) >= 2:
|
|
342
|
+
source_verdicts[source] = None
|
|
343
|
+
continue
|
|
344
|
+
if source == "judge" and primary_timeout_marker is not None:
|
|
345
|
+
continue
|
|
346
|
+
if REQUIRED_SOURCE_FILES.get(source) == name:
|
|
347
|
+
findings.append({
|
|
348
|
+
"id": f"verify-merge-required-source-missing-{source}",
|
|
349
|
+
"rule_id": "verify.findings.required-source-missing",
|
|
350
|
+
"severity": "CRITICAL",
|
|
351
|
+
"confidence": "high",
|
|
352
|
+
"file": name,
|
|
353
|
+
"line": 1,
|
|
354
|
+
"message": f"Required VERIFY {source} findings file is missing: {name}",
|
|
355
|
+
"criterion_ref": "verify-merge",
|
|
356
|
+
"source": source,
|
|
357
|
+
})
|
|
358
|
+
source_verdicts[source] = "BLOCKED"
|
|
138
359
|
continue
|
|
139
360
|
if source_verdicts[source] is None:
|
|
140
361
|
source_verdicts[source] = "PASS"
|
|
@@ -168,7 +389,109 @@ def read_findings(devlyn: pathlib.Path) -> tuple[list[dict[str, Any]], dict[str,
|
|
|
168
389
|
source_verdicts[source] = worse(
|
|
169
390
|
source_verdicts[source], RANK_VERDICT[finding_rank(item)]
|
|
170
391
|
)
|
|
392
|
+
evidence_violation = mechanical_evidence_violation(devlyn)
|
|
393
|
+
if evidence_violation is not None:
|
|
394
|
+
findings.append(evidence_violation)
|
|
395
|
+
source_verdicts["mechanical"] = "BLOCKED"
|
|
396
|
+
else:
|
|
397
|
+
outcome = mechanical_evidence_outcome(devlyn)
|
|
398
|
+
if outcome is not None and outcome["verdict"] != "PASS":
|
|
399
|
+
blocked = outcome["verdict"] == "BLOCKED"
|
|
400
|
+
ids = (
|
|
401
|
+
[item["id"] for item in outcome["capability_denials"]]
|
|
402
|
+
if blocked else outcome["failed_ids"]
|
|
403
|
+
)
|
|
404
|
+
findings.append({
|
|
405
|
+
"id": (
|
|
406
|
+
"verify-mechanical-capability-denied"
|
|
407
|
+
if blocked else "verify-mechanical-expectation-mismatch"
|
|
408
|
+
),
|
|
409
|
+
"rule_id": (
|
|
410
|
+
"invariant.build-env-underprovisioned"
|
|
411
|
+
if blocked else "invariant.mechanical-expectation-mismatch"
|
|
412
|
+
),
|
|
413
|
+
"severity": "CRITICAL" if blocked else "HIGH",
|
|
414
|
+
"confidence": "high",
|
|
415
|
+
"file": "spec-verify.results.json",
|
|
416
|
+
"line": 1,
|
|
417
|
+
"message": (
|
|
418
|
+
"Sealed VERIFY MECHANICAL evidence records a capability denial: "
|
|
419
|
+
if blocked else
|
|
420
|
+
"Sealed VERIFY MECHANICAL evidence records failed expectations: "
|
|
421
|
+
) + ",".join(ids),
|
|
422
|
+
"criterion_ref": "process-evidence://mechanical",
|
|
423
|
+
"source": "mechanical",
|
|
424
|
+
"verdict_binding": True,
|
|
425
|
+
})
|
|
426
|
+
source_verdicts["mechanical"] = outcome["verdict"]
|
|
427
|
+
if primary_timeout_violation is not None:
|
|
428
|
+
findings.append(primary_timeout_violation)
|
|
429
|
+
source_verdicts["judge"] = "BLOCKED"
|
|
430
|
+
elif primary_timeout_marker is not None:
|
|
431
|
+
findings.append(primary_timeout_blocker(
|
|
432
|
+
"verify-primary-timeout",
|
|
433
|
+
"Primary JUDGE exceeded its authenticated 600-second wall budget.",
|
|
434
|
+
))
|
|
435
|
+
source_verdicts["judge"] = "BLOCKED"
|
|
171
436
|
findings.extend(detect_pair_stdout_contract_violations(devlyn, source_verdicts))
|
|
437
|
+
pair_summary_path = devlyn / "pair-judge.summary.json"
|
|
438
|
+
pair_carrier_exists = any(
|
|
439
|
+
(devlyn / name).is_file()
|
|
440
|
+
for name in ("verify.pair.findings.jsonl", "verify.pair-judge.findings.jsonl")
|
|
441
|
+
)
|
|
442
|
+
if pair_carrier_exists and pair_summary_path.is_file():
|
|
443
|
+
try:
|
|
444
|
+
pair_summary = loads_strict_json(pair_summary_path.read_text(encoding="utf-8"))
|
|
445
|
+
except ValueError as exc:
|
|
446
|
+
summary_error = f"pair-judge.summary.json is malformed JSON: {exc}"
|
|
447
|
+
else:
|
|
448
|
+
pair_verdict = pair_summary.get("verdict") if isinstance(pair_summary, dict) else None
|
|
449
|
+
summary_error = None
|
|
450
|
+
if not isinstance(pair_summary, dict):
|
|
451
|
+
summary_error = "pair-judge.summary.json must be a JSON object."
|
|
452
|
+
elif not isinstance(pair_verdict, str) or pair_verdict not in {
|
|
453
|
+
"PASS", "PASS_WITH_ISSUES", "FAIL", "NEEDS_WORK", "BLOCKED"
|
|
454
|
+
}:
|
|
455
|
+
summary_error = (
|
|
456
|
+
"pair-judge.summary.json verdict must be PASS, PASS_WITH_ISSUES, "
|
|
457
|
+
"FAIL, NEEDS_WORK, or BLOCKED."
|
|
458
|
+
)
|
|
459
|
+
if summary_error is not None:
|
|
460
|
+
findings.append(
|
|
461
|
+
pair_blocker(
|
|
462
|
+
"verify-pair-summary-invalid",
|
|
463
|
+
summary_error,
|
|
464
|
+
pair_summary_path.name,
|
|
465
|
+
)
|
|
466
|
+
)
|
|
467
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
468
|
+
elif (
|
|
469
|
+
source_verdicts["pair_judge"] != "TIMEOUT"
|
|
470
|
+
or rank(pair_verdict) > rank("TIMEOUT")
|
|
471
|
+
):
|
|
472
|
+
source_verdicts["pair_judge"] = worse(
|
|
473
|
+
source_verdicts["pair_judge"], pair_verdict
|
|
474
|
+
)
|
|
475
|
+
if (devlyn / "pipeline.state.json").is_file():
|
|
476
|
+
try:
|
|
477
|
+
state = loads_strict_json((devlyn / "pipeline.state.json").read_text())
|
|
478
|
+
required = JUDGE_ROLE_EVIDENCE["required_roles"](state)
|
|
479
|
+
if rank(source_verdicts.get("mechanical")) >= 2:
|
|
480
|
+
required = []
|
|
481
|
+
for role in required:
|
|
482
|
+
source = "judge" if role == "primary_judge" else "pair_judge"
|
|
483
|
+
if source_verdicts.get(source) in {"BLOCKED", "TIMEOUT"}:
|
|
484
|
+
continue
|
|
485
|
+
try:
|
|
486
|
+
JUDGE_ROLE_EVIDENCE["authenticate"](devlyn, state, role)
|
|
487
|
+
except (ValueError, OSError, TypeError, KeyError) as exc:
|
|
488
|
+
source_verdicts[source] = "BLOCKED"
|
|
489
|
+
findings.append({**primary_timeout_blocker("verify-role-evidence-invalid", str(exc)),
|
|
490
|
+
"source": source, "rule_id": "verify.role-evidence"})
|
|
491
|
+
except (ValueError, OSError, TypeError, KeyError) as exc:
|
|
492
|
+
source_verdicts["judge"] = "BLOCKED"
|
|
493
|
+
findings.append({**primary_timeout_blocker("verify-role-resolution-invalid", str(exc)),
|
|
494
|
+
"rule_id": "verify.role-resolution"})
|
|
172
495
|
return findings, source_verdicts
|
|
173
496
|
|
|
174
497
|
|
|
@@ -180,6 +503,29 @@ def has_pair_findings(devlyn: pathlib.Path) -> bool:
|
|
|
180
503
|
return False
|
|
181
504
|
|
|
182
505
|
|
|
506
|
+
def canonical_other_judge_stdout(devlyn: pathlib.Path, primary_engine: str) -> list[pathlib.Path]:
|
|
507
|
+
adapters = pathlib.Path(__file__).with_name("adapters")
|
|
508
|
+
engine_names = {
|
|
509
|
+
path.stem for path in adapters.glob("*.md")
|
|
510
|
+
if path.stem != "README"
|
|
511
|
+
}
|
|
512
|
+
return sorted(
|
|
513
|
+
devlyn / f"{engine}-judge.stdout"
|
|
514
|
+
for engine in engine_names
|
|
515
|
+
if engine != primary_engine and (devlyn / f"{engine}-judge.stdout").is_file()
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def pair_capture_exists(devlyn: pathlib.Path, stdout_paths: list[pathlib.Path]) -> bool:
|
|
520
|
+
return bool(stdout_paths) or has_pair_findings(devlyn) or any(
|
|
521
|
+
(devlyn / name).is_file()
|
|
522
|
+
for name in (
|
|
523
|
+
"verify.pair.timeout.json",
|
|
524
|
+
"pair-judge.summary.json",
|
|
525
|
+
)
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
|
|
183
529
|
def pair_trigger_status(devlyn: pathlib.Path) -> tuple[bool, dict[str, Any] | None]:
|
|
184
530
|
state_path = devlyn / "pipeline.state.json"
|
|
185
531
|
if not state_path.is_file():
|
|
@@ -346,12 +692,41 @@ def risk_profile_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | No
|
|
|
346
692
|
def verify_state_contract_violation(devlyn: pathlib.Path) -> dict[str, Any] | None:
|
|
347
693
|
state_path = devlyn / "pipeline.state.json"
|
|
348
694
|
if not state_path.is_file():
|
|
349
|
-
return
|
|
695
|
+
return {
|
|
696
|
+
"id": "verify-state-missing",
|
|
697
|
+
"rule_id": "verify.state.missing",
|
|
698
|
+
"message": "pipeline.state.json is required before VERIFY merge.",
|
|
699
|
+
"file": "pipeline.state.json",
|
|
700
|
+
}
|
|
350
701
|
try:
|
|
351
702
|
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
352
703
|
except ValueError:
|
|
353
|
-
return
|
|
354
|
-
|
|
704
|
+
return {
|
|
705
|
+
"id": "verify-pair-trigger-state-malformed",
|
|
706
|
+
"rule_id": "verify.pair.emission-contract",
|
|
707
|
+
"message": "pipeline.state.json is malformed; cannot verify pair_trigger contract.",
|
|
708
|
+
"file": "pipeline.state.json",
|
|
709
|
+
}
|
|
710
|
+
if not isinstance(state, dict):
|
|
711
|
+
return {
|
|
712
|
+
"id": "verify-state-malformed",
|
|
713
|
+
"rule_id": "verify.state.malformed",
|
|
714
|
+
"message": "pipeline.state.json must be a JSON object before VERIFY merge.",
|
|
715
|
+
"file": "pipeline.state.json",
|
|
716
|
+
}
|
|
717
|
+
try:
|
|
718
|
+
engine = resolved_primary_engine(state)
|
|
719
|
+
except ValueError:
|
|
720
|
+
engine = None
|
|
721
|
+
if not isinstance(engine, str) or not engine.strip():
|
|
722
|
+
rule = "verify.state.engine-malformed"
|
|
723
|
+
return {
|
|
724
|
+
"id": rule,
|
|
725
|
+
"rule_id": rule,
|
|
726
|
+
"message": "pipeline.state.json requires a valid primary engine before VERIFY merge.",
|
|
727
|
+
"file": "pipeline.state.json",
|
|
728
|
+
}
|
|
729
|
+
if not state_uses_default_pair_contract(state):
|
|
355
730
|
return None
|
|
356
731
|
source = state.get("source")
|
|
357
732
|
if not isinstance(source, dict) or source.get("type") != "generated":
|
|
@@ -768,14 +1143,47 @@ def read_pair_timeout_marker(devlyn: pathlib.Path) -> tuple[dict[str, Any] | Non
|
|
|
768
1143
|
return {"engine": engine, "budget_seconds": budget_seconds}, None
|
|
769
1144
|
|
|
770
1145
|
|
|
1146
|
+
def is_result_less_message_stream(stdout_text: str) -> bool:
|
|
1147
|
+
"""True when a declared whole-message stream stops before its terminal result.
|
|
1148
|
+
|
|
1149
|
+
The timeout marker attests the killed process. This classifier admits only a
|
|
1150
|
+
valid stream prefix, optionally ending in one truncated JSON record.
|
|
1151
|
+
"""
|
|
1152
|
+
lines = [raw for line in stdout_text.splitlines() if (raw := line.strip())]
|
|
1153
|
+
if not lines:
|
|
1154
|
+
return False
|
|
1155
|
+
try:
|
|
1156
|
+
first = JUDGE_OUTPUT_PARSER["loads_strict_json"](lines[0])
|
|
1157
|
+
except ValueError:
|
|
1158
|
+
return False
|
|
1159
|
+
if (
|
|
1160
|
+
not isinstance(first, dict)
|
|
1161
|
+
or first.get("type") != "system"
|
|
1162
|
+
or first.get("subtype") != "init"
|
|
1163
|
+
or not isinstance(first.get("session_id"), str)
|
|
1164
|
+
):
|
|
1165
|
+
return False
|
|
1166
|
+
session_id = first["session_id"]
|
|
1167
|
+
for index, raw in enumerate(lines[1:], 1):
|
|
1168
|
+
try:
|
|
1169
|
+
item = JUDGE_OUTPUT_PARSER["loads_strict_json"](raw)
|
|
1170
|
+
except ValueError:
|
|
1171
|
+
return index == len(lines) - 1 and raw.startswith("{") and not raw.endswith("}")
|
|
1172
|
+
if (
|
|
1173
|
+
not isinstance(item, dict)
|
|
1174
|
+
or item.get("type") not in {"assistant", "user"}
|
|
1175
|
+
or item.get("session_id") != session_id
|
|
1176
|
+
):
|
|
1177
|
+
return False
|
|
1178
|
+
return True
|
|
1179
|
+
|
|
1180
|
+
|
|
771
1181
|
def detect_pair_stdout_contract_violations(
|
|
772
1182
|
devlyn: pathlib.Path,
|
|
773
1183
|
source_verdicts: dict[str, str | None],
|
|
774
1184
|
) -> list[dict[str, Any]]:
|
|
775
|
-
#
|
|
776
|
-
#
|
|
777
|
-
# contract is engine-neutral.
|
|
778
|
-
stdout_paths = sorted(devlyn.glob("*-judge.stdout"))
|
|
1185
|
+
# The primary uses its explicit phase engine, or the legacy executor; only the OTHER engine's
|
|
1186
|
+
# capture is pair evidence.
|
|
779
1187
|
timeout_marker, timeout_violation = read_pair_timeout_marker(devlyn)
|
|
780
1188
|
if timeout_violation is not None:
|
|
781
1189
|
source_verdicts["pair_judge"] = "BLOCKED"
|
|
@@ -827,6 +1235,11 @@ def detect_pair_stdout_contract_violations(
|
|
|
827
1235
|
state_violation["rule_id"],
|
|
828
1236
|
)
|
|
829
1237
|
]
|
|
1238
|
+
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
1239
|
+
assert isinstance(state, dict)
|
|
1240
|
+
engine = resolved_primary_engine(state)
|
|
1241
|
+
assert isinstance(engine, str) and engine.strip()
|
|
1242
|
+
stdout_paths = canonical_other_judge_stdout(devlyn, engine)
|
|
830
1243
|
if not required and not pair_trigger_present(devlyn):
|
|
831
1244
|
missing_violation = pair_trigger_missing_contract_violation(devlyn, source_verdicts)
|
|
832
1245
|
if missing_violation is not None:
|
|
@@ -858,27 +1271,65 @@ def detect_pair_stdout_contract_violations(
|
|
|
858
1271
|
reason_violation["file"],
|
|
859
1272
|
)
|
|
860
1273
|
]
|
|
861
|
-
|
|
1274
|
+
trigger_present = pair_trigger_present(devlyn)
|
|
1275
|
+
if trigger_present and not required:
|
|
1276
|
+
if pair_capture_exists(devlyn, stdout_paths):
|
|
1277
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1278
|
+
return [
|
|
1279
|
+
pair_blocker(
|
|
1280
|
+
"verify-pair-skipped-output-present",
|
|
1281
|
+
"Pair state is skipped or ineligible, but a canonical OTHER-judge carrier exists.",
|
|
1282
|
+
stdout_paths[0].name if stdout_paths else "verify.pair.findings.jsonl",
|
|
1283
|
+
"verify.pair.state-capture-contradiction",
|
|
1284
|
+
)
|
|
1285
|
+
]
|
|
1286
|
+
source_verdicts["pair_judge"] = None
|
|
862
1287
|
return []
|
|
1288
|
+
if len(stdout_paths) > 1:
|
|
1289
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1290
|
+
return [
|
|
1291
|
+
pair_blocker(
|
|
1292
|
+
"verify-pair-output-ambiguous",
|
|
1293
|
+
"Multiple canonical OTHER-engine stdout carriers exist: "
|
|
1294
|
+
+ ", ".join(path.name for path in stdout_paths),
|
|
1295
|
+
stdout_paths[0].name,
|
|
1296
|
+
)
|
|
1297
|
+
]
|
|
1298
|
+
if timeout_marker is not None:
|
|
1299
|
+
expected_timeout_stdout = f"{timeout_marker['engine']}-judge.stdout"
|
|
1300
|
+
if timeout_marker["engine"] == engine or (
|
|
1301
|
+
stdout_paths and stdout_paths[0].name != expected_timeout_stdout
|
|
1302
|
+
):
|
|
1303
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1304
|
+
return [
|
|
1305
|
+
pair_blocker(
|
|
1306
|
+
"verify-pair-timeout-engine-mismatch",
|
|
1307
|
+
"Pair timeout engine does not match the canonical OTHER-engine carrier.",
|
|
1308
|
+
"verify.pair.timeout.json",
|
|
1309
|
+
)
|
|
1310
|
+
]
|
|
863
1311
|
if not stdout_paths:
|
|
864
1312
|
if timeout_marker is not None:
|
|
865
|
-
source_verdicts["pair_judge"]
|
|
1313
|
+
if rank(source_verdicts["pair_judge"]) <= rank("TIMEOUT"):
|
|
1314
|
+
source_verdicts["pair_judge"] = "TIMEOUT"
|
|
866
1315
|
return []
|
|
867
1316
|
if required:
|
|
868
1317
|
source_verdicts["pair_judge"] = "BLOCKED"
|
|
869
1318
|
return [
|
|
870
1319
|
pair_blocker(
|
|
871
1320
|
"verify-pair-required-output-missing",
|
|
872
|
-
"Pair-mode was required, but the pair-JUDGE produced no
|
|
1321
|
+
"Pair-mode was required, but the pair-JUDGE produced no canonical OTHER-engine stdout.",
|
|
873
1322
|
"verify.pair.findings.jsonl",
|
|
874
1323
|
)
|
|
875
1324
|
]
|
|
876
1325
|
return []
|
|
1326
|
+
if has_pair_findings(devlyn):
|
|
1327
|
+
return []
|
|
877
1328
|
if source_verdicts["pair_judge"] is None and timeout_marker is None:
|
|
878
1329
|
source_verdicts["pair_judge"] = "PASS"
|
|
879
1330
|
for stdout_path in stdout_paths:
|
|
880
|
-
|
|
881
|
-
if not
|
|
1331
|
+
stdout_text = stdout_path.read_text(encoding="utf-8")
|
|
1332
|
+
if not stdout_text.strip():
|
|
882
1333
|
if timeout_marker is not None:
|
|
883
1334
|
continue
|
|
884
1335
|
source_verdicts["pair_judge"] = "BLOCKED"
|
|
@@ -889,41 +1340,27 @@ def detect_pair_stdout_contract_violations(
|
|
|
889
1340
|
stdout_path.name,
|
|
890
1341
|
)
|
|
891
1342
|
]
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
if not raw:
|
|
897
|
-
continue
|
|
898
|
-
if raw.startswith("# SUMMARY "):
|
|
899
|
-
try:
|
|
900
|
-
summary = loads_strict_json(raw.removeprefix("# SUMMARY ").strip())
|
|
901
|
-
except ValueError:
|
|
902
|
-
continue
|
|
903
|
-
if summary.get("verdict") in {"NEEDS_WORK", "FAIL", "BLOCKED"}:
|
|
904
|
-
has_nonpass_summary = True
|
|
905
|
-
continue
|
|
906
|
-
if raw.startswith("#"):
|
|
907
|
-
continue
|
|
908
|
-
try:
|
|
909
|
-
item = loads_strict_json(raw)
|
|
910
|
-
except ValueError:
|
|
1343
|
+
try:
|
|
1344
|
+
stdout_findings, stdout_summary = JUDGE_OUTPUT_PARSER["collect_stdout"](stdout_path)
|
|
1345
|
+
except SystemExit as exc:
|
|
1346
|
+
if timeout_marker is not None and is_result_less_message_stream(stdout_text):
|
|
911
1347
|
continue
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
1348
|
+
source_verdicts["pair_judge"] = "BLOCKED"
|
|
1349
|
+
return [
|
|
1350
|
+
pair_blocker(
|
|
1351
|
+
"verify-pair-emission-contract-violated",
|
|
1352
|
+
f"pair-JUDGE stdout {stdout_path.name} violates the shared emission parser: {exc}",
|
|
1353
|
+
stdout_path.name,
|
|
1354
|
+
)
|
|
1355
|
+
]
|
|
1356
|
+
if stdout_findings or stdout_summary is None or stdout_summary["verdict"] != "PASS":
|
|
920
1357
|
source_verdicts["pair_judge"] = "BLOCKED"
|
|
921
1358
|
return [
|
|
922
1359
|
pair_blocker(
|
|
923
1360
|
"verify-pair-emission-contract-violated",
|
|
924
1361
|
(
|
|
925
1362
|
f"pair-JUDGE stdout {stdout_path.name} contained findings or a non-PASS "
|
|
926
|
-
"
|
|
1363
|
+
"verdict, but the canonical pair findings JSONL file was empty."
|
|
927
1364
|
),
|
|
928
1365
|
stdout_path.name,
|
|
929
1366
|
)
|
|
@@ -966,11 +1403,47 @@ def write_state(devlyn: pathlib.Path, summary: dict[str, Any]) -> None:
|
|
|
966
1403
|
if not state_path.is_file():
|
|
967
1404
|
raise SystemExit(f"error: {state_path} not found")
|
|
968
1405
|
state = loads_strict_json(state_path.read_text(encoding="utf-8"))
|
|
1406
|
+
try:
|
|
1407
|
+
carrier = mechanical_evidence_carrier(devlyn)
|
|
1408
|
+
except (PROCESS_EVIDENCE["EvidenceError"], OSError, UnicodeError, ValueError) as exc:
|
|
1409
|
+
if summary.get("source_verdicts", {}).get("mechanical") != "BLOCKED":
|
|
1410
|
+
raise SystemExit(f"error: invalid MECHANICAL process evidence: {exc}") from exc
|
|
1411
|
+
carrier = None
|
|
1412
|
+
if carrier is not None:
|
|
1413
|
+
bound = state.get("process_evidence")
|
|
1414
|
+
if bound is None:
|
|
1415
|
+
bound = []
|
|
1416
|
+
elif not isinstance(bound, list):
|
|
1417
|
+
raise SystemExit("error: state.process_evidence must be null or an array")
|
|
1418
|
+
same_round = [
|
|
1419
|
+
item for item in bound
|
|
1420
|
+
if isinstance(item, dict)
|
|
1421
|
+
and item.get("phase") == carrier["phase"]
|
|
1422
|
+
and item.get("round") == carrier["round"]
|
|
1423
|
+
]
|
|
1424
|
+
if same_round and same_round != [carrier]:
|
|
1425
|
+
raise SystemExit("error: state-bound VERIFY evidence disagrees with MECHANICAL results")
|
|
1426
|
+
if not same_round:
|
|
1427
|
+
bound.append(carrier)
|
|
1428
|
+
state["process_evidence"] = bound
|
|
969
1429
|
phases = state.setdefault("phases", {})
|
|
970
1430
|
verify = phases.get("verify")
|
|
971
1431
|
if not isinstance(verify, dict):
|
|
972
1432
|
verify = {}
|
|
973
1433
|
phases["verify"] = verify
|
|
1434
|
+
role_evidence = {}
|
|
1435
|
+
try:
|
|
1436
|
+
required_roles = JUDGE_ROLE_EVIDENCE["required_roles"](state)
|
|
1437
|
+
except ValueError as exc:
|
|
1438
|
+
if summary.get("source_verdicts", {}).get("judge") != "BLOCKED":
|
|
1439
|
+
raise SystemExit(str(exc)) from exc
|
|
1440
|
+
required_roles = [] # Persist the invalid-resolution failure; no legacy dispatch.
|
|
1441
|
+
for role in required_roles:
|
|
1442
|
+
source = "judge" if role == "primary_judge" else "pair_judge"
|
|
1443
|
+
if summary.get("source_verdicts", {}).get(source) in {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK"}:
|
|
1444
|
+
role_evidence[role] = JUDGE_ROLE_EVIDENCE["authenticate"](devlyn, state, role)
|
|
1445
|
+
if role_evidence:
|
|
1446
|
+
verify["role_evidence"] = role_evidence
|
|
974
1447
|
verify["verdict"] = summary["verdict"]
|
|
975
1448
|
sub = verify.get("sub_verdicts")
|
|
976
1449
|
if sub is None:
|
|
@@ -996,15 +1469,28 @@ def write_state(devlyn: pathlib.Path, summary: dict[str, Any]) -> None:
|
|
|
996
1469
|
|
|
997
1470
|
|
|
998
1471
|
def self_test() -> int:
|
|
1472
|
+
import subprocess
|
|
1473
|
+
|
|
1474
|
+
try:
|
|
1475
|
+
loads_strict_json('{"verdict":"PASS","verdict":"BLOCKED"}')
|
|
1476
|
+
except ValueError as exc:
|
|
1477
|
+
assert "duplicate JSON key" in str(exc)
|
|
1478
|
+
else:
|
|
1479
|
+
raise AssertionError("duplicate VERIFY authority key was accepted")
|
|
999
1480
|
with tempfile.TemporaryDirectory() as tmp:
|
|
1000
1481
|
devlyn = pathlib.Path(tmp)
|
|
1001
1482
|
|
|
1483
|
+
# Every completed VERIFY has deterministic mechanical and primary
|
|
1484
|
+
# judge carriers. Missing either one is an evidence failure, not PASS.
|
|
1485
|
+
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1486
|
+
|
|
1002
1487
|
# state-phase-write.py's spawn always writes sub_verdicts: null (the
|
|
1003
1488
|
# per-round reset contract, state-schema.md#write-protocol) — this is
|
|
1004
1489
|
# the real shape write_state() sees on every VERIFY completion, not
|
|
1005
1490
|
# the pre-populated {} the other scenarios below seed for brevity.
|
|
1006
1491
|
(devlyn / "pipeline.state.json").write_text(
|
|
1007
1492
|
json.dumps({
|
|
1493
|
+
"engine": "claude",
|
|
1008
1494
|
"phases": {
|
|
1009
1495
|
"verify": {
|
|
1010
1496
|
"verdict": None,
|
|
@@ -1016,6 +1502,152 @@ def self_test() -> int:
|
|
|
1016
1502
|
encoding="utf-8",
|
|
1017
1503
|
)
|
|
1018
1504
|
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1505
|
+
|
|
1506
|
+
sealed_work = pathlib.Path(tmp) / "sealed-work"
|
|
1507
|
+
sealed_devlyn = sealed_work / ".devlyn"
|
|
1508
|
+
sealed_devlyn.mkdir(parents=True)
|
|
1509
|
+
sealed_state = {
|
|
1510
|
+
"run_id": "rs-sealed-mechanical",
|
|
1511
|
+
"engine": "claude",
|
|
1512
|
+
"process_evidence": None,
|
|
1513
|
+
"phases": {"verify": {"round": 2, "verdict": None, "sub_verdicts": None}},
|
|
1514
|
+
}
|
|
1515
|
+
obligation = PROCESS_EVIDENCE["normalize_obligation"]({
|
|
1516
|
+
"id": "mechanical-0",
|
|
1517
|
+
"phase": "verify",
|
|
1518
|
+
"argv": [sys.executable, "-c", "print('sealed')"],
|
|
1519
|
+
})
|
|
1520
|
+
manifest_rel = PROCESS_EVIDENCE["manifest_relative_path"](sealed_state, "verify")
|
|
1521
|
+
PROCESS_EVIDENCE["capture_process"](
|
|
1522
|
+
sealed_work, sealed_work / manifest_rel, sealed_state["run_id"],
|
|
1523
|
+
"verify", 2, obligation,
|
|
1524
|
+
)
|
|
1525
|
+
carrier = PROCESS_EVIDENCE["validate_manifest"](
|
|
1526
|
+
sealed_work, manifest_rel, sealed_state["run_id"], "verify", 2,
|
|
1527
|
+
[obligation], require_expectations=False,
|
|
1528
|
+
)
|
|
1529
|
+
(sealed_devlyn / "pipeline.state.json").write_text(
|
|
1530
|
+
json.dumps(sealed_state), encoding="utf-8",
|
|
1531
|
+
)
|
|
1532
|
+
sealed_commands = PROCESS_EVIDENCE["bound_carrier_summary_commands"](
|
|
1533
|
+
sealed_work, carrier,
|
|
1534
|
+
)
|
|
1535
|
+
(sealed_devlyn / "spec-verify.results.json").write_text(
|
|
1536
|
+
json.dumps({"commands": sealed_commands, "process_evidence": carrier}),
|
|
1537
|
+
encoding="utf-8",
|
|
1538
|
+
)
|
|
1539
|
+
(sealed_devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1540
|
+
(sealed_devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1541
|
+
sealed_findings, sealed_verdicts = read_findings(sealed_devlyn)
|
|
1542
|
+
assert sealed_verdicts["mechanical"] == "PASS", sealed_findings
|
|
1543
|
+
sealed_summary = write_outputs(sealed_devlyn, sealed_findings, sealed_verdicts)
|
|
1544
|
+
write_state(sealed_devlyn, sealed_summary)
|
|
1545
|
+
bound_state = loads_strict_json(
|
|
1546
|
+
(sealed_devlyn / "pipeline.state.json").read_text(encoding="utf-8")
|
|
1547
|
+
)
|
|
1548
|
+
assert bound_state["process_evidence"] == [carrier], bound_state
|
|
1549
|
+
|
|
1550
|
+
failed_work = pathlib.Path(tmp) / "failed-work"
|
|
1551
|
+
failed_devlyn = failed_work / ".devlyn"
|
|
1552
|
+
failed_devlyn.mkdir(parents=True)
|
|
1553
|
+
failed_state = {
|
|
1554
|
+
"version": "3.0",
|
|
1555
|
+
"run_id": "rs-failed-mechanical",
|
|
1556
|
+
"engine": "claude",
|
|
1557
|
+
"source": {"type": "spec", "spec_path": "docs/failed/spec.md"},
|
|
1558
|
+
"process_evidence": None,
|
|
1559
|
+
"phases": {"verify": {"round": 0, "verdict": None, "sub_verdicts": None}},
|
|
1560
|
+
}
|
|
1561
|
+
failed_spec = failed_work / "docs" / "failed"
|
|
1562
|
+
failed_spec.mkdir(parents=True)
|
|
1563
|
+
(failed_spec / "spec.md").write_text("# failed fixture\n", encoding="utf-8")
|
|
1564
|
+
(failed_spec / "spec.expected.json").write_text(json.dumps({
|
|
1565
|
+
"verification_commands": [{"cmd": "exit 7", "exit_code": 0}],
|
|
1566
|
+
}) + "\n", encoding="utf-8")
|
|
1567
|
+
failed_obligation = PROCESS_EVIDENCE["normalize_obligation"]({
|
|
1568
|
+
"id": "verification-command-0001",
|
|
1569
|
+
"phase": "verify",
|
|
1570
|
+
"cmd": "exit 7",
|
|
1571
|
+
})
|
|
1572
|
+
failed_manifest = PROCESS_EVIDENCE["manifest_relative_path"](failed_state, "verify")
|
|
1573
|
+
PROCESS_EVIDENCE["capture_process"](
|
|
1574
|
+
failed_work, failed_work / failed_manifest, failed_state["run_id"],
|
|
1575
|
+
"verify", 0, failed_obligation,
|
|
1576
|
+
)
|
|
1577
|
+
failed_carrier = PROCESS_EVIDENCE["validate_manifest"](
|
|
1578
|
+
failed_work, failed_manifest, failed_state["run_id"], "verify", 0,
|
|
1579
|
+
[failed_obligation], require_expectations=False,
|
|
1580
|
+
)
|
|
1581
|
+
(failed_devlyn / "pipeline.state.json").write_text(
|
|
1582
|
+
json.dumps(failed_state), encoding="utf-8",
|
|
1583
|
+
)
|
|
1584
|
+
# Exact laundering attempt: mutable derivatives say PASS/empty while
|
|
1585
|
+
# the state-identical sealed manifest still records the failed command.
|
|
1586
|
+
laundered_commands = PROCESS_EVIDENCE["bound_carrier_summary_commands"](
|
|
1587
|
+
failed_work, failed_carrier,
|
|
1588
|
+
)
|
|
1589
|
+
laundered_commands[0]["pass"] = True
|
|
1590
|
+
(failed_devlyn / "spec-verify.results.json").write_text(json.dumps({
|
|
1591
|
+
"commands": laundered_commands, "process_evidence": failed_carrier,
|
|
1592
|
+
}), encoding="utf-8")
|
|
1593
|
+
(failed_devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1594
|
+
(failed_devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1595
|
+
failed_findings, failed_verdicts = read_findings(failed_devlyn)
|
|
1596
|
+
assert failed_verdicts["mechanical"] == "BLOCKED", failed_verdicts
|
|
1597
|
+
assert any(
|
|
1598
|
+
item.get("id") == "verify-mechanical-evidence-invalid"
|
|
1599
|
+
for item in failed_findings
|
|
1600
|
+
), failed_findings
|
|
1601
|
+
assert write_outputs(failed_devlyn, failed_findings, failed_verdicts)["verdict"] == "BLOCKED"
|
|
1602
|
+
|
|
1603
|
+
(failed_devlyn / "spec-verify.results.json").write_text(json.dumps({
|
|
1604
|
+
"commands": [], "process_evidence": None,
|
|
1605
|
+
}), encoding="utf-8")
|
|
1606
|
+
removed_findings, removed_verdicts = read_findings(failed_devlyn)
|
|
1607
|
+
assert removed_verdicts["mechanical"] == "BLOCKED", removed_verdicts
|
|
1608
|
+
assert any(
|
|
1609
|
+
item.get("id") == "verify-mechanical-evidence-invalid"
|
|
1610
|
+
for item in removed_findings
|
|
1611
|
+
), removed_findings
|
|
1612
|
+
print("PASS iter-0112 sealed VERIFY outcome resists mutable-derivative laundering")
|
|
1613
|
+
|
|
1614
|
+
sealed_stdout = sealed_work / carrier["streams"][0]["stdout"]["path"]
|
|
1615
|
+
sealed_stdout.write_bytes(sealed_stdout.read_bytes() + b"altered")
|
|
1616
|
+
altered_findings, altered_verdicts = read_findings(sealed_devlyn)
|
|
1617
|
+
assert altered_verdicts["mechanical"] == "BLOCKED", altered_verdicts
|
|
1618
|
+
assert any(
|
|
1619
|
+
finding.get("id") == "verify-mechanical-evidence-invalid"
|
|
1620
|
+
for finding in altered_findings
|
|
1621
|
+
), altered_findings
|
|
1622
|
+
|
|
1623
|
+
skipped_work = pathlib.Path(tmp) / "skipped-work"
|
|
1624
|
+
skipped_work.mkdir()
|
|
1625
|
+
(skipped_work / "pipeline.state.json").write_text(json.dumps({
|
|
1626
|
+
"engine": "claude",
|
|
1627
|
+
"risk_profile": {"pair_default_enabled": False},
|
|
1628
|
+
"phases": {"verify": {"pair_trigger": {
|
|
1629
|
+
"eligible": False,
|
|
1630
|
+
"reasons": [],
|
|
1631
|
+
"skipped_reason": "user_no_pair",
|
|
1632
|
+
}}},
|
|
1633
|
+
}), encoding="utf-8")
|
|
1634
|
+
(skipped_work / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
1635
|
+
(skipped_work / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1636
|
+
generic_primary = skipped_work / "verify-judge.stdout"
|
|
1637
|
+
for generic_output in ("PASS\n", "NEEDS_WORK\n"):
|
|
1638
|
+
generic_primary.write_text(generic_output, encoding="utf-8")
|
|
1639
|
+
generic_findings, generic_verdicts = read_findings(skipped_work)
|
|
1640
|
+
assert generic_verdicts["pair_judge"] is None, generic_verdicts
|
|
1641
|
+
assert not any(
|
|
1642
|
+
finding.get("file") == generic_primary.name for finding in generic_findings
|
|
1643
|
+
), generic_findings
|
|
1644
|
+
(skipped_work / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
1645
|
+
contradiction_findings, contradiction_verdicts = read_findings(skipped_work)
|
|
1646
|
+
assert contradiction_verdicts["pair_judge"] == "BLOCKED", contradiction_verdicts
|
|
1647
|
+
assert any(
|
|
1648
|
+
finding.get("id") == "verify-pair-skipped-output-present"
|
|
1649
|
+
for finding in contradiction_findings
|
|
1650
|
+
), contradiction_findings
|
|
1019
1651
|
findings, source_verdicts = read_findings(devlyn)
|
|
1020
1652
|
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1021
1653
|
write_state(devlyn, summary)
|
|
@@ -1027,12 +1659,92 @@ def self_test() -> int:
|
|
|
1027
1659
|
assert state["phases"]["verify"]["judge_durations_ms"] == {
|
|
1028
1660
|
"judge": 23, "pair_judge": 31,
|
|
1029
1661
|
}, state
|
|
1662
|
+
original_state = (devlyn / "pipeline.state.json").read_bytes()
|
|
1663
|
+
for malformed in (None, {}, {"roles": {}}):
|
|
1664
|
+
bad_state = loads_strict_json(original_state)
|
|
1665
|
+
bad_state["role_resolution"] = malformed
|
|
1666
|
+
(devlyn / "pipeline.state.json").write_text(json.dumps(bad_state))
|
|
1667
|
+
result = subprocess.run([sys.executable, str(pathlib.Path(__file__).resolve()),
|
|
1668
|
+
"--devlyn-dir", str(devlyn), "--write-state"], capture_output=True, text=True)
|
|
1669
|
+
assert result.returncode == 0 and "Traceback" not in result.stderr, result.stderr
|
|
1670
|
+
persisted = loads_strict_json((devlyn / "pipeline.state.json").read_text())
|
|
1671
|
+
assert persisted["phases"]["verify"]["verdict"] == "BLOCKED"
|
|
1672
|
+
assert "verify-role-resolution-invalid" in (devlyn / "verify-merged.findings.jsonl").read_text()
|
|
1673
|
+
(devlyn / "pipeline.state.json").write_bytes(original_state)
|
|
1674
|
+
different_primary = {"engine": "codex", "phases": {"verify": {"engine": "claude"}}}
|
|
1675
|
+
(devlyn / "pipeline.state.json").write_text(json.dumps(different_primary))
|
|
1676
|
+
(devlyn / "verify.primary.timeout.json").write_text(json.dumps({"engine": "claude", "budget_seconds": 600}))
|
|
1677
|
+
marker, violation = read_primary_timeout_marker(devlyn)
|
|
1678
|
+
assert marker is not None and violation is None
|
|
1679
|
+
different_primary["phases"]["verify"]["engine"] = None
|
|
1680
|
+
(devlyn / "pipeline.state.json").write_text(json.dumps(different_primary))
|
|
1681
|
+
assert read_primary_timeout_marker(devlyn)[1] is not None
|
|
1682
|
+
(devlyn / "verify.primary.timeout.json").unlink()
|
|
1683
|
+
(devlyn / "pipeline.state.json").write_bytes(original_state)
|
|
1684
|
+
|
|
1685
|
+
(devlyn / "verify.findings.jsonl").unlink()
|
|
1686
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
1687
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1688
|
+
assert summary["verdict"] == "BLOCKED", summary
|
|
1689
|
+
assert summary["source_verdicts"]["judge"] == "BLOCKED", summary
|
|
1690
|
+
assert any(
|
|
1691
|
+
finding["id"] == "verify-merge-required-source-missing-judge"
|
|
1692
|
+
for finding in findings
|
|
1693
|
+
), findings
|
|
1694
|
+
|
|
1695
|
+
# iter-0113: a valid primary timeout is an explicit BLOCKED source,
|
|
1696
|
+
# never the generic missing-source path or pair-style TIMEOUT.
|
|
1697
|
+
(devlyn / "verify.primary.timeout.json").write_text(
|
|
1698
|
+
json.dumps({"engine": "claude", "budget_seconds": 600}) + "\n",
|
|
1699
|
+
encoding="utf-8",
|
|
1700
|
+
)
|
|
1701
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
1702
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1703
|
+
assert summary["verdict"] == "BLOCKED", summary
|
|
1704
|
+
assert summary["source_verdicts"]["judge"] == "BLOCKED", summary
|
|
1705
|
+
assert any(
|
|
1706
|
+
finding["id"] == "verify-primary-timeout"
|
|
1707
|
+
for finding in findings
|
|
1708
|
+
), findings
|
|
1709
|
+
assert not any(
|
|
1710
|
+
finding["id"] == "verify-merge-required-source-missing-judge"
|
|
1711
|
+
for finding in findings
|
|
1712
|
+
), findings
|
|
1713
|
+
|
|
1714
|
+
(devlyn / "verify.findings.jsonl").write_text(
|
|
1715
|
+
json.dumps({"id": "primary-timeout-high", "severity": "HIGH"}) + "\n",
|
|
1716
|
+
encoding="utf-8",
|
|
1717
|
+
)
|
|
1718
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
1719
|
+
assert source_verdicts["judge"] == "BLOCKED", source_verdicts
|
|
1720
|
+
assert {finding["id"] for finding in findings} >= {
|
|
1721
|
+
"primary-timeout-high", "verify-primary-timeout",
|
|
1722
|
+
}, findings
|
|
1723
|
+
|
|
1724
|
+
for marker, expected_id in (
|
|
1725
|
+
("{\n", "verify-primary-timeout-marker-malformed"),
|
|
1726
|
+
(
|
|
1727
|
+
json.dumps({"engine": "codex", "budget_seconds": 600}) + "\n",
|
|
1728
|
+
"verify-primary-timeout-engine-mismatch",
|
|
1729
|
+
),
|
|
1730
|
+
(
|
|
1731
|
+
json.dumps({"engine": "claude", "budget_seconds": 601}) + "\n",
|
|
1732
|
+
"verify-primary-timeout-budget-mismatch",
|
|
1733
|
+
),
|
|
1734
|
+
):
|
|
1735
|
+
(devlyn / "verify.primary.timeout.json").write_text(marker, encoding="utf-8")
|
|
1736
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
1737
|
+
assert source_verdicts["judge"] == "BLOCKED", source_verdicts
|
|
1738
|
+
assert any(finding["id"] == expected_id for finding in findings), findings
|
|
1739
|
+
(devlyn / "verify.primary.timeout.json").unlink()
|
|
1740
|
+
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
1030
1741
|
|
|
1031
1742
|
# iter-0072 Amendment 3: generated schema-v3 runs mechanically prove
|
|
1032
1743
|
# raw-goal persistence and required SURFACE_CLOSE dispatch.
|
|
1033
1744
|
(devlyn / "pipeline.state.json").write_text(
|
|
1034
1745
|
json.dumps({
|
|
1035
1746
|
"version": "3.0",
|
|
1747
|
+
"engine": "claude",
|
|
1036
1748
|
"complexity": "large",
|
|
1037
1749
|
"source": {"type": "generated"},
|
|
1038
1750
|
"phases": {"verify": {"verdict": None, "sub_verdicts": None}},
|
|
@@ -1051,6 +1763,7 @@ def self_test() -> int:
|
|
|
1051
1763
|
(devlyn / "pipeline.state.json").write_text(
|
|
1052
1764
|
json.dumps({
|
|
1053
1765
|
"version": "3.0",
|
|
1766
|
+
"engine": "claude",
|
|
1054
1767
|
"complexity": "large",
|
|
1055
1768
|
"source": {"type": "generated", "goal_path": "", "goal_sha256": "A" * 64},
|
|
1056
1769
|
}),
|
|
@@ -1068,6 +1781,7 @@ def self_test() -> int:
|
|
|
1068
1781
|
(devlyn / "pipeline.state.json").write_text(
|
|
1069
1782
|
json.dumps({
|
|
1070
1783
|
"version": "3.0",
|
|
1784
|
+
"engine": "claude",
|
|
1071
1785
|
"complexity": "medium",
|
|
1072
1786
|
"source": generated_source,
|
|
1073
1787
|
"phases": {
|
|
@@ -1096,6 +1810,7 @@ def self_test() -> int:
|
|
|
1096
1810
|
(devlyn / "pipeline.state.json").write_text(
|
|
1097
1811
|
json.dumps({
|
|
1098
1812
|
"version": "3.0",
|
|
1813
|
+
"engine": "claude",
|
|
1099
1814
|
"complexity": "medium",
|
|
1100
1815
|
"source": generated_source,
|
|
1101
1816
|
"phases": {"surface_close": surface_entry},
|
|
@@ -1107,6 +1822,7 @@ def self_test() -> int:
|
|
|
1107
1822
|
(devlyn / "pipeline.state.json").write_text(
|
|
1108
1823
|
json.dumps({
|
|
1109
1824
|
"version": "3.0",
|
|
1825
|
+
"engine": "claude",
|
|
1110
1826
|
"complexity": "medium",
|
|
1111
1827
|
"source": generated_source,
|
|
1112
1828
|
"phases": {
|
|
@@ -1123,15 +1839,15 @@ def self_test() -> int:
|
|
|
1123
1839
|
|
|
1124
1840
|
for state_shape in (
|
|
1125
1841
|
{
|
|
1126
|
-
"version": "3.0", "complexity": "medium",
|
|
1842
|
+
"version": "3.0", "engine": "claude", "complexity": "medium",
|
|
1127
1843
|
"source": {"type": "spec"}, "phases": {"surface_close": None},
|
|
1128
1844
|
},
|
|
1129
1845
|
{
|
|
1130
|
-
"version": "3.0", "complexity": "large",
|
|
1846
|
+
"version": "3.0", "engine": "claude", "complexity": "large",
|
|
1131
1847
|
"source": generated_source, "phases": {"surface_close": None},
|
|
1132
1848
|
},
|
|
1133
1849
|
{
|
|
1134
|
-
"version": "2.0", "complexity": "medium",
|
|
1850
|
+
"version": "2.0", "engine": "claude", "complexity": "medium",
|
|
1135
1851
|
"source": {"type": "generated"}, "phases": {"surface_close": None},
|
|
1136
1852
|
},
|
|
1137
1853
|
):
|
|
@@ -1146,6 +1862,7 @@ def self_test() -> int:
|
|
|
1146
1862
|
(devlyn / "pipeline.state.json").write_text(
|
|
1147
1863
|
json.dumps({
|
|
1148
1864
|
"version": "3.0",
|
|
1865
|
+
"engine": "claude",
|
|
1149
1866
|
"mode": "spec",
|
|
1150
1867
|
"risk_profile": {"pair_default_enabled": True},
|
|
1151
1868
|
"phases": {
|
|
@@ -1173,7 +1890,7 @@ def self_test() -> int:
|
|
|
1173
1890
|
# A non-null, non-dict sub_verdicts is corrupted state, not a legal
|
|
1174
1891
|
# placeholder — write_state() must fail loud, not silently coerce it.
|
|
1175
1892
|
(devlyn / "pipeline.state.json").write_text(
|
|
1176
|
-
json.dumps({"phases": {"verify": {"verdict": None, "sub_verdicts": "corrupt"}}}),
|
|
1893
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": None, "sub_verdicts": "corrupt"}}}),
|
|
1177
1894
|
encoding="utf-8",
|
|
1178
1895
|
)
|
|
1179
1896
|
findings, source_verdicts = read_findings(devlyn)
|
|
@@ -1187,6 +1904,7 @@ def self_test() -> int:
|
|
|
1187
1904
|
|
|
1188
1905
|
(devlyn / "pipeline.state.json").write_text(
|
|
1189
1906
|
json.dumps({
|
|
1907
|
+
"engine": "claude",
|
|
1190
1908
|
"phases": {
|
|
1191
1909
|
"verify": {
|
|
1192
1910
|
"verdict": "PASS",
|
|
@@ -1209,6 +1927,7 @@ def self_test() -> int:
|
|
|
1209
1927
|
json.dumps({"id": "p1", "severity": "HIGH"}) + "\n",
|
|
1210
1928
|
encoding="utf-8",
|
|
1211
1929
|
)
|
|
1930
|
+
(devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
1212
1931
|
findings, source_verdicts = read_findings(devlyn)
|
|
1213
1932
|
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
1214
1933
|
write_state(devlyn, summary)
|
|
@@ -1217,6 +1936,7 @@ def self_test() -> int:
|
|
|
1217
1936
|
assert state["phases"]["verify"]["verdict"] == "NEEDS_WORK", state
|
|
1218
1937
|
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "NEEDS_WORK", state
|
|
1219
1938
|
assert (devlyn / "verify-merged.findings.jsonl").read_text(encoding="utf-8")
|
|
1939
|
+
(devlyn / "codex-judge.stdout").unlink()
|
|
1220
1940
|
(devlyn / "verify.findings.jsonl").write_text(
|
|
1221
1941
|
'{"id":"nan","severity":NaN}\n',
|
|
1222
1942
|
encoding="utf-8",
|
|
@@ -1230,7 +1950,7 @@ def self_test() -> int:
|
|
|
1230
1950
|
for finding in findings
|
|
1231
1951
|
), findings
|
|
1232
1952
|
(devlyn / "pipeline.state.json").write_text(
|
|
1233
|
-
json.dumps({"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1953
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1234
1954
|
encoding="utf-8",
|
|
1235
1955
|
)
|
|
1236
1956
|
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
@@ -1258,6 +1978,10 @@ def self_test() -> int:
|
|
|
1258
1978
|
# Engine-neutral stdout contract (iter-0060): a Claude pair-judge
|
|
1259
1979
|
# capture (claude-judge.stdout, adapters/claude.md ## Invocation)
|
|
1260
1980
|
# binds the same emission contract as the Codex one.
|
|
1981
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
1982
|
+
json.dumps({"engine": "codex", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1983
|
+
encoding="utf-8",
|
|
1984
|
+
)
|
|
1261
1985
|
(devlyn / "claude-judge.stdout").write_text(
|
|
1262
1986
|
json.dumps({"id": "clj1", "severity": "HIGH"}) + "\n",
|
|
1263
1987
|
encoding="utf-8",
|
|
@@ -1276,6 +2000,7 @@ def self_test() -> int:
|
|
|
1276
2000
|
# records TIMEOUT and leaves the merged verdict to mechanical+primary.
|
|
1277
2001
|
(devlyn / "pipeline.state.json").write_text(
|
|
1278
2002
|
json.dumps({
|
|
2003
|
+
"engine": "codex",
|
|
1279
2004
|
"phases": {
|
|
1280
2005
|
"verify": {
|
|
1281
2006
|
"verdict": "PASS",
|
|
@@ -1315,7 +2040,7 @@ def self_test() -> int:
|
|
|
1315
2040
|
|
|
1316
2041
|
# iter-0065 case 2: canonical pair findings still bind after timeout.
|
|
1317
2042
|
(devlyn / "pipeline.state.json").write_text(
|
|
1318
|
-
json.dumps({"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2043
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1319
2044
|
encoding="utf-8",
|
|
1320
2045
|
)
|
|
1321
2046
|
(devlyn / "verify.pair.findings.jsonl").write_text(
|
|
@@ -1337,7 +2062,7 @@ def self_test() -> int:
|
|
|
1337
2062
|
# iter-0065 case 2b: stdout-only HIGH still blocks on emission contract;
|
|
1338
2063
|
# timeout never converts an observed finding into a solo pass.
|
|
1339
2064
|
(devlyn / "pipeline.state.json").write_text(
|
|
1340
|
-
json.dumps({"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2065
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1341
2066
|
encoding="utf-8",
|
|
1342
2067
|
)
|
|
1343
2068
|
(devlyn / "codex-judge.stdout").write_text(
|
|
@@ -1359,9 +2084,125 @@ def self_test() -> int:
|
|
|
1359
2084
|
(devlyn / "codex-judge.stdout").unlink()
|
|
1360
2085
|
(devlyn / "verify.pair.timeout.json").unlink()
|
|
1361
2086
|
|
|
2087
|
+
# iter-0106: a budget abort can truncate the whole-message stream before
|
|
2088
|
+
# its terminal result. With a valid marker that capture stays TIMEOUT.
|
|
2089
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
2090
|
+
json.dumps({
|
|
2091
|
+
"engine": "claude",
|
|
2092
|
+
"phases": {
|
|
2093
|
+
"verify": {
|
|
2094
|
+
"verdict": "PASS",
|
|
2095
|
+
"sub_verdicts": {},
|
|
2096
|
+
"pair_trigger": {
|
|
2097
|
+
"eligible": True,
|
|
2098
|
+
"reasons": ["judge.warning"],
|
|
2099
|
+
"skipped_reason": None,
|
|
2100
|
+
},
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
}),
|
|
2104
|
+
encoding="utf-8",
|
|
2105
|
+
)
|
|
2106
|
+
partial_stream = (
|
|
2107
|
+
'{"type":"system","subtype":"init","session_id":"s106"}\n'
|
|
2108
|
+
'{"type":"assistant","session_id":"s106","message":{"stop_reason":null,'
|
|
2109
|
+
'"content":[{"type":"text","text":"reading the diff"}]}}\n'
|
|
2110
|
+
)
|
|
2111
|
+
(devlyn / "grok-judge.stdout").write_text(partial_stream, encoding="utf-8")
|
|
2112
|
+
(devlyn / "verify.pair.timeout.json").write_text(
|
|
2113
|
+
json.dumps({"engine": "grok", "budget_seconds": 600}),
|
|
2114
|
+
encoding="utf-8",
|
|
2115
|
+
)
|
|
2116
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2117
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2118
|
+
assert summary["verdict"] == "PASS", summary
|
|
2119
|
+
assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
|
|
2120
|
+
assert summary["pair_timeout"] == {"engine": "grok", "budget_seconds": 600}, summary
|
|
2121
|
+
assert not findings, findings
|
|
2122
|
+
|
|
2123
|
+
# A kill may cut the final JSON record mid-write; one malformed tail is
|
|
2124
|
+
# still a valid prefix, while malformed/unknown complete records are not.
|
|
2125
|
+
(devlyn / "grok-judge.stdout").write_text(
|
|
2126
|
+
partial_stream + '{"type":"assistant","session_id":"s106"',
|
|
2127
|
+
encoding="utf-8",
|
|
2128
|
+
)
|
|
2129
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2130
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2131
|
+
assert summary["source_verdicts"]["pair_judge"] == "TIMEOUT", summary
|
|
2132
|
+
assert not findings, findings
|
|
2133
|
+
|
|
2134
|
+
for invalid_prefix in (
|
|
2135
|
+
'{"type":"system","subtype":"init"}\n',
|
|
2136
|
+
partial_stream + "trailing narration\n",
|
|
2137
|
+
partial_stream
|
|
2138
|
+
+ '{"type":"user","session_id":"other"}\n',
|
|
2139
|
+
partial_stream
|
|
2140
|
+
+ '{"type":"stream_event","session_id":"s106"}\n',
|
|
2141
|
+
partial_stream
|
|
2142
|
+
+ '{"type":"system","subtype":"compact_boundary","session_id":"s106"}\n',
|
|
2143
|
+
partial_stream
|
|
2144
|
+
+ '{"type":"assistant"\n'
|
|
2145
|
+
+ '{"type":"user","session_id":"s106"}\n',
|
|
2146
|
+
):
|
|
2147
|
+
(devlyn / "grok-judge.stdout").write_text(invalid_prefix, encoding="utf-8")
|
|
2148
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2149
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2150
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2151
|
+
assert any(
|
|
2152
|
+
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2153
|
+
for finding in findings
|
|
2154
|
+
), findings
|
|
2155
|
+
|
|
2156
|
+
# iter-0106 control: the marker conserves TIMEOUT only for that shape. A
|
|
2157
|
+
# stream that reached its result but welded narration into the terminal
|
|
2158
|
+
# message stays BLOCKED on the emission contract.
|
|
2159
|
+
(devlyn / "grok-judge.stdout").write_text(
|
|
2160
|
+
'{"type":"system","subtype":"init","session_id":"s106"}\n'
|
|
2161
|
+
+ json.dumps({
|
|
2162
|
+
"type": "assistant",
|
|
2163
|
+
"session_id": "s106",
|
|
2164
|
+
"message": {
|
|
2165
|
+
"stop_reason": "end_turn",
|
|
2166
|
+
"content": [{"type": "text", "text": "I reviewed the diff.\nPASS"}],
|
|
2167
|
+
},
|
|
2168
|
+
}) + "\n"
|
|
2169
|
+
+ json.dumps({
|
|
2170
|
+
"type": "result",
|
|
2171
|
+
"subtype": "success",
|
|
2172
|
+
"is_error": False,
|
|
2173
|
+
"stop_reason": "end_turn",
|
|
2174
|
+
"session_id": "s106",
|
|
2175
|
+
"result": "I reviewed the diff.\nPASS",
|
|
2176
|
+
}) + "\n",
|
|
2177
|
+
encoding="utf-8",
|
|
2178
|
+
)
|
|
2179
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2180
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2181
|
+
assert summary["verdict"] == "BLOCKED", summary
|
|
2182
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2183
|
+
assert any(
|
|
2184
|
+
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2185
|
+
for finding in findings
|
|
2186
|
+
), findings
|
|
2187
|
+
(devlyn / "verify.pair.timeout.json").unlink()
|
|
2188
|
+
|
|
2189
|
+
# iter-0106 control: without a marker the same truncated stream is a
|
|
2190
|
+
# plain emission-contract violation.
|
|
2191
|
+
(devlyn / "grok-judge.stdout").write_text(partial_stream, encoding="utf-8")
|
|
2192
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
2193
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2194
|
+
assert summary["verdict"] == "BLOCKED", summary
|
|
2195
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", summary
|
|
2196
|
+
assert any(
|
|
2197
|
+
finding.get("id") == "verify-pair-emission-contract-violated"
|
|
2198
|
+
for finding in findings
|
|
2199
|
+
), findings
|
|
2200
|
+
(devlyn / "grok-judge.stdout").unlink()
|
|
2201
|
+
|
|
1362
2202
|
# iter-0065 case 3: without a marker, required empty pair output remains BLOCKED.
|
|
1363
2203
|
(devlyn / "pipeline.state.json").write_text(
|
|
1364
2204
|
json.dumps({
|
|
2205
|
+
"engine": "codex",
|
|
1365
2206
|
"phases": {
|
|
1366
2207
|
"verify": {
|
|
1367
2208
|
"verdict": "PASS",
|
|
@@ -1388,7 +2229,7 @@ def self_test() -> int:
|
|
|
1388
2229
|
|
|
1389
2230
|
# iter-0065 malformed timeout markers fail closed as a CRITICAL pair blocker.
|
|
1390
2231
|
(devlyn / "pipeline.state.json").write_text(
|
|
1391
|
-
json.dumps({"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
2232
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}}}),
|
|
1392
2233
|
encoding="utf-8",
|
|
1393
2234
|
)
|
|
1394
2235
|
(devlyn / "verify.pair.timeout.json").write_text(
|
|
@@ -1407,6 +2248,7 @@ def self_test() -> int:
|
|
|
1407
2248
|
(devlyn / "verify.pair.timeout.json").unlink()
|
|
1408
2249
|
(devlyn / "pipeline.state.json").write_text(
|
|
1409
2250
|
json.dumps({
|
|
2251
|
+
"engine": "claude",
|
|
1410
2252
|
"phases": {
|
|
1411
2253
|
"verify": {
|
|
1412
2254
|
"verdict": "PASS",
|
|
@@ -1434,6 +2276,7 @@ def self_test() -> int:
|
|
|
1434
2276
|
|
|
1435
2277
|
(devlyn / "pipeline.state.json").write_text(
|
|
1436
2278
|
json.dumps({
|
|
2279
|
+
"engine": "claude",
|
|
1437
2280
|
"mode": "spec",
|
|
1438
2281
|
"risk_profile": {
|
|
1439
2282
|
"high_risk": True,
|
|
@@ -1454,6 +2297,7 @@ def self_test() -> int:
|
|
|
1454
2297
|
|
|
1455
2298
|
(devlyn / "pipeline.state.json").write_text(
|
|
1456
2299
|
json.dumps({
|
|
2300
|
+
"engine": "claude",
|
|
1457
2301
|
"mode": "spec",
|
|
1458
2302
|
"risk_profile": "enabled",
|
|
1459
2303
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1471,6 +2315,7 @@ def self_test() -> int:
|
|
|
1471
2315
|
|
|
1472
2316
|
(devlyn / "pipeline.state.json").write_text(
|
|
1473
2317
|
json.dumps({
|
|
2318
|
+
"engine": "claude",
|
|
1474
2319
|
"mode": "spec",
|
|
1475
2320
|
"risk_profile": {
|
|
1476
2321
|
"high_risk": True,
|
|
@@ -1492,6 +2337,7 @@ def self_test() -> int:
|
|
|
1492
2337
|
|
|
1493
2338
|
(devlyn / "pipeline.state.json").write_text(
|
|
1494
2339
|
json.dumps({
|
|
2340
|
+
"engine": "claude",
|
|
1495
2341
|
"mode": "spec",
|
|
1496
2342
|
"risk_profile": {
|
|
1497
2343
|
"high_risk": True,
|
|
@@ -1514,6 +2360,7 @@ def self_test() -> int:
|
|
|
1514
2360
|
|
|
1515
2361
|
(devlyn / "pipeline.state.json").write_text(
|
|
1516
2362
|
json.dumps({
|
|
2363
|
+
"engine": "claude",
|
|
1517
2364
|
"mode": "spec",
|
|
1518
2365
|
"pair_verify": True,
|
|
1519
2366
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1531,6 +2378,7 @@ def self_test() -> int:
|
|
|
1531
2378
|
|
|
1532
2379
|
(devlyn / "pipeline.state.json").write_text(
|
|
1533
2380
|
json.dumps({
|
|
2381
|
+
"engine": "claude",
|
|
1534
2382
|
"mode": "spec",
|
|
1535
2383
|
"complexity": "large",
|
|
1536
2384
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1553,6 +2401,7 @@ def self_test() -> int:
|
|
|
1553
2401
|
)
|
|
1554
2402
|
(devlyn / "pipeline.state.json").write_text(
|
|
1555
2403
|
json.dumps({
|
|
2404
|
+
"engine": "claude",
|
|
1556
2405
|
"mode": "spec",
|
|
1557
2406
|
"source": {"spec_path": str(spec_path)},
|
|
1558
2407
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1590,6 +2439,7 @@ def self_test() -> int:
|
|
|
1590
2439
|
) is False
|
|
1591
2440
|
(devlyn / "pipeline.state.json").write_text(
|
|
1592
2441
|
json.dumps({
|
|
2442
|
+
"engine": "claude",
|
|
1593
2443
|
"mode": "spec",
|
|
1594
2444
|
"source": {"spec_path": str(spec_path)},
|
|
1595
2445
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1638,6 +2488,7 @@ def self_test() -> int:
|
|
|
1638
2488
|
) is True
|
|
1639
2489
|
(devlyn / "pipeline.state.json").write_text(
|
|
1640
2490
|
json.dumps({
|
|
2491
|
+
"engine": "claude",
|
|
1641
2492
|
"mode": "spec",
|
|
1642
2493
|
"source": {"spec_path": str(spec_path)},
|
|
1643
2494
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1655,6 +2506,7 @@ def self_test() -> int:
|
|
|
1655
2506
|
|
|
1656
2507
|
(devlyn / "pipeline.state.json").write_text(
|
|
1657
2508
|
json.dumps({
|
|
2509
|
+
"engine": "claude",
|
|
1658
2510
|
"mode": "spec",
|
|
1659
2511
|
"source": {"spec_path": str(spec_path)},
|
|
1660
2512
|
"risk_profile": {
|
|
@@ -1695,6 +2547,7 @@ def self_test() -> int:
|
|
|
1695
2547
|
) is True
|
|
1696
2548
|
(devlyn / "pipeline.state.json").write_text(
|
|
1697
2549
|
json.dumps({
|
|
2550
|
+
"engine": "claude",
|
|
1698
2551
|
"mode": "free-form",
|
|
1699
2552
|
"source": {"criteria_path": str(criteria_path)},
|
|
1700
2553
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
@@ -1725,6 +2578,7 @@ def self_test() -> int:
|
|
|
1725
2578
|
|
|
1726
2579
|
(devlyn / "pipeline.state.json").write_text(
|
|
1727
2580
|
json.dumps({
|
|
2581
|
+
"engine": "claude",
|
|
1728
2582
|
"phases": {
|
|
1729
2583
|
"verify": {
|
|
1730
2584
|
"verdict": "PASS",
|
|
@@ -1749,6 +2603,7 @@ def self_test() -> int:
|
|
|
1749
2603
|
|
|
1750
2604
|
(devlyn / "pipeline.state.json").write_text(
|
|
1751
2605
|
json.dumps({
|
|
2606
|
+
"engine": "claude",
|
|
1752
2607
|
"phases": {
|
|
1753
2608
|
"verify": {
|
|
1754
2609
|
"verdict": "PASS",
|
|
@@ -1773,6 +2628,7 @@ def self_test() -> int:
|
|
|
1773
2628
|
|
|
1774
2629
|
(devlyn / "pipeline.state.json").write_text(
|
|
1775
2630
|
json.dumps({
|
|
2631
|
+
"engine": "claude",
|
|
1776
2632
|
"phases": {
|
|
1777
2633
|
"verify": {
|
|
1778
2634
|
"verdict": "PASS",
|
|
@@ -1798,6 +2654,7 @@ def self_test() -> int:
|
|
|
1798
2654
|
|
|
1799
2655
|
(devlyn / "pipeline.state.json").write_text(
|
|
1800
2656
|
json.dumps({
|
|
2657
|
+
"engine": "claude",
|
|
1801
2658
|
"phases": {
|
|
1802
2659
|
"verify": {
|
|
1803
2660
|
"verdict": "PASS",
|
|
@@ -1823,6 +2680,7 @@ def self_test() -> int:
|
|
|
1823
2680
|
|
|
1824
2681
|
(devlyn / "pipeline.state.json").write_text(
|
|
1825
2682
|
json.dumps({
|
|
2683
|
+
"engine": "claude",
|
|
1826
2684
|
"phases": {
|
|
1827
2685
|
"verify": {
|
|
1828
2686
|
"verdict": "PASS",
|
|
@@ -1848,6 +2706,7 @@ def self_test() -> int:
|
|
|
1848
2706
|
|
|
1849
2707
|
(devlyn / "pipeline.state.json").write_text(
|
|
1850
2708
|
json.dumps({
|
|
2709
|
+
"engine": "claude",
|
|
1851
2710
|
"phases": {
|
|
1852
2711
|
"verify": {
|
|
1853
2712
|
"verdict": "PASS",
|
|
@@ -1872,6 +2731,7 @@ def self_test() -> int:
|
|
|
1872
2731
|
|
|
1873
2732
|
(devlyn / "pipeline.state.json").write_text(
|
|
1874
2733
|
json.dumps({
|
|
2734
|
+
"engine": "claude",
|
|
1875
2735
|
"phases": {
|
|
1876
2736
|
"verify": {
|
|
1877
2737
|
"verdict": "PASS",
|
|
@@ -1896,6 +2756,7 @@ def self_test() -> int:
|
|
|
1896
2756
|
|
|
1897
2757
|
(devlyn / "pipeline.state.json").write_text(
|
|
1898
2758
|
json.dumps({
|
|
2759
|
+
"engine": "claude",
|
|
1899
2760
|
"phases": {
|
|
1900
2761
|
"verify": {
|
|
1901
2762
|
"verdict": "PASS",
|
|
@@ -1920,6 +2781,7 @@ def self_test() -> int:
|
|
|
1920
2781
|
|
|
1921
2782
|
(devlyn / "pipeline.state.json").write_text(
|
|
1922
2783
|
json.dumps({
|
|
2784
|
+
"engine": "claude",
|
|
1923
2785
|
"phases": {
|
|
1924
2786
|
"verify": {
|
|
1925
2787
|
"verdict": "PASS",
|
|
@@ -1944,6 +2806,7 @@ def self_test() -> int:
|
|
|
1944
2806
|
|
|
1945
2807
|
(devlyn / "pipeline.state.json").write_text(
|
|
1946
2808
|
json.dumps({
|
|
2809
|
+
"engine": "claude",
|
|
1947
2810
|
"mode": "spec",
|
|
1948
2811
|
"risk_profile": {
|
|
1949
2812
|
"high_risk": True,
|
|
@@ -1975,6 +2838,7 @@ def self_test() -> int:
|
|
|
1975
2838
|
|
|
1976
2839
|
(devlyn / "pipeline.state.json").write_text(
|
|
1977
2840
|
json.dumps({
|
|
2841
|
+
"engine": "claude",
|
|
1978
2842
|
"mode": "spec",
|
|
1979
2843
|
"risk_profile": {
|
|
1980
2844
|
"high_risk": True,
|
|
@@ -2006,6 +2870,7 @@ def self_test() -> int:
|
|
|
2006
2870
|
|
|
2007
2871
|
(devlyn / "pipeline.state.json").write_text(
|
|
2008
2872
|
json.dumps({
|
|
2873
|
+
"engine": "claude",
|
|
2009
2874
|
"pair_verify": True,
|
|
2010
2875
|
"risk_profile": {
|
|
2011
2876
|
"high_risk": True,
|
|
@@ -2036,6 +2901,7 @@ def self_test() -> int:
|
|
|
2036
2901
|
|
|
2037
2902
|
(devlyn / "pipeline.state.json").write_text(
|
|
2038
2903
|
json.dumps({
|
|
2904
|
+
"engine": "claude",
|
|
2039
2905
|
"mode": "spec",
|
|
2040
2906
|
"risk_profile": {
|
|
2041
2907
|
"high_risk": True,
|
|
@@ -2066,6 +2932,7 @@ def self_test() -> int:
|
|
|
2066
2932
|
|
|
2067
2933
|
(devlyn / "pipeline.state.json").write_text(
|
|
2068
2934
|
json.dumps({
|
|
2935
|
+
"engine": "claude",
|
|
2069
2936
|
"phases": {
|
|
2070
2937
|
"verify": {
|
|
2071
2938
|
"verdict": "PASS",
|
|
@@ -2090,6 +2957,7 @@ def self_test() -> int:
|
|
|
2090
2957
|
|
|
2091
2958
|
(devlyn / "pipeline.state.json").write_text(
|
|
2092
2959
|
json.dumps({
|
|
2960
|
+
"engine": "claude",
|
|
2093
2961
|
"phases": {
|
|
2094
2962
|
"verify": {
|
|
2095
2963
|
"verdict": "PASS",
|
|
@@ -2114,6 +2982,7 @@ def self_test() -> int:
|
|
|
2114
2982
|
|
|
2115
2983
|
(devlyn / "pipeline.state.json").write_text(
|
|
2116
2984
|
json.dumps({
|
|
2985
|
+
"engine": "claude",
|
|
2117
2986
|
"phases": {
|
|
2118
2987
|
"verify": {
|
|
2119
2988
|
"verdict": "PASS",
|
|
@@ -2147,10 +3016,34 @@ def self_test() -> int:
|
|
|
2147
3016
|
finding.get("id") == "verify-pair-trigger-mechanical-blocker-unsupported"
|
|
2148
3017
|
for finding in findings
|
|
2149
3018
|
), findings
|
|
3019
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
3020
|
+
json.dumps({
|
|
3021
|
+
"engine": "claude",
|
|
3022
|
+
"phases": {
|
|
3023
|
+
"verify": {
|
|
3024
|
+
"verdict": None,
|
|
3025
|
+
"sub_verdicts": None,
|
|
3026
|
+
"judge_durations_ms": {"judge": None, "pair_judge": None},
|
|
3027
|
+
}
|
|
3028
|
+
}
|
|
3029
|
+
}),
|
|
3030
|
+
encoding="utf-8",
|
|
3031
|
+
)
|
|
3032
|
+
(devlyn / "verify.findings.jsonl").unlink()
|
|
3033
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3034
|
+
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
3035
|
+
write_state(devlyn, summary)
|
|
3036
|
+
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
3037
|
+
assert summary["verdict"] == "NEEDS_WORK", summary
|
|
3038
|
+
assert summary["source_verdicts"]["judge"] is None, summary
|
|
3039
|
+
assert state["phases"]["verify"]["verdict"] == "NEEDS_WORK", state
|
|
3040
|
+
assert state["phases"]["verify"]["sub_verdicts"]["judge"] is None, state
|
|
3041
|
+
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
2150
3042
|
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
2151
3043
|
|
|
2152
3044
|
(devlyn / "pipeline.state.json").write_text(
|
|
2153
3045
|
json.dumps({
|
|
3046
|
+
"engine": "claude",
|
|
2154
3047
|
"phases": {
|
|
2155
3048
|
"verify": {
|
|
2156
3049
|
"verdict": "PASS",
|
|
@@ -2182,6 +3075,7 @@ def self_test() -> int:
|
|
|
2182
3075
|
(devlyn / "pipeline.state.json").write_text(
|
|
2183
3076
|
json.dumps({
|
|
2184
3077
|
"version": "2.0",
|
|
3078
|
+
"engine": "claude",
|
|
2185
3079
|
"mode": "spec",
|
|
2186
3080
|
"pair_verify": True,
|
|
2187
3081
|
"phases": {
|
|
@@ -2211,6 +3105,7 @@ def self_test() -> int:
|
|
|
2211
3105
|
(devlyn / "pipeline.state.json").write_text(
|
|
2212
3106
|
json.dumps({
|
|
2213
3107
|
"version": "3.0",
|
|
3108
|
+
"engine": "claude",
|
|
2214
3109
|
"mode": "spec",
|
|
2215
3110
|
"pair_verify": True,
|
|
2216
3111
|
"phases": {
|
|
@@ -2231,6 +3126,7 @@ def self_test() -> int:
|
|
|
2231
3126
|
json.dumps({"id": "p-preknown", "severity": "LOW"}) + "\n",
|
|
2232
3127
|
encoding="utf-8",
|
|
2233
3128
|
)
|
|
3129
|
+
(devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
2234
3130
|
findings, source_verdicts = read_findings(devlyn)
|
|
2235
3131
|
summary = write_outputs(devlyn, findings, source_verdicts)
|
|
2236
3132
|
assert summary["verdict"] == "BLOCKED", summary
|
|
@@ -2250,12 +3146,14 @@ def self_test() -> int:
|
|
|
2250
3146
|
assert '"id":"p-preknown"' in (
|
|
2251
3147
|
devlyn / "verify-merged.findings.jsonl"
|
|
2252
3148
|
).read_text(encoding="utf-8"), findings
|
|
3149
|
+
(devlyn / "codex-judge.stdout").unlink()
|
|
2253
3150
|
|
|
2254
3151
|
# Self-test: archived-v2 sequential primary blocker remains legal.
|
|
2255
3152
|
(devlyn / "verify.pair.findings.jsonl").unlink()
|
|
2256
3153
|
(devlyn / "pipeline.state.json").write_text(
|
|
2257
3154
|
json.dumps({
|
|
2258
3155
|
"version": "2.0",
|
|
3156
|
+
"engine": "claude",
|
|
2259
3157
|
"mode": "spec",
|
|
2260
3158
|
"phases": {
|
|
2261
3159
|
"verify": {
|
|
@@ -2298,6 +3196,7 @@ def self_test() -> int:
|
|
|
2298
3196
|
|
|
2299
3197
|
(devlyn / "pipeline.state.json").write_text(
|
|
2300
3198
|
json.dumps({
|
|
3199
|
+
"engine": "claude",
|
|
2301
3200
|
"phases": {"verify": {"verdict": "PASS", "sub_verdicts": {}}},
|
|
2302
3201
|
"verify": {
|
|
2303
3202
|
"pair_trigger": {
|
|
@@ -2321,7 +3220,7 @@ def self_test() -> int:
|
|
|
2321
3220
|
# PASS stdout from a claude judge promotes pair_judge null -> PASS.
|
|
2322
3221
|
(devlyn / "verify.pair.findings.jsonl").unlink()
|
|
2323
3222
|
(devlyn / "pipeline.state.json").write_text(
|
|
2324
|
-
json.dumps({"phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
|
|
3223
|
+
json.dumps({"engine": "codex", "phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
|
|
2325
3224
|
encoding="utf-8",
|
|
2326
3225
|
)
|
|
2327
3226
|
(devlyn / "claude-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
@@ -2331,6 +3230,184 @@ def self_test() -> int:
|
|
|
2331
3230
|
state = loads_strict_json((devlyn / "pipeline.state.json").read_text(encoding="utf-8"))
|
|
2332
3231
|
assert summary["verdict"] == "PASS", summary
|
|
2333
3232
|
assert state["phases"]["verify"]["sub_verdicts"]["pair_judge"] == "PASS", state
|
|
3233
|
+
(devlyn / "claude-judge.stdout").unlink()
|
|
3234
|
+
|
|
3235
|
+
# The primary capture must not be misattributed to the OTHER-engine
|
|
3236
|
+
# pair seat merely because both adapters use *-judge.stdout names.
|
|
3237
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
3238
|
+
json.dumps({"engine": "claude", "phases": {"verify": {"verdict": None, "sub_verdicts": None}}}),
|
|
3239
|
+
encoding="utf-8",
|
|
3240
|
+
)
|
|
3241
|
+
(devlyn / "claude-judge.stdout").write_text("primary prose\n", encoding="utf-8")
|
|
3242
|
+
(devlyn / "codex-judge.stdout").write_text("PASS\n", encoding="utf-8")
|
|
3243
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3244
|
+
assert source_verdicts["pair_judge"] == "PASS", source_verdicts
|
|
3245
|
+
assert not any(finding["source"] == "pair_judge" for finding in findings), findings
|
|
3246
|
+
(devlyn / "codex-judge.stdout").write_text("pair prose\n", encoding="utf-8")
|
|
3247
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3248
|
+
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3249
|
+
assert any(finding["id"] == "verify-pair-emission-contract-violated" for finding in findings)
|
|
3250
|
+
(devlyn / "claude-judge.stdout").unlink()
|
|
3251
|
+
(devlyn / "codex-judge.stdout").unlink()
|
|
3252
|
+
|
|
3253
|
+
# A primary capture cannot become pair evidence when the state needed
|
|
3254
|
+
# to identify the primary seat is absent, incomplete, or malformed.
|
|
3255
|
+
(devlyn / "claude-judge.stdout").write_text("primary prose\n", encoding="utf-8")
|
|
3256
|
+
(devlyn / "pipeline.state.json").unlink()
|
|
3257
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3258
|
+
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3259
|
+
assert any(finding["id"] == "verify-state-missing" for finding in findings), findings
|
|
3260
|
+
assert not any(finding.get("file") == "claude-judge.stdout" for finding in findings), findings
|
|
3261
|
+
|
|
3262
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
3263
|
+
json.dumps({"version": "3.0", "phases": {"verify": {}}}), encoding="utf-8",
|
|
3264
|
+
)
|
|
3265
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3266
|
+
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3267
|
+
assert any(finding.get("rule_id") == "verify.state.engine-malformed" for finding in findings), findings
|
|
3268
|
+
|
|
3269
|
+
(devlyn / "pipeline.state.json").write_text("{", encoding="utf-8")
|
|
3270
|
+
findings, source_verdicts = read_findings(devlyn)
|
|
3271
|
+
assert source_verdicts["pair_judge"] == "BLOCKED", source_verdicts
|
|
3272
|
+
assert any(finding["id"] == "verify-pair-trigger-state-malformed" for finding in findings), findings
|
|
3273
|
+
(devlyn / "claude-judge.stdout").unlink()
|
|
3274
|
+
|
|
3275
|
+
# iter-0083: canonical summary verdict conservation.
|
|
3276
|
+
iter_0083_paths = (
|
|
3277
|
+
"pipeline.state.json",
|
|
3278
|
+
"verify-mechanical.findings.jsonl",
|
|
3279
|
+
"verify.findings.jsonl",
|
|
3280
|
+
"verify.pair.findings.jsonl",
|
|
3281
|
+
"verify.pair-judge.findings.jsonl",
|
|
3282
|
+
"pair-judge.summary.json",
|
|
3283
|
+
"codex-primary-judge.summary.json",
|
|
3284
|
+
"grok-judge.summary.json",
|
|
3285
|
+
"verify.pair.timeout.json",
|
|
3286
|
+
"codex-judge.stdout",
|
|
3287
|
+
"claude-judge.stdout",
|
|
3288
|
+
)
|
|
3289
|
+
|
|
3290
|
+
def iter_0083_reset() -> None:
|
|
3291
|
+
for name in iter_0083_paths:
|
|
3292
|
+
(devlyn / name).unlink(missing_ok=True)
|
|
3293
|
+
|
|
3294
|
+
def iter_0083_case(
|
|
3295
|
+
summary_payload: object | None,
|
|
3296
|
+
severity: str | None,
|
|
3297
|
+
*,
|
|
3298
|
+
carrier: str | None = "verify.pair.findings.jsonl",
|
|
3299
|
+
other_summary: str | None = None,
|
|
3300
|
+
timeout: bool = False,
|
|
3301
|
+
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
|
3302
|
+
iter_0083_reset()
|
|
3303
|
+
(devlyn / "pipeline.state.json").write_text(
|
|
3304
|
+
json.dumps({"engine": "claude", "phases": {"verify": {}}}), encoding="utf-8",
|
|
3305
|
+
)
|
|
3306
|
+
(devlyn / "verify-mechanical.findings.jsonl").write_text("", encoding="utf-8")
|
|
3307
|
+
(devlyn / "verify.findings.jsonl").write_text("", encoding="utf-8")
|
|
3308
|
+
if carrier is not None:
|
|
3309
|
+
content = (
|
|
3310
|
+
json.dumps({"id": "iter-0083", "severity": severity}) + "\n"
|
|
3311
|
+
if severity
|
|
3312
|
+
else ""
|
|
3313
|
+
)
|
|
3314
|
+
(devlyn / carrier).write_text(content, encoding="utf-8")
|
|
3315
|
+
if summary_payload is not None:
|
|
3316
|
+
content = (
|
|
3317
|
+
summary_payload
|
|
3318
|
+
if isinstance(summary_payload, str)
|
|
3319
|
+
else json.dumps(summary_payload)
|
|
3320
|
+
)
|
|
3321
|
+
(devlyn / "pair-judge.summary.json").write_text(content, encoding="utf-8")
|
|
3322
|
+
if other_summary is not None:
|
|
3323
|
+
(devlyn / other_summary).write_text(
|
|
3324
|
+
json.dumps({"verdict": "BLOCKED"}), encoding="utf-8"
|
|
3325
|
+
)
|
|
3326
|
+
if timeout:
|
|
3327
|
+
(devlyn / "verify.pair.timeout.json").write_text(
|
|
3328
|
+
json.dumps({"engine": "codex", "budget_seconds": 600}),
|
|
3329
|
+
encoding="utf-8",
|
|
3330
|
+
)
|
|
3331
|
+
case_findings, case_source_verdicts = read_findings(devlyn)
|
|
3332
|
+
return case_findings, write_outputs(devlyn, case_findings, case_source_verdicts)
|
|
3333
|
+
|
|
3334
|
+
for case_id, verdict, severity, expected in (
|
|
3335
|
+
("P1", "NEEDS_WORK", "INFO", "NEEDS_WORK"),
|
|
3336
|
+
("P2", "NEEDS_WORK", "LOW", "NEEDS_WORK"),
|
|
3337
|
+
("P3", "NEEDS_WORK", "MEDIUM", "NEEDS_WORK"),
|
|
3338
|
+
("P4", "PASS", "HIGH", "NEEDS_WORK"),
|
|
3339
|
+
("P5", "BLOCKED", "INFO", "BLOCKED"),
|
|
3340
|
+
("P6", "FAIL", "INFO", "NEEDS_WORK"),
|
|
3341
|
+
("P7", "PASS", None, "PASS"),
|
|
3342
|
+
):
|
|
3343
|
+
_, summary = iter_0083_case({"verdict": verdict}, severity)
|
|
3344
|
+
assert summary["source_verdicts"]["pair_judge"] == expected, case_id
|
|
3345
|
+
assert summary["verdict"] == expected, case_id
|
|
3346
|
+
|
|
3347
|
+
for case_id, severity, expected in (
|
|
3348
|
+
("P8-INFO", "INFO", "PASS"),
|
|
3349
|
+
("P8-LOW", "LOW", "PASS_WITH_ISSUES"),
|
|
3350
|
+
):
|
|
3351
|
+
_, summary = iter_0083_case(None, severity)
|
|
3352
|
+
assert summary["source_verdicts"]["pair_judge"] == expected, case_id
|
|
3353
|
+
assert summary["verdict"] == expected, case_id
|
|
3354
|
+
|
|
3355
|
+
def assert_iter_0083_blocked(
|
|
3356
|
+
case_id: str,
|
|
3357
|
+
case_findings: list[dict[str, Any]],
|
|
3358
|
+
summary: dict[str, Any],
|
|
3359
|
+
) -> None:
|
|
3360
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", case_id
|
|
3361
|
+
assert summary["verdict"] == "BLOCKED", case_id
|
|
3362
|
+
assert any(
|
|
3363
|
+
finding.get("source") == "pair_judge"
|
|
3364
|
+
and finding.get("severity") == "CRITICAL"
|
|
3365
|
+
and finding.get("file") == "pair-judge.summary.json"
|
|
3366
|
+
for finding in case_findings
|
|
3367
|
+
), case_id
|
|
3368
|
+
|
|
3369
|
+
for case_id, payload in (
|
|
3370
|
+
("N1-malformed", "{"),
|
|
3371
|
+
("N1-non-object", []),
|
|
3372
|
+
("N1-unknown", {"verdict": "UNKNOWN"}),
|
|
3373
|
+
("N2", {}),
|
|
3374
|
+
("N7", {"verdict": "TIMEOUT"}),
|
|
3375
|
+
):
|
|
3376
|
+
case_findings, summary = iter_0083_case(payload, "INFO")
|
|
3377
|
+
assert_iter_0083_blocked(case_id, case_findings, summary)
|
|
3378
|
+
|
|
3379
|
+
_, summary = iter_0083_case({"verdict": "BLOCKED"}, None, carrier=None)
|
|
3380
|
+
assert summary["source_verdicts"]["pair_judge"] is None, summary
|
|
3381
|
+
assert summary["verdict"] == "PASS", summary
|
|
3382
|
+
|
|
3383
|
+
_, summary = iter_0083_case(
|
|
3384
|
+
{"verdict": "PASS"},
|
|
3385
|
+
"INFO",
|
|
3386
|
+
other_summary="codex-primary-judge.summary.json",
|
|
3387
|
+
)
|
|
3388
|
+
assert summary["source_verdicts"]["pair_judge"] == "PASS", "N4"
|
|
3389
|
+
assert summary["verdict"] == "PASS", "N4"
|
|
3390
|
+
|
|
3391
|
+
_, summary = iter_0083_case(
|
|
3392
|
+
None,
|
|
3393
|
+
"INFO",
|
|
3394
|
+
other_summary="grok-judge.summary.json",
|
|
3395
|
+
)
|
|
3396
|
+
assert summary["source_verdicts"]["pair_judge"] == "PASS", "N5"
|
|
3397
|
+
assert summary["verdict"] == "PASS", "N5"
|
|
3398
|
+
|
|
3399
|
+
_, summary = iter_0083_case(
|
|
3400
|
+
{"verdict": "NEEDS_WORK"},
|
|
3401
|
+
"INFO",
|
|
3402
|
+
carrier="verify.pair-judge.findings.jsonl",
|
|
3403
|
+
)
|
|
3404
|
+
assert summary["source_verdicts"]["pair_judge"] == "NEEDS_WORK", "N6"
|
|
3405
|
+
assert summary["verdict"] == "NEEDS_WORK", "N6"
|
|
3406
|
+
|
|
3407
|
+
case_findings, summary = iter_0083_case({"verdict": "BLOCKED"}, None, timeout=True)
|
|
3408
|
+
assert case_findings == [], "VERIFY-JUDGE-001"
|
|
3409
|
+
assert summary["source_verdicts"]["pair_judge"] == "BLOCKED", "VERIFY-JUDGE-001"
|
|
3410
|
+
assert summary["verdict"] == "BLOCKED", "VERIFY-JUDGE-001"
|
|
2334
3411
|
return 0
|
|
2335
3412
|
|
|
2336
3413
|
|