devlyn-cli 2.10.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -9
- package/CLAUDE.md +11 -15
- package/README.md +31 -7
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +25 -0
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +10 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +30 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +32 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +44 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +16 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +15 -0
- package/bin/devlyn.js +6 -1
- package/config/skills/_shared/.ruff.toml +2 -0
- package/config/skills/_shared/adapters/claude.md +9 -1
- package/config/skills/_shared/adapters/codex.md +5 -0
- package/config/skills/_shared/adapters/grok.md +4 -2
- package/config/skills/_shared/archive_run.py +738 -56
- package/config/skills/_shared/codex-config.md +19 -4
- package/config/skills/_shared/codex-monitored.sh +81 -12
- package/config/skills/_shared/collect-codex-findings.py +311 -128
- package/config/skills/_shared/engine-preflight.md +8 -7
- package/config/skills/_shared/expected.schema.json +61 -0
- package/config/skills/_shared/finish-gate.py +12 -11
- package/config/skills/_shared/grok-anchor-guard.py +18 -1
- package/config/skills/_shared/invocation-receipt.py +1041 -0
- package/config/skills/_shared/judge-output-parser.py +200 -0
- package/config/skills/_shared/judge-role-evidence.py +273 -0
- package/config/skills/_shared/process-evidence.py +909 -0
- package/config/skills/_shared/resolve-bootstrap.py +776 -84
- package/config/skills/_shared/resolve-stop-hook.py +33 -1
- package/config/skills/_shared/role-config.py +552 -0
- package/config/skills/_shared/run-bounded.py +1 -1
- package/config/skills/_shared/spec-verify-check.py +749 -181
- package/config/skills/_shared/state-phase-write.py +1838 -58
- package/config/skills/_shared/terminal-claim-check.py +15 -0
- package/config/skills/_shared/verify-merge-findings.py +958 -51
- package/config/skills/devlyn:engines/SKILL.md +8 -16
- package/config/skills/devlyn:ideate/references/spec-template.md +1 -0
- package/config/skills/devlyn:queue/SKILL.md +7 -4
- package/config/skills/devlyn:resolve/SKILL.md +91 -33
- package/config/skills/devlyn:resolve/references/outer-loop.md +15 -0
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +31 -2
- package/config/skills/devlyn:resolve/references/phases/implement.md +3 -2
- package/config/skills/devlyn:resolve/references/phases/verify.md +120 -84
- package/config/skills/devlyn:resolve/references/state-schema.md +22 -8
- package/package.json +5 -1
- package/scripts/lint-skills.sh +314 -285
|
@@ -31,10 +31,11 @@ Default mode (BUILD_GATE invocation, no args):
|
|
|
31
31
|
mis-config). Real-user mode silent no-op + drops any stale
|
|
32
32
|
pre-staged file (preserves iter-0019.6 backward compat for
|
|
33
33
|
handwritten specs without the carrier).
|
|
34
|
-
- For each verification_commands entry
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
- For each verification_commands entry and included risk probe, routes the
|
|
35
|
+
command through the sibling `process-evidence.py` runner. Raw stdout/stderr,
|
|
36
|
+
execution outcome, classification, and byte digests are written under the
|
|
37
|
+
authenticated run/phase/round manifest; expectations retain
|
|
38
|
+
run-fixture.sh's combined-stream matching semantics.
|
|
38
39
|
|
|
39
40
|
Check mode (`--check <markdown_path>`):
|
|
40
41
|
- Used by /devlyn:ideate after writing each item spec to validate that the
|
|
@@ -60,6 +61,8 @@ Output routing:
|
|
|
60
61
|
`SPEC_VERIFY_FINDINGS_FILE=verify-mechanical.findings.jsonl`, and
|
|
61
62
|
`SPEC_VERIFY_FINDING_PREFIX=VERIFY-MECH` so `verify-merge-findings.py` consumes
|
|
62
63
|
deterministic blockers directly.
|
|
64
|
+
- `.devlyn/spec-verify.results.json` points each result at its sealed raw
|
|
65
|
+
streams and includes the validated process-evidence carrier.
|
|
63
66
|
|
|
64
67
|
Why: iter-0018.5's prompt-only contract enforcement was empirically dead
|
|
65
68
|
(F9 verify=0.4 across all engines in iter-0019). Same lesson as iter-0008
|
|
@@ -80,11 +83,13 @@ Exit codes:
|
|
|
80
83
|
`SPEC_VERIFY_FINDINGS_FILE` (for example, VERIFY uses
|
|
81
84
|
`.devlyn/verify-mechanical.findings.jsonl`).
|
|
82
85
|
- 2: invocation error (unreadable spec-verify.json, missing markdown in
|
|
83
|
-
--check mode, etc.)
|
|
86
|
+
--check mode, etc.) or explicit evidence-backed
|
|
87
|
+
`BLOCKED:build-env-underprovisioned` capability denial.
|
|
84
88
|
"""
|
|
85
89
|
|
|
86
90
|
from __future__ import annotations
|
|
87
91
|
|
|
92
|
+
import importlib.util
|
|
88
93
|
import json
|
|
89
94
|
import hashlib
|
|
90
95
|
import os
|
|
@@ -99,8 +104,21 @@ def reject_json_constant(token: str) -> None:
|
|
|
99
104
|
raise ValueError(f"invalid JSON numeric constant: {token}")
|
|
100
105
|
|
|
101
106
|
|
|
107
|
+
def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict:
|
|
108
|
+
result = {}
|
|
109
|
+
for key, value in pairs:
|
|
110
|
+
if key in result:
|
|
111
|
+
raise ValueError(f"duplicate JSON key: {key}")
|
|
112
|
+
result[key] = value
|
|
113
|
+
return result
|
|
114
|
+
|
|
115
|
+
|
|
102
116
|
def loads_strict_json(text: str):
|
|
103
|
-
return json.loads(
|
|
117
|
+
return json.loads(
|
|
118
|
+
text,
|
|
119
|
+
parse_constant=reject_json_constant,
|
|
120
|
+
object_pairs_hook=reject_duplicate_keys,
|
|
121
|
+
)
|
|
104
122
|
|
|
105
123
|
|
|
106
124
|
def output_phase() -> str:
|
|
@@ -115,6 +133,95 @@ def output_finding_prefix() -> str:
|
|
|
115
133
|
return os.environ.get("SPEC_VERIFY_FINDING_PREFIX", "BGATE")
|
|
116
134
|
|
|
117
135
|
|
|
136
|
+
_PROCESS_EVIDENCE_MODULE = None
|
|
137
|
+
RUNNER_ENV_KEYS = (
|
|
138
|
+
"SPEC_VERIFY_PHASE",
|
|
139
|
+
"SPEC_VERIFY_FINDINGS_FILE",
|
|
140
|
+
"SPEC_VERIFY_FINDING_PREFIX",
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def process_evidence_module():
|
|
145
|
+
global _PROCESS_EVIDENCE_MODULE
|
|
146
|
+
if _PROCESS_EVIDENCE_MODULE is None:
|
|
147
|
+
module_path = Path(__file__).with_name("process-evidence.py")
|
|
148
|
+
spec = importlib.util.spec_from_file_location("devlyn_process_evidence", module_path)
|
|
149
|
+
if spec is None or spec.loader is None:
|
|
150
|
+
raise RuntimeError(f"cannot load process evidence runner: {module_path}")
|
|
151
|
+
module = importlib.util.module_from_spec(spec)
|
|
152
|
+
previous_bytecode_setting = sys.dont_write_bytecode
|
|
153
|
+
sys.dont_write_bytecode = True
|
|
154
|
+
try:
|
|
155
|
+
spec.loader.exec_module(module)
|
|
156
|
+
finally:
|
|
157
|
+
sys.dont_write_bytecode = previous_bytecode_setting
|
|
158
|
+
_PROCESS_EVIDENCE_MODULE = module
|
|
159
|
+
return _PROCESS_EVIDENCE_MODULE
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def mechanical_evidence_identity(state: dict, runner) -> tuple[str, str, int, str]:
|
|
163
|
+
phase = "verify" if output_phase() == "verify_mechanical" else "build_gate"
|
|
164
|
+
run_id = state.get("run_id")
|
|
165
|
+
phase_state = (state.get("phases") or {}).get(phase)
|
|
166
|
+
round_ = phase_state.get("round") if isinstance(phase_state, dict) else None
|
|
167
|
+
invalid_identity = (
|
|
168
|
+
not isinstance(run_id, str)
|
|
169
|
+
or runner.ID_RE.fullmatch(run_id) is None
|
|
170
|
+
or isinstance(round_, bool)
|
|
171
|
+
or not isinstance(round_, int)
|
|
172
|
+
or round_ < 0
|
|
173
|
+
)
|
|
174
|
+
if invalid_identity and state.get("version") == "3.0":
|
|
175
|
+
raise runner.EvidenceError(
|
|
176
|
+
f"schema-v3 MECHANICAL evidence identity is invalid for phase {phase}"
|
|
177
|
+
)
|
|
178
|
+
if invalid_identity:
|
|
179
|
+
# Standalone/benchmark invocations predate pipeline run identity. Keep
|
|
180
|
+
# them evidence-backed without letting one process reuse another's
|
|
181
|
+
# manifest. Real pipeline runs always take the authenticated branch.
|
|
182
|
+
run_id = f"standalone-{os.getpid()}"
|
|
183
|
+
round_ = 0
|
|
184
|
+
identity = {"run_id": run_id, "phases": {phase: {"round": round_}}}
|
|
185
|
+
return phase, run_id, round_, runner.manifest_relative_path(identity, phase)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def mechanical_obligation(vc: dict, idx: int, phase: str) -> dict:
|
|
189
|
+
evidence_id = (
|
|
190
|
+
f"risk-probe-{idx + 1:04d}"
|
|
191
|
+
if vc.get("_risk_probe")
|
|
192
|
+
else f"verification-command-{idx + 1:04d}"
|
|
193
|
+
)
|
|
194
|
+
return {
|
|
195
|
+
"id": evidence_id,
|
|
196
|
+
"phase": phase,
|
|
197
|
+
"cmd": vc["cmd"],
|
|
198
|
+
"exit_code": vc.get("exit_code", 0),
|
|
199
|
+
"timeout_sec": verification_timeout_sec(vc),
|
|
200
|
+
"stdout_contains": vc.get("stdout_contains", []) or [],
|
|
201
|
+
"stdout_not_contains": vc.get("stdout_not_contains", []) or [],
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def capture_mechanical_command(
|
|
206
|
+
runner, work: Path, manifest_path: Path, run_id: str, phase: str,
|
|
207
|
+
round_: int, obligation: dict,
|
|
208
|
+
) -> dict:
|
|
209
|
+
saved = {key: os.environ.get(key) for key in (*RUNNER_ENV_KEYS, "BENCH_WORKDIR")}
|
|
210
|
+
for key in RUNNER_ENV_KEYS:
|
|
211
|
+
os.environ.pop(key, None)
|
|
212
|
+
os.environ["BENCH_WORKDIR"] = str(work)
|
|
213
|
+
try:
|
|
214
|
+
return runner.capture_process(
|
|
215
|
+
work, manifest_path, run_id, phase, round_, obligation,
|
|
216
|
+
)
|
|
217
|
+
finally:
|
|
218
|
+
for key, value in saved.items():
|
|
219
|
+
if value is None:
|
|
220
|
+
os.environ.pop(key, None)
|
|
221
|
+
else:
|
|
222
|
+
os.environ[key] = value
|
|
223
|
+
|
|
224
|
+
|
|
118
225
|
VERIFICATION_SECTION_RE = re.compile(
|
|
119
226
|
r'(?ms)^<!--[ \t]*devlyn:verification[ \t]*-->[ \t]*\n(#{1,6}[ \t]+[^\n]*\n.*?)(?=^#{1,6}[ \t]+|\Z)'
|
|
120
227
|
)
|
|
@@ -127,8 +234,9 @@ FORBIDDEN_RISK_PROBE_CMD_RE = re.compile(
|
|
|
127
234
|
)
|
|
128
235
|
EXTERNAL_URL_RE = re.compile(r"https?://([^/\s\"']+)", re.IGNORECASE)
|
|
129
236
|
RISK_PROBE_SCRIPT_RE = re.compile(
|
|
130
|
-
r'(?<![\w./-])(\.devlyn/probes/[A-Za-z0-9][A-Za-z0-9._/-]*)'
|
|
237
|
+
r'(?<![\w./-])(?:\./)?(\.devlyn/probes/[A-Za-z0-9][A-Za-z0-9._/-]*)'
|
|
131
238
|
)
|
|
239
|
+
RISK_PROBE_RAW_REF = ".devlyn/probes/"
|
|
132
240
|
RISK_PROBE_INTEGRITY_FIX_HINT = (
|
|
133
241
|
"Probe artifacts changed after PHASE 1.5. Only the orchestrator may regenerate probes: "
|
|
134
242
|
"re-run probe validation and re-write state.risk_probes_digest; workers must never modify "
|
|
@@ -480,7 +588,28 @@ def referenced_risk_probe_scripts(cmd: str) -> list[str]:
|
|
|
480
588
|
return scripts
|
|
481
589
|
|
|
482
590
|
|
|
591
|
+
def unrecognized_risk_probe_reference(cmd: str) -> str | None:
|
|
592
|
+
command = cmd or ""
|
|
593
|
+
matches = list(RISK_PROBE_SCRIPT_RE.finditer(command))
|
|
594
|
+
offset = command.find(RISK_PROBE_RAW_REF)
|
|
595
|
+
while offset >= 0:
|
|
596
|
+
if not any(
|
|
597
|
+
match.start() <= offset and offset + len(RISK_PROBE_RAW_REF) <= match.end()
|
|
598
|
+
for match in matches
|
|
599
|
+
):
|
|
600
|
+
tail = command[offset:].split(maxsplit=1)[0][:80]
|
|
601
|
+
return (
|
|
602
|
+
f"unrecognized probe script reference {tail!r}; write it as "
|
|
603
|
+
".devlyn/probes/<file> (a ./ prefix is the only alias)"
|
|
604
|
+
)
|
|
605
|
+
offset = command.find(RISK_PROBE_RAW_REF, offset + len(RISK_PROBE_RAW_REF))
|
|
606
|
+
return None
|
|
607
|
+
|
|
608
|
+
|
|
483
609
|
def validate_risk_probe_scripts(cmd: str, index: int, work: Path) -> str | None:
|
|
610
|
+
unrecognized = unrecognized_risk_probe_reference(cmd)
|
|
611
|
+
if unrecognized:
|
|
612
|
+
return f"risk-probes[{index}].cmd has {unrecognized}"
|
|
484
613
|
for rel_path in referenced_risk_probe_scripts(cmd):
|
|
485
614
|
path = Path(rel_path)
|
|
486
615
|
if ".." in path.parts:
|
|
@@ -540,6 +669,9 @@ def risk_probes_digest(devlyn_dir: Path) -> tuple[str | None, str | None]:
|
|
|
540
669
|
except ValueError as e:
|
|
541
670
|
return (None, f"risk-probes[{index}] invalid JSON: {e}")
|
|
542
671
|
cmd = probe.get("cmd") if isinstance(probe, dict) else ""
|
|
672
|
+
unrecognized = unrecognized_risk_probe_reference(cmd)
|
|
673
|
+
if unrecognized:
|
|
674
|
+
return (None, f"risk-probes[{index}].cmd has {unrecognized}")
|
|
543
675
|
scripts.update(referenced_risk_probe_scripts(cmd))
|
|
544
676
|
|
|
545
677
|
digest = hashlib.sha256()
|
|
@@ -1115,7 +1247,13 @@ def stage_from_expected(
|
|
|
1115
1247
|
return (True, True, None, expected_path, data)
|
|
1116
1248
|
|
|
1117
1249
|
|
|
1118
|
-
def write_malformed_finding(
|
|
1250
|
+
def write_malformed_finding(
|
|
1251
|
+
devlyn_dir: Path,
|
|
1252
|
+
error: str,
|
|
1253
|
+
source_path: Path | None,
|
|
1254
|
+
*,
|
|
1255
|
+
fix_hint: str | None = None,
|
|
1256
|
+
) -> None:
|
|
1119
1257
|
"""Emit a single CRITICAL finding for a malformed verification carrier."""
|
|
1120
1258
|
devlyn_dir.mkdir(parents=True, exist_ok=True)
|
|
1121
1259
|
findings_path = devlyn_dir / output_findings_name()
|
|
@@ -1131,7 +1269,7 @@ def write_malformed_finding(devlyn_dir: Path, error: str, source_path: Path | No
|
|
|
1131
1269
|
"line": 1,
|
|
1132
1270
|
"phase": output_phase(),
|
|
1133
1271
|
"criterion_ref": "spec-verify://carrier",
|
|
1134
|
-
"fix_hint": (
|
|
1272
|
+
"fix_hint": fix_hint if fix_hint is not None else (
|
|
1135
1273
|
"Fix the sibling `spec.expected.json` file or the `## Verification` "
|
|
1136
1274
|
"```json``` block: a JSON object with a non-empty `verification_commands` array of "
|
|
1137
1275
|
"{cmd, exit_code?, stdout_contains?, stdout_not_contains?} "
|
|
@@ -1360,6 +1498,24 @@ def expected_contract_findings(
|
|
|
1360
1498
|
return (findings, seq)
|
|
1361
1499
|
|
|
1362
1500
|
|
|
1501
|
+
def validate_surface_brace_glob(entry: str) -> str | None:
|
|
1502
|
+
if "{" not in entry and "}" not in entry:
|
|
1503
|
+
return None
|
|
1504
|
+
brace = re.search(r"\{([^{}]*)\}", entry)
|
|
1505
|
+
alternatives = brace.group(1).split(",") if brace else []
|
|
1506
|
+
if (
|
|
1507
|
+
entry.count("{") != 1
|
|
1508
|
+
or entry.count("}") != 1
|
|
1509
|
+
or len(alternatives) < 2
|
|
1510
|
+
or any(not value or any(char in value for char in "{},/*") for value in alternatives)
|
|
1511
|
+
):
|
|
1512
|
+
return (
|
|
1513
|
+
f"unsupported brace glob {entry!r}; supported form is {{alt1,alt2,...}} "
|
|
1514
|
+
"with at least two non-empty plain alternatives"
|
|
1515
|
+
)
|
|
1516
|
+
return None
|
|
1517
|
+
|
|
1518
|
+
|
|
1363
1519
|
def validate_authorized_surface_shape(data: object) -> str | None:
|
|
1364
1520
|
if not isinstance(data, dict):
|
|
1365
1521
|
return "top-level must be a JSON object"
|
|
@@ -1384,17 +1540,34 @@ def validate_authorized_surface_shape(data: object) -> str | None:
|
|
|
1384
1540
|
return f"authorized_surface directory grant needs a non-empty prefix before '/**': {entry!r}"
|
|
1385
1541
|
if ".." in stem.split("/"):
|
|
1386
1542
|
return f"authorized_surface entry must not contain '..': {entry!r}"
|
|
1543
|
+
brace_error = validate_surface_brace_glob(entry)
|
|
1544
|
+
if brace_error is not None:
|
|
1545
|
+
return brace_error
|
|
1387
1546
|
return None
|
|
1388
1547
|
|
|
1389
1548
|
|
|
1390
1549
|
def path_matches_surface(path: str, surface: list[str]) -> bool:
|
|
1391
1550
|
for entry in surface:
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1551
|
+
brace_error = validate_surface_brace_glob(entry)
|
|
1552
|
+
if brace_error is not None:
|
|
1553
|
+
raise ValueError(brace_error)
|
|
1554
|
+
if "{" in entry:
|
|
1555
|
+
brace = re.search(r"\{([^{}]*)\}", entry)
|
|
1556
|
+
assert brace is not None
|
|
1557
|
+
alternatives = brace.group(1).split(",")
|
|
1558
|
+
entries = tuple(
|
|
1559
|
+
entry[:brace.start()] + value + entry[brace.end():]
|
|
1560
|
+
for value in alternatives
|
|
1561
|
+
)
|
|
1562
|
+
else:
|
|
1563
|
+
entries = (entry,)
|
|
1564
|
+
for expanded in entries:
|
|
1565
|
+
if expanded.endswith("/**"):
|
|
1566
|
+
prefix = expanded[:-3].rstrip("/")
|
|
1567
|
+
if path == prefix or path.startswith(f"{prefix}/"):
|
|
1568
|
+
return True
|
|
1569
|
+
elif path == expanded:
|
|
1395
1570
|
return True
|
|
1396
|
-
elif path == entry:
|
|
1397
|
-
return True
|
|
1398
1571
|
return False
|
|
1399
1572
|
|
|
1400
1573
|
|
|
@@ -1753,6 +1926,12 @@ def run_check_expected_mode(expected_path: Path) -> int:
|
|
|
1753
1926
|
|
|
1754
1927
|
|
|
1755
1928
|
def run_self_test() -> int:
|
|
1929
|
+
try:
|
|
1930
|
+
loads_strict_json('{"verification_commands":[],"verification_commands":[{}]}')
|
|
1931
|
+
except ValueError as exc:
|
|
1932
|
+
assert "duplicate JSON key" in str(exc)
|
|
1933
|
+
else:
|
|
1934
|
+
raise AssertionError("duplicate expected-contract key was accepted")
|
|
1756
1935
|
# Hermeticity: a pipeline replay exports BENCH_WORKDIR at the live repo;
|
|
1757
1936
|
# inherited into scenario children it wins over their tmp cwd (the
|
|
1758
1937
|
# default-mode work resolution below) and re-executes the LIVE
|
|
@@ -1761,10 +1940,53 @@ def run_self_test() -> int:
|
|
|
1761
1940
|
# (observed 2026-08-03, iter-0089 canary).
|
|
1762
1941
|
os.environ.pop("BENCH_WORKDIR", None)
|
|
1763
1942
|
script_path = str(Path(__file__).resolve())
|
|
1943
|
+
with tempfile.TemporaryDirectory() as help_tmp:
|
|
1944
|
+
for help_flag in ("--help", "-h"):
|
|
1945
|
+
help_result = subprocess.run(
|
|
1946
|
+
[sys.executable, script_path, help_flag],
|
|
1947
|
+
cwd=help_tmp,
|
|
1948
|
+
capture_output=True,
|
|
1949
|
+
text=True,
|
|
1950
|
+
)
|
|
1951
|
+
if help_result.returncode != 0 or "usage:" not in help_result.stdout:
|
|
1952
|
+
print(f"{help_flag} did not return usage successfully", file=sys.stderr)
|
|
1953
|
+
return 1
|
|
1954
|
+
if (Path(help_tmp) / ".devlyn").exists():
|
|
1955
|
+
print(f"{help_flag} mutated .devlyn", file=sys.stderr)
|
|
1956
|
+
return 1
|
|
1957
|
+
unknown_result = subprocess.run(
|
|
1958
|
+
[sys.executable, script_path, "--not-a-real-option"],
|
|
1959
|
+
cwd=help_tmp,
|
|
1960
|
+
capture_output=True,
|
|
1961
|
+
text=True,
|
|
1962
|
+
)
|
|
1963
|
+
if unknown_result.returncode != 2 or "unknown argument" not in unknown_result.stderr:
|
|
1964
|
+
print("unknown option did not fail closed", file=sys.stderr)
|
|
1965
|
+
return 1
|
|
1966
|
+
if (Path(help_tmp) / ".devlyn").exists():
|
|
1967
|
+
print("unknown option mutated .devlyn", file=sys.stderr)
|
|
1968
|
+
return 1
|
|
1969
|
+
runner = process_evidence_module()
|
|
1970
|
+
try:
|
|
1971
|
+
mechanical_evidence_identity({"version": "3.0", "phases": {}}, runner)
|
|
1972
|
+
except runner.EvidenceError as exc:
|
|
1973
|
+
assert "schema-v3 MECHANICAL evidence identity is invalid" in str(exc)
|
|
1974
|
+
else:
|
|
1975
|
+
raise AssertionError("schema-v3 MECHANICAL evidence accepted missing run identity")
|
|
1764
1976
|
with tempfile.TemporaryDirectory() as td:
|
|
1765
1977
|
work = Path(td)
|
|
1766
1978
|
devlyn = work / ".devlyn"
|
|
1767
1979
|
devlyn.mkdir()
|
|
1980
|
+
assert path_matches_surface("src/a/example.py", ["src/{a,b}/**"])
|
|
1981
|
+
assert path_matches_surface("src/b/example.py", ["src/{a,b}/**"])
|
|
1982
|
+
assert not path_matches_surface("src/c/example.py", ["src/{a,b}/**"])
|
|
1983
|
+
for entry in ("src/{a,b", "src/{a,}/**", "src/{a,{b,c}}", "src/{a,**}"):
|
|
1984
|
+
try:
|
|
1985
|
+
path_matches_surface("src/a/example.py", [entry])
|
|
1986
|
+
except ValueError as exc:
|
|
1987
|
+
assert entry in str(exc) and "supported form" in str(exc)
|
|
1988
|
+
else:
|
|
1989
|
+
raise AssertionError(f"malformed brace glob accepted: {entry}")
|
|
1768
1990
|
spec_md = work / "spec.md"
|
|
1769
1991
|
spec_md.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- probe must pass visible marker.\n")
|
|
1770
1992
|
(devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
@@ -1915,13 +2137,18 @@ def run_self_test() -> int:
|
|
|
1915
2137
|
"cmd": "python3 -c \"import time; time.sleep(1)\"",
|
|
1916
2138
|
"timeout_sec": 5,
|
|
1917
2139
|
},
|
|
2140
|
+
{
|
|
2141
|
+
"cmd": "printf 'Operation not permitted' >&2; exit 1",
|
|
2142
|
+
},
|
|
1918
2143
|
]
|
|
1919
2144
|
})
|
|
1920
2145
|
+ "\n```\n",
|
|
1921
2146
|
encoding="utf-8",
|
|
1922
2147
|
)
|
|
1923
2148
|
(timeout_run_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
1924
|
-
"
|
|
2149
|
+
"run_id": "rs-timeout-run",
|
|
2150
|
+
"source": {"type": "spec", "spec_path": str(timeout_run_spec)},
|
|
2151
|
+
"phases": {"build_gate": {"round": 2}},
|
|
1925
2152
|
}))
|
|
1926
2153
|
timeout_run = subprocess.run(
|
|
1927
2154
|
[sys.executable, script_path],
|
|
@@ -1932,9 +2159,10 @@ def run_self_test() -> int:
|
|
|
1932
2159
|
if timeout_run.returncode == 0:
|
|
1933
2160
|
print("declared one-second timeout did not fail", file=sys.stderr)
|
|
1934
2161
|
return 1
|
|
1935
|
-
|
|
2162
|
+
timeout_document = loads_strict_json(
|
|
1936
2163
|
(timeout_run_devlyn / "spec-verify.results.json").read_text()
|
|
1937
|
-
)
|
|
2164
|
+
)
|
|
2165
|
+
timeout_results = timeout_document["commands"]
|
|
1938
2166
|
timeout_findings = [
|
|
1939
2167
|
loads_strict_json(line)
|
|
1940
2168
|
for line in (timeout_run_devlyn / output_findings_name()).read_text().splitlines()
|
|
@@ -1944,10 +2172,37 @@ def run_self_test() -> int:
|
|
|
1944
2172
|
timeout_results[0].get("reason") != "timeout"
|
|
1945
2173
|
or timeout_results[0].get("timeout_sec") != 1
|
|
1946
2174
|
or not timeout_results[1].get("pass")
|
|
2175
|
+
or timeout_results[2].get("reason") != "exit"
|
|
2176
|
+
or timeout_results[2].get("classification") != {
|
|
2177
|
+
"kind": "product_result", "operation": None,
|
|
2178
|
+
}
|
|
1947
2179
|
):
|
|
1948
2180
|
print("declared timeout budgets were not honored", file=sys.stderr)
|
|
1949
2181
|
print(timeout_results, file=sys.stderr)
|
|
1950
2182
|
return 1
|
|
2183
|
+
timeout_carrier = timeout_document.get("process_evidence") or {}
|
|
2184
|
+
timeout_manifest_path = (
|
|
2185
|
+
".devlyn/process-evidence/rs-timeout-run/build_gate/round-2/manifest.json"
|
|
2186
|
+
)
|
|
2187
|
+
timeout_manifest = loads_strict_json(
|
|
2188
|
+
(timeout_run_root / timeout_manifest_path).read_text()
|
|
2189
|
+
)
|
|
2190
|
+
if (
|
|
2191
|
+
timeout_carrier.get("manifest", {}).get("path") != timeout_manifest_path
|
|
2192
|
+
or [entry.get("id") for entry in timeout_manifest.get("entries", [])] != [
|
|
2193
|
+
"verification-command-0001",
|
|
2194
|
+
"verification-command-0002",
|
|
2195
|
+
"verification-command-0003",
|
|
2196
|
+
]
|
|
2197
|
+
or any(
|
|
2198
|
+
not (timeout_run_root / entry[stream]["path"]).is_file()
|
|
2199
|
+
for entry in timeout_manifest.get("entries", [])
|
|
2200
|
+
for stream in ("stdout", "stderr")
|
|
2201
|
+
)
|
|
2202
|
+
):
|
|
2203
|
+
print("literal commands did not emit sealed BUILD_GATE evidence", file=sys.stderr)
|
|
2204
|
+
print(timeout_document, file=sys.stderr)
|
|
2205
|
+
return 1
|
|
1951
2206
|
timeout_finding = timeout_findings[0] if timeout_findings else {}
|
|
1952
2207
|
if (
|
|
1953
2208
|
timeout_finding.get("rule_id") != "correctness.verification-timeout"
|
|
@@ -1958,6 +2213,17 @@ def run_self_test() -> int:
|
|
|
1958
2213
|
print("timeout finding did not carry the distinct budget contract", file=sys.stderr)
|
|
1959
2214
|
print(timeout_finding, file=sys.stderr)
|
|
1960
2215
|
return 1
|
|
2216
|
+
if (
|
|
2217
|
+
not any(
|
|
2218
|
+
finding.get("rule_id") == "correctness.spec-literal-mismatch"
|
|
2219
|
+
for finding in timeout_findings
|
|
2220
|
+
)
|
|
2221
|
+
or "BLOCKED:build-env-underprovisioned" in timeout_run.stderr
|
|
2222
|
+
):
|
|
2223
|
+
print("ordinary restricted-looking stderr did not remain a product finding", file=sys.stderr)
|
|
2224
|
+
print(timeout_findings, file=sys.stderr)
|
|
2225
|
+
print(timeout_run.stderr, file=sys.stderr)
|
|
2226
|
+
return 1
|
|
1961
2227
|
|
|
1962
2228
|
runner_env_root = work / "runner-env-isolation"
|
|
1963
2229
|
runner_env_root.mkdir()
|
|
@@ -1980,7 +2246,9 @@ def run_self_test() -> int:
|
|
|
1980
2246
|
encoding="utf-8",
|
|
1981
2247
|
)
|
|
1982
2248
|
(runner_env_devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
1983
|
-
"
|
|
2249
|
+
"run_id": "rs-runner-env",
|
|
2250
|
+
"source": {"type": "spec", "spec_path": str(runner_env_spec)},
|
|
2251
|
+
"phases": {"verify": {"round": 3}},
|
|
1984
2252
|
}))
|
|
1985
2253
|
runner_env = os.environ.copy()
|
|
1986
2254
|
runner_env.update({
|
|
@@ -1995,13 +2263,20 @@ def run_self_test() -> int:
|
|
|
1995
2263
|
capture_output=True,
|
|
1996
2264
|
text=True,
|
|
1997
2265
|
)
|
|
1998
|
-
|
|
2266
|
+
runner_env_document = loads_strict_json(
|
|
1999
2267
|
(runner_env_devlyn / "spec-verify.results.json").read_text()
|
|
2000
|
-
)
|
|
2268
|
+
)
|
|
2269
|
+
runner_env_results = runner_env_document["commands"]
|
|
2001
2270
|
if runner_env_run.returncode != 0 or not runner_env_results[0].get("pass"):
|
|
2002
2271
|
print("runner-directed environment leaked into a verification command", file=sys.stderr)
|
|
2003
2272
|
print(runner_env_run.stderr, file=sys.stderr)
|
|
2004
2273
|
return 1
|
|
2274
|
+
if runner_env_document.get("process_evidence", {}).get("manifest", {}).get("path") != (
|
|
2275
|
+
".devlyn/process-evidence/rs-runner-env/verify/round-3/manifest.json"
|
|
2276
|
+
):
|
|
2277
|
+
print("VERIFY MECHANICAL evidence used the wrong phase identity", file=sys.stderr)
|
|
2278
|
+
print(runner_env_document, file=sys.stderr)
|
|
2279
|
+
return 1
|
|
2005
2280
|
env = os.environ.copy()
|
|
2006
2281
|
env["BENCH_WORKDIR"] = str(work)
|
|
2007
2282
|
validate_without_digest = subprocess.run(
|
|
@@ -2032,9 +2307,11 @@ def run_self_test() -> int:
|
|
|
2032
2307
|
print(repr(digest_run.stdout), file=sys.stderr)
|
|
2033
2308
|
return 1
|
|
2034
2309
|
(devlyn / "pipeline.state.json").write_text(json.dumps({
|
|
2310
|
+
"run_id": "rs-risk-probes",
|
|
2035
2311
|
"source": {"type": "spec", "spec_path": str(spec_md)},
|
|
2036
2312
|
"risk_profile": {"risk_probes_enabled": True},
|
|
2037
2313
|
"risk_probes_digest": risk_digest,
|
|
2314
|
+
"phases": {"build_gate": {"round": 4}},
|
|
2038
2315
|
}))
|
|
2039
2316
|
good = subprocess.run(
|
|
2040
2317
|
[sys.executable, script_path, "--include-risk-probes"],
|
|
@@ -2046,6 +2323,24 @@ def run_self_test() -> int:
|
|
|
2046
2323
|
if good.returncode != 0:
|
|
2047
2324
|
print(good.stderr, file=sys.stderr)
|
|
2048
2325
|
return 1
|
|
2326
|
+
good_document = loads_strict_json(
|
|
2327
|
+
(devlyn / "spec-verify.results.json").read_text()
|
|
2328
|
+
)
|
|
2329
|
+
good_manifest_path = good_document.get("process_evidence", {}).get("manifest", {}).get("path")
|
|
2330
|
+
good_manifest = (
|
|
2331
|
+
loads_strict_json((work / good_manifest_path).read_text())
|
|
2332
|
+
if isinstance(good_manifest_path, str)
|
|
2333
|
+
else {}
|
|
2334
|
+
)
|
|
2335
|
+
if (
|
|
2336
|
+
good_manifest_path != ".devlyn/process-evidence/rs-risk-probes/build_gate/round-4/manifest.json"
|
|
2337
|
+
or [entry.get("id") for entry in good_manifest.get("entries", [])] != [
|
|
2338
|
+
"verification-command-0001", "risk-probe-0002",
|
|
2339
|
+
]
|
|
2340
|
+
):
|
|
2341
|
+
print("literal command and risk probe did not share the sealed manifest", file=sys.stderr)
|
|
2342
|
+
print(good_document, file=sys.stderr)
|
|
2343
|
+
return 1
|
|
2049
2344
|
|
|
2050
2345
|
probe_script.write_text("print('mutated-probe')\n", encoding="utf-8")
|
|
2051
2346
|
mutated_script = subprocess.run(
|
|
@@ -2424,22 +2719,13 @@ def run_self_test() -> int:
|
|
|
2424
2719
|
probes_dir = devlyn / "probes"
|
|
2425
2720
|
probes_dir.mkdir(exist_ok=True)
|
|
2426
2721
|
(probes_dir / "Pscript.py").write_text("print('script-ok')\n", encoding="utf-8")
|
|
2427
|
-
|
|
2722
|
+
script_probe_payload = {
|
|
2723
|
+
**risk_probe_payload,
|
|
2428
2724
|
"id": "Pscript",
|
|
2429
|
-
"derived_from": "probe must pass visible marker.",
|
|
2430
2725
|
"cmd": "python3 .devlyn/probes/Pscript.py",
|
|
2431
|
-
"exit_code": 0,
|
|
2432
2726
|
"stdout_contains": ["script-ok"],
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
"tag_evidence": {
|
|
2436
|
-
"shape_contract": [
|
|
2437
|
-
"uses_visible_input_key_names",
|
|
2438
|
-
"asserts_visible_output_key_names",
|
|
2439
|
-
"asserts_no_unexpected_output_keys",
|
|
2440
|
-
],
|
|
2441
|
-
},
|
|
2442
|
-
}) + "\n")
|
|
2727
|
+
}
|
|
2728
|
+
(devlyn / "risk-probes.jsonl").write_text(json.dumps(script_probe_payload) + "\n")
|
|
2443
2729
|
good_script_probe = subprocess.run(
|
|
2444
2730
|
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2445
2731
|
cwd=work,
|
|
@@ -2452,10 +2738,47 @@ def run_self_test() -> int:
|
|
|
2452
2738
|
print(good_script_probe.stderr, file=sys.stderr)
|
|
2453
2739
|
return 1
|
|
2454
2740
|
|
|
2741
|
+
mixed_script_payload = {
|
|
2742
|
+
**script_probe_payload,
|
|
2743
|
+
"id": "Pmixed",
|
|
2744
|
+
"cmd": "python3 .devlyn/probes/Pscript.py && python3 ./.devlyn/probes/P1.py",
|
|
2745
|
+
}
|
|
2746
|
+
(devlyn / "risk-probes.jsonl").write_text(json.dumps(mixed_script_payload) + "\n")
|
|
2747
|
+
mixed_script_probe = subprocess.run(
|
|
2748
|
+
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2749
|
+
cwd=work,
|
|
2750
|
+
env=env,
|
|
2751
|
+
capture_output=True,
|
|
2752
|
+
text=True,
|
|
2753
|
+
)
|
|
2754
|
+
if mixed_script_probe.returncode != 0:
|
|
2755
|
+
print("canonical and ./-alias probe script references were rejected", file=sys.stderr)
|
|
2756
|
+
print(mixed_script_probe.stderr, file=sys.stderr)
|
|
2757
|
+
return 1
|
|
2758
|
+
mixed_digest, mixed_digest_error = risk_probes_digest(devlyn)
|
|
2759
|
+
if mixed_digest_error:
|
|
2760
|
+
print(mixed_digest_error, file=sys.stderr)
|
|
2761
|
+
return 1
|
|
2762
|
+
(probes_dir / "Pscript.py").write_text("print('script-mutated')\n", encoding="utf-8")
|
|
2763
|
+
canonical_mutated_digest, canonical_mutated_error = risk_probes_digest(devlyn)
|
|
2764
|
+
(probes_dir / "Pscript.py").write_text("print('script-ok')\n", encoding="utf-8")
|
|
2765
|
+
if canonical_mutated_error:
|
|
2766
|
+
print(canonical_mutated_error, file=sys.stderr)
|
|
2767
|
+
return 1
|
|
2768
|
+
probe_script.write_text("print('probe-mutated')\n", encoding="utf-8")
|
|
2769
|
+
alias_mutated_digest, alias_mutated_error = risk_probes_digest(devlyn)
|
|
2770
|
+
probe_script.write_text("print('probe-ok')\n", encoding="utf-8")
|
|
2771
|
+
if alias_mutated_error:
|
|
2772
|
+
print(alias_mutated_error, file=sys.stderr)
|
|
2773
|
+
return 1
|
|
2774
|
+
if len({mixed_digest, canonical_mutated_digest, alias_mutated_digest}) != 3:
|
|
2775
|
+
print("probe digest did not change for both canonical and ./-alias scripts", file=sys.stderr)
|
|
2776
|
+
return 1
|
|
2777
|
+
|
|
2455
2778
|
(devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2456
2779
|
"id": "Pmissing",
|
|
2457
2780
|
"derived_from": "probe must pass visible marker.",
|
|
2458
|
-
"cmd": "python3
|
|
2781
|
+
"cmd": "python3 ./.devlyn/probes/missing.py",
|
|
2459
2782
|
"exit_code": 0,
|
|
2460
2783
|
}) + "\n")
|
|
2461
2784
|
missing_script_probe = subprocess.run(
|
|
@@ -2488,6 +2811,45 @@ def run_self_test() -> int:
|
|
|
2488
2811
|
print(missing_script_digest.stderr, file=sys.stderr)
|
|
2489
2812
|
return 1
|
|
2490
2813
|
|
|
2814
|
+
for bad_form in (
|
|
2815
|
+
"../.devlyn/probes/Pscript.py",
|
|
2816
|
+
"/tmp/.devlyn/probes/Pscript.py",
|
|
2817
|
+
"$PWD/.devlyn/probes/Pscript.py",
|
|
2818
|
+
"././.devlyn/probes/Pscript.py",
|
|
2819
|
+
):
|
|
2820
|
+
(devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2821
|
+
"id": "Pbadref",
|
|
2822
|
+
"derived_from": "probe must pass visible marker.",
|
|
2823
|
+
"cmd": f"python3 {bad_form}",
|
|
2824
|
+
"exit_code": 0,
|
|
2825
|
+
}) + "\n")
|
|
2826
|
+
bad_ref_probe = subprocess.run(
|
|
2827
|
+
[sys.executable, script_path, "--validate-risk-probes"],
|
|
2828
|
+
cwd=work,
|
|
2829
|
+
env=env,
|
|
2830
|
+
capture_output=True,
|
|
2831
|
+
text=True,
|
|
2832
|
+
)
|
|
2833
|
+
bad_ref_digest = subprocess.run(
|
|
2834
|
+
[sys.executable, script_path, "--print-risk-probes-digest"],
|
|
2835
|
+
cwd=work,
|
|
2836
|
+
env=env,
|
|
2837
|
+
capture_output=True,
|
|
2838
|
+
text=True,
|
|
2839
|
+
)
|
|
2840
|
+
unrecognized = unrecognized_risk_probe_reference(f"python3 {bad_form}")
|
|
2841
|
+
expected_error = f"risk-probes[0].cmd has {unrecognized}"
|
|
2842
|
+
if (
|
|
2843
|
+
bad_ref_probe.returncode == 0
|
|
2844
|
+
or expected_error not in bad_ref_probe.stderr
|
|
2845
|
+
or bad_ref_digest.returncode == 0
|
|
2846
|
+
or expected_error not in bad_ref_digest.stderr
|
|
2847
|
+
):
|
|
2848
|
+
print(f"bad probe script reference was not rejected: {bad_form}", file=sys.stderr)
|
|
2849
|
+
print(bad_ref_probe.stderr, file=sys.stderr)
|
|
2850
|
+
print(bad_ref_digest.stderr, file=sys.stderr)
|
|
2851
|
+
return 1
|
|
2852
|
+
|
|
2491
2853
|
(probes_dir / "Phidden.py").write_text("print('benchmark/auto-resolve/fixtures')\n", encoding="utf-8")
|
|
2492
2854
|
(devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2493
2855
|
"id": "Phidden",
|
|
@@ -3038,6 +3400,103 @@ def run_self_test() -> int:
|
|
|
3038
3400
|
print(expected_nan_json.stderr, file=sys.stderr)
|
|
3039
3401
|
return 1
|
|
3040
3402
|
|
|
3403
|
+
external_diff_root = work / "external-diff-mode-authority"
|
|
3404
|
+
external_diff_root.mkdir()
|
|
3405
|
+
external_diff_devlyn = external_diff_root / ".devlyn"
|
|
3406
|
+
external_diff_devlyn.mkdir()
|
|
3407
|
+
external_diff_spec = external_diff_root / "spec.md"
|
|
3408
|
+
external_diff_spec.write_text(
|
|
3409
|
+
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
3410
|
+
"- external diff remains verify-only.\n",
|
|
3411
|
+
encoding="utf-8",
|
|
3412
|
+
)
|
|
3413
|
+
(external_diff_root / "spec.expected.json").write_text(json.dumps({
|
|
3414
|
+
"verification_commands": [
|
|
3415
|
+
{"cmd": "printf external-diff-ok", "stdout_contains": ["external-diff-ok"]}
|
|
3416
|
+
]
|
|
3417
|
+
}) + "\n", encoding="utf-8")
|
|
3418
|
+
(external_diff_root / "external-only.txt").write_text("base\n", encoding="utf-8")
|
|
3419
|
+
(external_diff_root / "outside.txt").write_text("base\n", encoding="utf-8")
|
|
3420
|
+
subprocess.run(["git", "init", "-q"], cwd=external_diff_root, check=True)
|
|
3421
|
+
subprocess.run(["git", "add", "-A"], cwd=external_diff_root, check=True)
|
|
3422
|
+
subprocess.run(
|
|
3423
|
+
["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "base"],
|
|
3424
|
+
cwd=external_diff_root,
|
|
3425
|
+
check=True,
|
|
3426
|
+
)
|
|
3427
|
+
external_diff_base_sha = subprocess.check_output(
|
|
3428
|
+
["git", "rev-parse", "HEAD"], cwd=external_diff_root, text=True,
|
|
3429
|
+
).strip()
|
|
3430
|
+
(external_diff_root / "outside.txt").write_text("worktree-only\n", encoding="utf-8")
|
|
3431
|
+
(external_diff_devlyn / "external-diff.patch").write_text(
|
|
3432
|
+
"diff --git a/external-only.txt b/external-only.txt\n"
|
|
3433
|
+
"--- a/external-only.txt\n"
|
|
3434
|
+
"+++ b/external-only.txt\n"
|
|
3435
|
+
"@@ -1 +1 @@\n"
|
|
3436
|
+
"-base\n"
|
|
3437
|
+
"+external\n",
|
|
3438
|
+
encoding="utf-8",
|
|
3439
|
+
)
|
|
3440
|
+
(external_diff_devlyn / "plan.md").write_text(
|
|
3441
|
+
"<!-- devlyn:authorized-surface -->\n## Files to touch\n\n"
|
|
3442
|
+
"```json\n{\"authorized_surface\": [\"external-only.txt\"]}\n```\n",
|
|
3443
|
+
encoding="utf-8",
|
|
3444
|
+
)
|
|
3445
|
+
(external_diff_devlyn / "untracked.baseline").write_text("", encoding="utf-8")
|
|
3446
|
+
external_diff_state = {
|
|
3447
|
+
"mode": "free-form",
|
|
3448
|
+
"source": {"type": "spec", "spec_path": str(external_diff_spec)},
|
|
3449
|
+
"base_ref": {"sha": external_diff_base_sha},
|
|
3450
|
+
}
|
|
3451
|
+
external_diff_state_path = external_diff_devlyn / "pipeline.state.json"
|
|
3452
|
+
external_diff_state_path.write_text(json.dumps(external_diff_state) + "\n")
|
|
3453
|
+
external_diff_free_form = subprocess.run(
|
|
3454
|
+
[sys.executable, script_path],
|
|
3455
|
+
cwd=external_diff_root,
|
|
3456
|
+
capture_output=True,
|
|
3457
|
+
text=True,
|
|
3458
|
+
)
|
|
3459
|
+
external_diff_findings_path = external_diff_devlyn / output_findings_name()
|
|
3460
|
+
external_diff_free_form_findings = (
|
|
3461
|
+
external_diff_findings_path.read_text()
|
|
3462
|
+
if external_diff_findings_path.is_file()
|
|
3463
|
+
else ""
|
|
3464
|
+
)
|
|
3465
|
+
external_diff_state["mode"] = "verify-only"
|
|
3466
|
+
external_diff_state_path.write_text(json.dumps(external_diff_state) + "\n")
|
|
3467
|
+
external_diff_verify_only = subprocess.run(
|
|
3468
|
+
[sys.executable, script_path],
|
|
3469
|
+
cwd=external_diff_root,
|
|
3470
|
+
capture_output=True,
|
|
3471
|
+
text=True,
|
|
3472
|
+
)
|
|
3473
|
+
if external_diff_free_form.returncode != 1:
|
|
3474
|
+
print("non-verify-only mode accepted .devlyn/external-diff.patch", file=sys.stderr)
|
|
3475
|
+
print(external_diff_free_form.stderr, file=sys.stderr)
|
|
3476
|
+
return 1
|
|
3477
|
+
if (
|
|
3478
|
+
'"rule_id": "correctness.spec-verify-malformed"'
|
|
3479
|
+
not in external_diff_free_form_findings
|
|
3480
|
+
or '"severity": "CRITICAL"' not in external_diff_free_form_findings
|
|
3481
|
+
or ".devlyn/external-diff.patch" not in external_diff_free_form_findings
|
|
3482
|
+
or "free-form" not in external_diff_free_form_findings
|
|
3483
|
+
or "verify-only" not in external_diff_free_form_findings
|
|
3484
|
+
or "Remove `.devlyn/external-diff.patch` for ordinary runs"
|
|
3485
|
+
not in external_diff_free_form_findings
|
|
3486
|
+
or "only when intentionally verifying an external patch"
|
|
3487
|
+
not in external_diff_free_form_findings
|
|
3488
|
+
):
|
|
3489
|
+
print(
|
|
3490
|
+
"non-verify-only external diff did not emit the named CRITICAL finding and remediation",
|
|
3491
|
+
file=sys.stderr,
|
|
3492
|
+
)
|
|
3493
|
+
print(external_diff_free_form_findings, file=sys.stderr)
|
|
3494
|
+
return 1
|
|
3495
|
+
if external_diff_verify_only.returncode != 0:
|
|
3496
|
+
print("verify-only external diff was rejected or not consumed", file=sys.stderr)
|
|
3497
|
+
print(external_diff_verify_only.stderr, file=sys.stderr)
|
|
3498
|
+
return 1
|
|
3499
|
+
|
|
3041
3500
|
spec_integrity = work / "spec-integrity"
|
|
3042
3501
|
spec_integrity.mkdir()
|
|
3043
3502
|
spec_integrity_devlyn = spec_integrity / ".devlyn"
|
|
@@ -3975,6 +4434,32 @@ def run_self_test() -> int:
|
|
|
3975
4434
|
print("--print-authorized-surface accepted malformed plan.md", file=sys.stderr)
|
|
3976
4435
|
return 1
|
|
3977
4436
|
|
|
4437
|
+
for entry in ("src/{a,b", "src/{a,}/**", "src/{a,{b,c}}", "src/{a,**}"):
|
|
4438
|
+
(scope_devlyn / "plan.md").write_text(
|
|
4439
|
+
"# PLAN\n\n<!-- devlyn:authorized-surface -->\n## 1. Files to touch\n\n"
|
|
4440
|
+
"```json\n"
|
|
4441
|
+
+ json.dumps({"authorized_surface": [entry]}) + "\n```\n"
|
|
4442
|
+
)
|
|
4443
|
+
brace_build_gate = subprocess.run(
|
|
4444
|
+
[sys.executable, script_path], cwd=scope_root, env=scope_build_gate_env,
|
|
4445
|
+
capture_output=True, text=True,
|
|
4446
|
+
)
|
|
4447
|
+
brace_print_surface = subprocess.run(
|
|
4448
|
+
[sys.executable, script_path, "--print-authorized-surface"], cwd=scope_root,
|
|
4449
|
+
capture_output=True, text=True,
|
|
4450
|
+
)
|
|
4451
|
+
findings_text = scope_findings_path.read_text(encoding="utf-8")
|
|
4452
|
+
if (
|
|
4453
|
+
brace_build_gate.returncode == 0
|
|
4454
|
+
or brace_print_surface.returncode == 0
|
|
4455
|
+
or "scope.authorized-surface-malformed" not in findings_text
|
|
4456
|
+
or "supported form" not in findings_text
|
|
4457
|
+
or "supported form" not in brace_print_surface.stderr
|
|
4458
|
+
or "Traceback" in brace_build_gate.stderr + brace_print_surface.stderr
|
|
4459
|
+
):
|
|
4460
|
+
print(f"malformed brace glob escaped the authorized-surface carrier: {entry}", file=sys.stderr)
|
|
4461
|
+
return 1
|
|
4462
|
+
|
|
3978
4463
|
# Test 3: valid surface, in-scope-only diff -> no scope findings, exit 0.
|
|
3979
4464
|
(scope_devlyn / "plan.md").write_text(
|
|
3980
4465
|
"# PLAN\n\n<!-- devlyn:authorized-surface -->\n## 1. Files to touch\n\n"
|
|
@@ -4107,12 +4592,16 @@ def run_self_test() -> int:
|
|
|
4107
4592
|
print(verify_mech_findings, file=sys.stderr)
|
|
4108
4593
|
return 1
|
|
4109
4594
|
|
|
4110
|
-
# Test 7: shape validation rejects absolute paths, `..`, and
|
|
4595
|
+
# Test 7: shape validation rejects absolute paths, `..`, duplicates, and malformed braces.
|
|
4111
4596
|
for bad_surface in (
|
|
4112
4597
|
{"authorized_surface": ["/etc/passwd"]},
|
|
4113
4598
|
{"authorized_surface": ["bin/../etc/passwd"]},
|
|
4114
4599
|
{"authorized_surface": ["bin/cli.js", "bin/cli.js"]},
|
|
4115
4600
|
{"authorized_surface": []},
|
|
4601
|
+
{"authorized_surface": ["src/{a,b"]},
|
|
4602
|
+
{"authorized_surface": ["src/{a,}/**"]},
|
|
4603
|
+
{"authorized_surface": ["src/{a,{b,c}}"]},
|
|
4604
|
+
{"authorized_surface": ["src/{a,**}"]},
|
|
4116
4605
|
):
|
|
4117
4606
|
err = validate_authorized_surface_shape(bad_surface)
|
|
4118
4607
|
if err is None:
|
|
@@ -4166,6 +4655,14 @@ def run_self_test() -> int:
|
|
|
4166
4655
|
|
|
4167
4656
|
|
|
4168
4657
|
def main() -> int:
|
|
4658
|
+
if "--help" in sys.argv[1:] or "-h" in sys.argv[1:]:
|
|
4659
|
+
print(
|
|
4660
|
+
"usage: spec-verify-check.py [-h | --help | --include-risk-probes | "
|
|
4661
|
+
"--validate-risk-probes | --print-risk-probes-digest | "
|
|
4662
|
+
"--print-authorized-surface | --write-untracked-baseline | "
|
|
4663
|
+
"--check <markdown-path> | --check-expected <json-path> | --self-test]"
|
|
4664
|
+
)
|
|
4665
|
+
return 0
|
|
4169
4666
|
include_risk_probes = False
|
|
4170
4667
|
validate_risk_probes_only = False
|
|
4171
4668
|
print_risk_probes_digest = False
|
|
@@ -4202,6 +4699,10 @@ def main() -> int:
|
|
|
4202
4699
|
return 2
|
|
4203
4700
|
return run_check_expected_mode(Path(sys.argv[2]))
|
|
4204
4701
|
|
|
4702
|
+
if len(sys.argv) != 1:
|
|
4703
|
+
print(f"unknown argument(s): {' '.join(sys.argv[1:])}", file=sys.stderr)
|
|
4704
|
+
return 2
|
|
4705
|
+
|
|
4205
4706
|
bench_mode = "BENCH_WORKDIR" in os.environ
|
|
4206
4707
|
work = Path(os.environ.get("BENCH_WORKDIR") or os.getcwd())
|
|
4207
4708
|
devlyn_dir = work / ".devlyn"
|
|
@@ -4254,6 +4755,24 @@ def main() -> int:
|
|
|
4254
4755
|
trust_bench_staged = bench_mode and pre_staged
|
|
4255
4756
|
src_type, source_md = read_source(work, devlyn_dir)
|
|
4256
4757
|
state = read_state(devlyn_dir)
|
|
4758
|
+
external_diff = devlyn_dir / "external-diff.patch"
|
|
4759
|
+
if external_diff.is_file() and state.get("mode") != "verify-only":
|
|
4760
|
+
error = (
|
|
4761
|
+
".devlyn/external-diff.patch requires pipeline.state.json mode "
|
|
4762
|
+
f"'verify-only'; actual mode is {state.get('mode')!r}"
|
|
4763
|
+
)
|
|
4764
|
+
print(f"[spec-verify] carrier malformed: {error}", file=sys.stderr)
|
|
4765
|
+
write_malformed_finding(
|
|
4766
|
+
devlyn_dir,
|
|
4767
|
+
error,
|
|
4768
|
+
external_diff,
|
|
4769
|
+
fix_hint=(
|
|
4770
|
+
"Remove `.devlyn/external-diff.patch` for ordinary runs, or use "
|
|
4771
|
+
"`/devlyn:resolve --verify-only <diff-or-PR-ref> --spec <path>` only when "
|
|
4772
|
+
"intentionally verifying an external patch."
|
|
4773
|
+
),
|
|
4774
|
+
)
|
|
4775
|
+
return 1
|
|
4257
4776
|
integrity_error = source_integrity_error(src_type, state, source_md)
|
|
4258
4777
|
if integrity_error:
|
|
4259
4778
|
print(f"[spec-verify] carrier malformed: {integrity_error}", file=sys.stderr)
|
|
@@ -4372,18 +4891,17 @@ def main() -> int:
|
|
|
4372
4891
|
results_path = devlyn_dir / "spec-verify.results.json"
|
|
4373
4892
|
findings_path = devlyn_dir / output_findings_name()
|
|
4374
4893
|
|
|
4375
|
-
verify_env = os.environ.copy()
|
|
4376
|
-
for runner_key in (
|
|
4377
|
-
"SPEC_VERIFY_PHASE",
|
|
4378
|
-
"SPEC_VERIFY_FINDINGS_FILE",
|
|
4379
|
-
"SPEC_VERIFY_FINDING_PREFIX",
|
|
4380
|
-
):
|
|
4381
|
-
verify_env.pop(runner_key, None)
|
|
4382
|
-
verify_env["BENCH_WORKDIR"] = str(work)
|
|
4383
|
-
|
|
4384
4894
|
results: list[dict] = []
|
|
4385
4895
|
findings: list[dict] = []
|
|
4386
4896
|
finding_seq = 1
|
|
4897
|
+
runner = process_evidence_module()
|
|
4898
|
+
evidence_phase, evidence_run_id, evidence_round, manifest_relative = (
|
|
4899
|
+
mechanical_evidence_identity(state, runner)
|
|
4900
|
+
)
|
|
4901
|
+
manifest_path = work / manifest_relative
|
|
4902
|
+
obligations: list[dict] = []
|
|
4903
|
+
capability_denials: list[dict] = []
|
|
4904
|
+
evidence_error: str | None = None
|
|
4387
4905
|
|
|
4388
4906
|
for idx, vc in enumerate(commands):
|
|
4389
4907
|
cmd = vc.get("cmd")
|
|
@@ -4397,177 +4915,185 @@ def main() -> int:
|
|
|
4397
4915
|
stdout_contains = vc.get("stdout_contains", []) or []
|
|
4398
4916
|
stdout_not_contains = vc.get("stdout_not_contains", []) or []
|
|
4399
4917
|
timeout_sec = verification_timeout_sec(vc)
|
|
4918
|
+
obligation = mechanical_obligation(vc, idx, evidence_phase)
|
|
4919
|
+
obligations.append(obligation)
|
|
4920
|
+
criterion_ref = (
|
|
4921
|
+
f"risk-probe:{vc.get('id')}"
|
|
4922
|
+
if is_risk_probe
|
|
4923
|
+
else f"spec-verify://verification_commands/{idx}"
|
|
4924
|
+
)
|
|
4925
|
+
file_ref = (
|
|
4926
|
+
".devlyn/risk-probes.jsonl"
|
|
4927
|
+
if is_risk_probe
|
|
4928
|
+
else ".devlyn/spec-verify.json"
|
|
4929
|
+
)
|
|
4400
4930
|
|
|
4401
4931
|
try:
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
shell=True,
|
|
4406
|
-
env=verify_env,
|
|
4407
|
-
capture_output=True,
|
|
4408
|
-
text=True,
|
|
4409
|
-
timeout=timeout_sec,
|
|
4932
|
+
entry = capture_mechanical_command(
|
|
4933
|
+
runner, work, manifest_path, evidence_run_id, evidence_phase,
|
|
4934
|
+
evidence_round, obligation,
|
|
4410
4935
|
)
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
ok_not = not any(s in out for s in stdout_not_contains)
|
|
4416
|
-
passed = bool(ok_exit and ok_contains and ok_not)
|
|
4417
|
-
|
|
4418
|
-
if passed:
|
|
4419
|
-
reason = None
|
|
4420
|
-
elif not ok_exit:
|
|
4421
|
-
reason = "exit"
|
|
4422
|
-
elif not ok_contains:
|
|
4423
|
-
reason = "missing_contains"
|
|
4424
|
-
else:
|
|
4425
|
-
reason = "unexpected_text"
|
|
4426
|
-
|
|
4936
|
+
stdout = (work / entry["stdout"]["path"]).read_bytes()
|
|
4937
|
+
stderr = (work / entry["stderr"]["path"]).read_bytes()
|
|
4938
|
+
except (runner.EvidenceError, OSError, UnicodeError, ValueError) as exc:
|
|
4939
|
+
evidence_error = str(exc)
|
|
4427
4940
|
results.append({
|
|
4428
4941
|
"index": idx,
|
|
4429
4942
|
"cmd": cmd,
|
|
4430
|
-
"
|
|
4431
|
-
"
|
|
4432
|
-
"stdout_contains": stdout_contains,
|
|
4433
|
-
"stdout_not_contains": stdout_not_contains,
|
|
4434
|
-
"pass": passed,
|
|
4435
|
-
"reason": reason,
|
|
4436
|
-
"stdout_tail": out[-500:],
|
|
4943
|
+
"pass": False,
|
|
4944
|
+
"reason": "process_evidence_invalid",
|
|
4437
4945
|
})
|
|
4438
|
-
|
|
4439
|
-
if not passed:
|
|
4440
|
-
# Construct fine-grained message naming the specific failure.
|
|
4441
|
-
if not ok_exit:
|
|
4442
|
-
msg = (
|
|
4443
|
-
f"Verification command #{idx + 1} failed: expected exit "
|
|
4444
|
-
f"{expected_exit}, got {proc.returncode}."
|
|
4445
|
-
)
|
|
4446
|
-
elif not ok_contains:
|
|
4447
|
-
missing = [s for s in stdout_contains if s not in out]
|
|
4448
|
-
msg = (
|
|
4449
|
-
f"Verification command #{idx + 1} failed: expected "
|
|
4450
|
-
f"output to contain {missing!r}."
|
|
4451
|
-
)
|
|
4452
|
-
else:
|
|
4453
|
-
forbidden = [s for s in stdout_not_contains if s in out]
|
|
4454
|
-
msg = (
|
|
4455
|
-
f"Verification command #{idx + 1} failed: output "
|
|
4456
|
-
f"contained forbidden literal(s) {forbidden!r}."
|
|
4457
|
-
)
|
|
4458
|
-
|
|
4459
|
-
fix_hint = (
|
|
4460
|
-
f"See .devlyn/spec-verify.results.json for the captured "
|
|
4461
|
-
f"output. Update implementation so `{cmd}` matches the "
|
|
4462
|
-
f"contract (exit_code={expected_exit}, "
|
|
4463
|
-
f"contains={stdout_contains}, not_contains={stdout_not_contains})."
|
|
4464
|
-
)
|
|
4465
|
-
|
|
4466
|
-
rule_id = (
|
|
4467
|
-
"correctness.risk-probe-failed"
|
|
4468
|
-
if is_risk_probe
|
|
4469
|
-
else "correctness.spec-literal-mismatch"
|
|
4470
|
-
)
|
|
4471
|
-
criterion_ref = (
|
|
4472
|
-
f"risk-probe:{vc.get('id')}"
|
|
4473
|
-
if is_risk_probe
|
|
4474
|
-
else f"spec-verify://verification_commands/{idx}"
|
|
4475
|
-
)
|
|
4476
|
-
file_ref = (
|
|
4477
|
-
".devlyn/risk-probes.jsonl"
|
|
4478
|
-
if is_risk_probe
|
|
4479
|
-
else ".devlyn/spec-verify.json"
|
|
4480
|
-
)
|
|
4481
|
-
if is_risk_probe:
|
|
4482
|
-
fix_hint = (
|
|
4483
|
-
f"Risk probe `{vc.get('id')}` derived from "
|
|
4484
|
-
f"{vc.get('derived_from')!r} failed. See "
|
|
4485
|
-
".devlyn/spec-verify.results.json for captured output "
|
|
4486
|
-
"and update the implementation to satisfy the visible "
|
|
4487
|
-
"verification bullet."
|
|
4488
|
-
)
|
|
4489
|
-
|
|
4490
|
-
findings.append({
|
|
4491
|
-
"id": f"{output_finding_prefix()}-{finding_seq:04d}",
|
|
4492
|
-
"rule_id": rule_id,
|
|
4493
|
-
"level": "error",
|
|
4494
|
-
"severity": "CRITICAL",
|
|
4495
|
-
"confidence": 1.0,
|
|
4496
|
-
"message": msg,
|
|
4497
|
-
"file": file_ref,
|
|
4498
|
-
"line": 1,
|
|
4499
|
-
"phase": output_phase(),
|
|
4500
|
-
"criterion_ref": criterion_ref,
|
|
4501
|
-
"fix_hint": fix_hint,
|
|
4502
|
-
"blocking": True,
|
|
4503
|
-
"status": "open",
|
|
4504
|
-
})
|
|
4505
|
-
finding_seq += 1
|
|
4506
|
-
|
|
4507
|
-
except subprocess.TimeoutExpired:
|
|
4508
|
-
results.append({"index": idx, "cmd": cmd, "pass": False,
|
|
4509
|
-
"reason": "timeout", "timeout_sec": timeout_sec})
|
|
4510
4946
|
findings.append({
|
|
4511
4947
|
"id": f"{output_finding_prefix()}-{finding_seq:04d}",
|
|
4512
|
-
"rule_id": "
|
|
4948
|
+
"rule_id": "invariant.process-evidence-invalid",
|
|
4513
4949
|
"level": "error",
|
|
4514
4950
|
"severity": "CRITICAL",
|
|
4515
4951
|
"confidence": 1.0,
|
|
4516
|
-
"message":
|
|
4517
|
-
|
|
4518
|
-
f"(timeout_sec={timeout_sec}, maximum 600)."
|
|
4519
|
-
),
|
|
4520
|
-
"file": ".devlyn/risk-probes.jsonl" if vc.get("_risk_probe") else ".devlyn/spec-verify.json",
|
|
4952
|
+
"message": f"MECHANICAL process evidence is invalid: {exc}.",
|
|
4953
|
+
"file": manifest_relative,
|
|
4521
4954
|
"line": 1,
|
|
4522
4955
|
"phase": output_phase(),
|
|
4523
|
-
"criterion_ref":
|
|
4524
|
-
f"risk-probe:{vc.get('id')}"
|
|
4525
|
-
if vc.get("_risk_probe")
|
|
4526
|
-
else f"spec-verify://verification_commands/{idx}"
|
|
4527
|
-
),
|
|
4956
|
+
"criterion_ref": "process-evidence://mechanical",
|
|
4528
4957
|
"fix_hint": (
|
|
4529
|
-
|
|
4530
|
-
"
|
|
4531
|
-
"needs more time, or fix a hang in the implementation."
|
|
4958
|
+
"Preserve the existing manifest and raw streams, then fix the "
|
|
4959
|
+
"runner identity or evidence mutation before rerunning this phase."
|
|
4532
4960
|
),
|
|
4533
4961
|
"blocking": True,
|
|
4534
4962
|
"status": "open",
|
|
4535
4963
|
})
|
|
4536
4964
|
finding_seq += 1
|
|
4537
|
-
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4965
|
+
break
|
|
4966
|
+
|
|
4967
|
+
outcome = entry["outcome"]
|
|
4968
|
+
classification = entry["classification"]
|
|
4969
|
+
combined = stdout + stderr
|
|
4970
|
+
actual_exit = outcome["exit_code"] if outcome["kind"] == "exit" else None
|
|
4971
|
+
ok_exit = outcome["kind"] == "exit" and actual_exit == expected_exit
|
|
4972
|
+
ok_contains = all(s.encode("utf-8") in combined for s in stdout_contains)
|
|
4973
|
+
ok_not = not any(s.encode("utf-8") in combined for s in stdout_not_contains)
|
|
4974
|
+
passed = entry["expectation_met"]
|
|
4975
|
+
if classification["kind"] == "capability_denied":
|
|
4976
|
+
reason = "capability_denied"
|
|
4977
|
+
capability_denials.append({
|
|
4978
|
+
"cmd": cmd,
|
|
4979
|
+
"operation": classification["operation"],
|
|
4980
|
+
"evidence_id": entry["id"],
|
|
4981
|
+
})
|
|
4982
|
+
elif passed:
|
|
4983
|
+
reason = None
|
|
4984
|
+
elif outcome["kind"] == "timeout":
|
|
4985
|
+
reason = "timeout"
|
|
4986
|
+
elif not ok_exit:
|
|
4987
|
+
reason = "exit"
|
|
4988
|
+
elif not ok_contains:
|
|
4989
|
+
reason = "missing_contains"
|
|
4990
|
+
else:
|
|
4991
|
+
reason = "unexpected_text"
|
|
4992
|
+
|
|
4993
|
+
results.append({
|
|
4994
|
+
"index": idx,
|
|
4995
|
+
"cmd": cmd,
|
|
4996
|
+
"expected_exit": expected_exit,
|
|
4997
|
+
"actual_exit": actual_exit,
|
|
4998
|
+
"timeout_sec": timeout_sec,
|
|
4999
|
+
"stdout_contains": stdout_contains,
|
|
5000
|
+
"stdout_not_contains": stdout_not_contains,
|
|
5001
|
+
"pass": passed,
|
|
5002
|
+
"reason": reason,
|
|
5003
|
+
"evidence_id": entry["id"],
|
|
5004
|
+
"outcome": outcome,
|
|
5005
|
+
"classification": classification,
|
|
5006
|
+
"stdout": entry["stdout"],
|
|
5007
|
+
"stderr": entry["stderr"],
|
|
5008
|
+
})
|
|
5009
|
+
|
|
5010
|
+
if classification["kind"] == "capability_denied" or passed:
|
|
5011
|
+
continue
|
|
5012
|
+
|
|
5013
|
+
if outcome["kind"] == "timeout":
|
|
4545
5014
|
findings.append({
|
|
4546
5015
|
"id": f"{output_finding_prefix()}-{finding_seq:04d}",
|
|
4547
|
-
"rule_id":
|
|
5016
|
+
"rule_id": "correctness.verification-timeout",
|
|
4548
5017
|
"level": "error",
|
|
4549
5018
|
"severity": "CRITICAL",
|
|
4550
5019
|
"confidence": 1.0,
|
|
4551
5020
|
"message": (
|
|
4552
|
-
f"Verification command #{idx + 1}
|
|
4553
|
-
f"{
|
|
5021
|
+
f"Verification command #{idx + 1} timed out after {timeout_sec}s "
|
|
5022
|
+
f"(timeout_sec={timeout_sec}, maximum 600)."
|
|
4554
5023
|
),
|
|
4555
|
-
"file":
|
|
5024
|
+
"file": file_ref,
|
|
4556
5025
|
"line": 1,
|
|
4557
5026
|
"phase": output_phase(),
|
|
4558
|
-
"criterion_ref":
|
|
4559
|
-
f"risk-probe:{vc.get('id')}"
|
|
4560
|
-
if vc.get("_risk_probe")
|
|
4561
|
-
else f"spec-verify://verification_commands/{idx}"
|
|
4562
|
-
),
|
|
5027
|
+
"criterion_ref": criterion_ref,
|
|
4563
5028
|
"fix_hint": (
|
|
4564
|
-
f"Command `{cmd}`
|
|
4565
|
-
|
|
5029
|
+
f"Command `{cmd}` exceeded its {timeout_sec}s timeout_sec budget. "
|
|
5030
|
+
"Increase timeout_sec up to 600 when the verification legitimately "
|
|
5031
|
+
"needs more time, or fix a hang in the implementation."
|
|
4566
5032
|
),
|
|
4567
5033
|
"blocking": True,
|
|
4568
5034
|
"status": "open",
|
|
4569
5035
|
})
|
|
4570
5036
|
finding_seq += 1
|
|
5037
|
+
continue
|
|
5038
|
+
|
|
5039
|
+
if not ok_exit:
|
|
5040
|
+
actual = (
|
|
5041
|
+
str(actual_exit)
|
|
5042
|
+
if outcome["kind"] == "exit"
|
|
5043
|
+
else f"{outcome['kind']}"
|
|
5044
|
+
+ (f" signal {outcome['signal']}" if outcome["kind"] == "signal" else "")
|
|
5045
|
+
)
|
|
5046
|
+
msg = (
|
|
5047
|
+
f"Verification command #{idx + 1} failed: expected exit "
|
|
5048
|
+
f"{expected_exit}, got {actual}."
|
|
5049
|
+
)
|
|
5050
|
+
elif not ok_contains:
|
|
5051
|
+
missing = [s for s in stdout_contains if s.encode("utf-8") not in combined]
|
|
5052
|
+
msg = (
|
|
5053
|
+
f"Verification command #{idx + 1} failed: expected "
|
|
5054
|
+
f"output to contain {missing!r}."
|
|
5055
|
+
)
|
|
5056
|
+
else:
|
|
5057
|
+
forbidden = [s for s in stdout_not_contains if s.encode("utf-8") in combined]
|
|
5058
|
+
msg = (
|
|
5059
|
+
f"Verification command #{idx + 1} failed: output "
|
|
5060
|
+
f"contained forbidden literal(s) {forbidden!r}."
|
|
5061
|
+
)
|
|
5062
|
+
|
|
5063
|
+
rule_id = (
|
|
5064
|
+
"correctness.risk-probe-failed"
|
|
5065
|
+
if is_risk_probe
|
|
5066
|
+
else "correctness.spec-literal-mismatch"
|
|
5067
|
+
)
|
|
5068
|
+
fix_hint = (
|
|
5069
|
+
f"Inspect `{manifest_relative}` entry `{entry['id']}` and its sealed raw "
|
|
5070
|
+
f"streams. Update implementation so `{cmd}` matches the contract "
|
|
5071
|
+
f"(exit_code={expected_exit}, contains={stdout_contains}, "
|
|
5072
|
+
f"not_contains={stdout_not_contains})."
|
|
5073
|
+
)
|
|
5074
|
+
if is_risk_probe:
|
|
5075
|
+
fix_hint = (
|
|
5076
|
+
f"Risk probe `{vc.get('id')}` derived from {vc.get('derived_from')!r} "
|
|
5077
|
+
f"failed. Inspect `{manifest_relative}` entry `{entry['id']}` and "
|
|
5078
|
+
"update the implementation to satisfy the visible verification bullet."
|
|
5079
|
+
)
|
|
5080
|
+
|
|
5081
|
+
findings.append({
|
|
5082
|
+
"id": f"{output_finding_prefix()}-{finding_seq:04d}",
|
|
5083
|
+
"rule_id": rule_id,
|
|
5084
|
+
"level": "error",
|
|
5085
|
+
"severity": "CRITICAL",
|
|
5086
|
+
"confidence": 1.0,
|
|
5087
|
+
"message": msg,
|
|
5088
|
+
"file": file_ref,
|
|
5089
|
+
"line": 1,
|
|
5090
|
+
"phase": output_phase(),
|
|
5091
|
+
"criterion_ref": criterion_ref,
|
|
5092
|
+
"fix_hint": fix_hint,
|
|
5093
|
+
"blocking": True,
|
|
5094
|
+
"status": "open",
|
|
5095
|
+
})
|
|
5096
|
+
finding_seq += 1
|
|
4571
5097
|
|
|
4572
5098
|
expected_findings, finding_seq = expected_contract_findings(
|
|
4573
5099
|
expected_data,
|
|
@@ -4586,7 +5112,40 @@ def main() -> int:
|
|
|
4586
5112
|
)
|
|
4587
5113
|
findings.extend(surface_findings)
|
|
4588
5114
|
|
|
4589
|
-
|
|
5115
|
+
evidence_carrier = None
|
|
5116
|
+
if evidence_error is None and obligations:
|
|
5117
|
+
try:
|
|
5118
|
+
evidence_carrier = runner.validate_manifest(
|
|
5119
|
+
work, manifest_relative, evidence_run_id, evidence_phase,
|
|
5120
|
+
evidence_round, obligations, require_expectations=False,
|
|
5121
|
+
)
|
|
5122
|
+
results = runner.bound_carrier_summary_commands(work, evidence_carrier)
|
|
5123
|
+
except (runner.EvidenceError, OSError, UnicodeError, ValueError) as exc:
|
|
5124
|
+
evidence_error = str(exc)
|
|
5125
|
+
findings.append({
|
|
5126
|
+
"id": f"{output_finding_prefix()}-{finding_seq:04d}",
|
|
5127
|
+
"rule_id": "invariant.process-evidence-invalid",
|
|
5128
|
+
"level": "error",
|
|
5129
|
+
"severity": "CRITICAL",
|
|
5130
|
+
"confidence": 1.0,
|
|
5131
|
+
"message": f"MECHANICAL process evidence validation failed: {exc}.",
|
|
5132
|
+
"file": manifest_relative,
|
|
5133
|
+
"line": 1,
|
|
5134
|
+
"phase": output_phase(),
|
|
5135
|
+
"criterion_ref": "process-evidence://mechanical",
|
|
5136
|
+
"fix_hint": (
|
|
5137
|
+
"Preserve the manifest and raw streams, then correct the "
|
|
5138
|
+
"missing, altered, or path-escaping evidence before rerunning."
|
|
5139
|
+
),
|
|
5140
|
+
"blocking": True,
|
|
5141
|
+
"status": "open",
|
|
5142
|
+
})
|
|
5143
|
+
finding_seq += 1
|
|
5144
|
+
|
|
5145
|
+
results_path.write_text(json.dumps({
|
|
5146
|
+
"commands": results,
|
|
5147
|
+
"process_evidence": evidence_carrier,
|
|
5148
|
+
}, indent=2) + "\n")
|
|
4590
5149
|
|
|
4591
5150
|
# Append findings (jsonl). BUILD_GATE merge step concatenates this onto
|
|
4592
5151
|
# build_gate.findings.jsonl; never overwrite the orchestrator's own gate
|
|
@@ -4597,6 +5156,15 @@ def main() -> int:
|
|
|
4597
5156
|
|
|
4598
5157
|
failed = [r for r in results if r.get("pass") is False]
|
|
4599
5158
|
blocking_findings = [f for f in findings if f.get("severity") in {"CRITICAL", "HIGH"}]
|
|
5159
|
+
if capability_denials:
|
|
5160
|
+
denial = capability_denials[0]
|
|
5161
|
+
print(
|
|
5162
|
+
"BLOCKED:build-env-underprovisioned: "
|
|
5163
|
+
f"{denial['operation']} denied for `{denial['cmd']}`; "
|
|
5164
|
+
f"evidence {manifest_relative}#{denial['evidence_id']}",
|
|
5165
|
+
file=sys.stderr,
|
|
5166
|
+
)
|
|
5167
|
+
return 2
|
|
4600
5168
|
if failed or blocking_findings:
|
|
4601
5169
|
print(
|
|
4602
5170
|
f"[spec-verify] {len(failed)}/{len(results)} command(s) failed; "
|