devlyn-cli 2.10.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -9
- package/CLAUDE.md +11 -15
- package/README.md +31 -7
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +25 -0
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +10 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +28 -0
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +30 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +32 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +44 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +16 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +18 -0
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +9 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +28 -0
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +15 -0
- package/benchmark/auto-resolve/scripts/run-fixture.sh +11 -1
- package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +83 -1
- package/bin/devlyn.js +59 -9
- package/config/skills/_shared/.ruff.toml +2 -0
- package/config/skills/_shared/adapters/claude.md +9 -1
- package/config/skills/_shared/adapters/codex.md +5 -0
- package/config/skills/_shared/adapters/grok.md +37 -13
- package/config/skills/_shared/archive_run.py +738 -56
- package/config/skills/_shared/codex-config.md +19 -4
- package/config/skills/_shared/codex-monitored.sh +81 -12
- package/config/skills/_shared/collect-codex-findings.py +419 -60
- package/config/skills/_shared/engine-preflight.md +8 -7
- package/config/skills/_shared/expected.schema.json +68 -0
- package/config/skills/_shared/finish-gate.py +12 -11
- package/config/skills/_shared/grok-anchor-guard.py +78 -0
- package/config/skills/_shared/invocation-receipt.py +1041 -0
- package/config/skills/_shared/judge-output-parser.py +200 -0
- package/config/skills/_shared/judge-role-evidence.py +273 -0
- package/config/skills/_shared/phase-prompt-render.py +181 -0
- package/config/skills/_shared/process-evidence.py +909 -0
- package/config/skills/_shared/resolve-bootstrap.py +776 -84
- package/config/skills/_shared/resolve-stop-hook.py +33 -1
- package/config/skills/_shared/role-config.py +552 -0
- package/config/skills/_shared/run-bounded.py +1 -1
- package/config/skills/_shared/spec-verify-check.py +1037 -192
- package/config/skills/_shared/state-phase-write.py +2551 -83
- package/config/skills/_shared/terminal-claim-check.py +15 -0
- package/config/skills/_shared/verify-merge-findings.py +1127 -50
- package/config/skills/devlyn:engines/SKILL.md +8 -16
- package/config/skills/devlyn:ideate/SKILL.md +1 -1
- package/config/skills/devlyn:ideate/references/spec-template.md +1 -0
- package/config/skills/devlyn:queue/SKILL.md +7 -4
- package/config/skills/devlyn:resolve/SKILL.md +94 -35
- package/config/skills/devlyn:resolve/references/outer-loop.md +15 -0
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +33 -4
- package/config/skills/devlyn:resolve/references/phases/implement.md +4 -3
- package/config/skills/devlyn:resolve/references/phases/plan.md +1 -5
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +4 -2
- package/config/skills/devlyn:resolve/references/phases/verify.md +120 -80
- package/config/skills/devlyn:resolve/references/state-schema.md +23 -9
- package/package.json +5 -1
- package/scripts/lint-fixtures.sh +4 -1
- package/scripts/lint-skills.sh +447 -290
|
@@ -0,0 +1,1041 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Create and validate run-owned Codex invocation receipts."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import pathlib
|
|
10
|
+
import re
|
|
11
|
+
import signal
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import tempfile
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
SCHEMA_VERSION = "2.0"
|
|
19
|
+
PHASES = {"plan", "implement", "build_gate", "cleanup"}
|
|
20
|
+
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
21
|
+
SAFE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
|
22
|
+
DANGEROUS_FLAGS = {"--dangerously-bypass-approvals-and-sandbox", "--yolo"}
|
|
23
|
+
NETWORK_ACCESS_CONFIG = "sandbox_workspace_write.network_access"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ReceiptError(ValueError):
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def reject_constant(token: str) -> None:
|
|
31
|
+
raise ValueError(f"invalid JSON numeric constant: {token}")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict:
|
|
35
|
+
result = {}
|
|
36
|
+
for key, value in pairs:
|
|
37
|
+
if key in result:
|
|
38
|
+
raise ValueError(f"duplicate JSON key: {key}")
|
|
39
|
+
result[key] = value
|
|
40
|
+
return result
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def loads_strict_json(text: str):
|
|
44
|
+
return json.loads(
|
|
45
|
+
text,
|
|
46
|
+
parse_constant=reject_constant,
|
|
47
|
+
object_pairs_hook=reject_duplicate_keys,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def sha256(raw: bytes) -> str:
|
|
52
|
+
return hashlib.sha256(raw).hexdigest()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def atomic_write(path: pathlib.Path, value: object) -> None:
|
|
56
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
57
|
+
raw = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
|
58
|
+
fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=path.name + ".tmp.")
|
|
59
|
+
try:
|
|
60
|
+
with os.fdopen(fd, "wb") as handle:
|
|
61
|
+
handle.write(raw)
|
|
62
|
+
pathlib.Path(temporary).replace(path)
|
|
63
|
+
except BaseException:
|
|
64
|
+
pathlib.Path(temporary).unlink(missing_ok=True)
|
|
65
|
+
raise
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def relative_file(work: pathlib.Path, raw_path: str, label: str, *, require: bool) -> tuple[pathlib.Path, str]:
|
|
69
|
+
path = pathlib.Path(raw_path)
|
|
70
|
+
candidate = path if path.is_absolute() else work / path
|
|
71
|
+
try:
|
|
72
|
+
if require and candidate.is_symlink():
|
|
73
|
+
raise ReceiptError(f"{label} must not be a symlink: {raw_path}")
|
|
74
|
+
resolved = candidate.resolve(strict=require)
|
|
75
|
+
relative = resolved.relative_to(work.resolve()).as_posix()
|
|
76
|
+
except (OSError, ValueError) as exc:
|
|
77
|
+
raise ReceiptError(f"{label} is missing or escapes the worktree: {raw_path}") from exc
|
|
78
|
+
if require and not resolved.is_file():
|
|
79
|
+
raise ReceiptError(f"{label} is not a regular file: {raw_path}")
|
|
80
|
+
return resolved, relative
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def option_values(argv: list[str], short: str, long: str) -> list[str]:
|
|
84
|
+
values = []
|
|
85
|
+
index = 0
|
|
86
|
+
while index < len(argv):
|
|
87
|
+
token = argv[index]
|
|
88
|
+
if token in {short, long}:
|
|
89
|
+
if index + 1 >= len(argv):
|
|
90
|
+
raise ReceiptError(f"{token} requires a value")
|
|
91
|
+
values.append(argv[index + 1])
|
|
92
|
+
index += 2
|
|
93
|
+
continue
|
|
94
|
+
if token.startswith(long + "="):
|
|
95
|
+
values.append(token.split("=", 1)[1])
|
|
96
|
+
elif token.startswith(short + "="):
|
|
97
|
+
values.append(token.split("=", 1)[1])
|
|
98
|
+
index += 1
|
|
99
|
+
return values
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def option_value(argv: list[str], short: str, long: str) -> str | None:
|
|
103
|
+
values = option_values(argv, short, long)
|
|
104
|
+
if len(values) > 1:
|
|
105
|
+
raise ReceiptError(f"duplicate {long} option")
|
|
106
|
+
return values[0] if values else None
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def sandbox_network_access(argv: list[str], phase: str) -> bool:
|
|
110
|
+
enabled = phase == "build_gate"
|
|
111
|
+
expected = f"{NETWORK_ACCESS_CONFIG}={'true' if enabled else 'false'}"
|
|
112
|
+
related = [
|
|
113
|
+
raw for raw in option_values(argv, "-c", "--config")
|
|
114
|
+
if "sandbox_workspace_write" in raw or "network_access" in raw
|
|
115
|
+
]
|
|
116
|
+
if related != [expected]:
|
|
117
|
+
raise ReceiptError(
|
|
118
|
+
f"Codex {phase} requires exactly one -c {expected}; "
|
|
119
|
+
"alternate, table, missing, and duplicate network overrides are forbidden"
|
|
120
|
+
)
|
|
121
|
+
return enabled
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def start_receipt(
|
|
125
|
+
work: pathlib.Path,
|
|
126
|
+
receipt_path: pathlib.Path,
|
|
127
|
+
run_id: str,
|
|
128
|
+
phase: str,
|
|
129
|
+
round_: int,
|
|
130
|
+
prompt_path: str,
|
|
131
|
+
session_path: str,
|
|
132
|
+
argv: list[str],
|
|
133
|
+
) -> None:
|
|
134
|
+
if receipt_path.exists():
|
|
135
|
+
raise ReceiptError(f"invocation receipt already exists: {receipt_path}")
|
|
136
|
+
if SAFE_ID_RE.fullmatch(run_id) is None:
|
|
137
|
+
raise ReceiptError("run id is invalid")
|
|
138
|
+
if phase not in PHASES:
|
|
139
|
+
raise ReceiptError(f"unsupported invocation phase: {phase}")
|
|
140
|
+
if round_ < 0:
|
|
141
|
+
raise ReceiptError("round must be non-negative")
|
|
142
|
+
evidence_manifest = (
|
|
143
|
+
work / ".devlyn" / "process-evidence" / run_id / phase
|
|
144
|
+
/ f"round-{round_}" / "manifest.json"
|
|
145
|
+
)
|
|
146
|
+
if evidence_manifest.is_file():
|
|
147
|
+
manifest = loads_strict_json(evidence_manifest.read_text(encoding="utf-8"))
|
|
148
|
+
entries = manifest.get("entries") if isinstance(manifest, dict) else None
|
|
149
|
+
if not isinstance(entries, list):
|
|
150
|
+
raise ReceiptError("existing process-evidence manifest is malformed")
|
|
151
|
+
if any(
|
|
152
|
+
isinstance(entry, dict)
|
|
153
|
+
and isinstance(entry.get("classification"), dict)
|
|
154
|
+
and entry["classification"].get("kind") == "capability_denied"
|
|
155
|
+
for entry in entries
|
|
156
|
+
):
|
|
157
|
+
raise ReceiptError(
|
|
158
|
+
"capability-denied phase/round cannot launch another Codex invocation"
|
|
159
|
+
)
|
|
160
|
+
forbidden = sorted(DANGEROUS_FLAGS.intersection(argv))
|
|
161
|
+
if forbidden:
|
|
162
|
+
raise ReceiptError(f"forbidden Codex bypass flag: {forbidden[0]}")
|
|
163
|
+
if "--json" not in argv[:-1]:
|
|
164
|
+
raise ReceiptError("receipt-bound Codex invocation requires --json before the prompt")
|
|
165
|
+
model = option_value(argv, "-m", "--model")
|
|
166
|
+
sandbox = option_value(argv, "-s", "--sandbox")
|
|
167
|
+
invocation_workdir = option_value(argv, "-C", "--cd")
|
|
168
|
+
if not model:
|
|
169
|
+
raise ReceiptError("Codex invocation requires an explicit model")
|
|
170
|
+
if not sandbox:
|
|
171
|
+
raise ReceiptError("Codex invocation requires an explicit sandbox")
|
|
172
|
+
if sandbox != "workspace-write":
|
|
173
|
+
raise ReceiptError(
|
|
174
|
+
f"Codex {phase} sandbox must remain workspace-write, got {sandbox}"
|
|
175
|
+
)
|
|
176
|
+
network_access = sandbox_network_access(argv, phase)
|
|
177
|
+
if not invocation_workdir:
|
|
178
|
+
raise ReceiptError("Codex invocation requires an explicit workdir")
|
|
179
|
+
try:
|
|
180
|
+
if pathlib.Path(invocation_workdir).resolve(strict=True) != work.resolve():
|
|
181
|
+
raise ReceiptError("Codex invocation workdir does not match receipt workdir")
|
|
182
|
+
except OSError as exc:
|
|
183
|
+
raise ReceiptError("Codex invocation workdir is missing") from exc
|
|
184
|
+
|
|
185
|
+
prompt_file, prompt_relative = relative_file(work, prompt_path, "invocation prompt", require=True)
|
|
186
|
+
session_file, session_relative = relative_file(work, session_path, "worker session", require=False)
|
|
187
|
+
receipt_file, receipt_relative = relative_file(work, str(receipt_path), "invocation receipt", require=False)
|
|
188
|
+
expected_receipt = f".devlyn/{phase}.invocation.{round_}.json"
|
|
189
|
+
expected_session = f".devlyn/{phase}.worker-session.{round_}.jsonl"
|
|
190
|
+
expected_prompt = f".devlyn/{phase}.prompt.{round_}"
|
|
191
|
+
if receipt_relative != expected_receipt:
|
|
192
|
+
raise ReceiptError(
|
|
193
|
+
f"invocation receipt path expected {expected_receipt}, got {receipt_relative}"
|
|
194
|
+
)
|
|
195
|
+
if session_relative != expected_session:
|
|
196
|
+
raise ReceiptError(
|
|
197
|
+
f"worker session path expected {expected_session}, got {session_relative}"
|
|
198
|
+
)
|
|
199
|
+
if prompt_relative != expected_prompt:
|
|
200
|
+
raise ReceiptError(
|
|
201
|
+
f"invocation prompt path expected {expected_prompt}, got {prompt_relative}"
|
|
202
|
+
)
|
|
203
|
+
prompt_raw = prompt_file.read_bytes()
|
|
204
|
+
try:
|
|
205
|
+
prompt_argument = prompt_raw.decode("utf-8").rstrip("\n")
|
|
206
|
+
except UnicodeError as exc:
|
|
207
|
+
raise ReceiptError("invocation prompt is not UTF-8") from exc
|
|
208
|
+
if not argv or argv[-1] != prompt_argument:
|
|
209
|
+
raise ReceiptError("Codex prompt argument does not match the canonical prompt file")
|
|
210
|
+
|
|
211
|
+
atomic_write(receipt_file, {
|
|
212
|
+
"schema_version": SCHEMA_VERSION,
|
|
213
|
+
"run_id": run_id,
|
|
214
|
+
"phase": phase,
|
|
215
|
+
"round": round_,
|
|
216
|
+
"engine": "codex",
|
|
217
|
+
"model": model,
|
|
218
|
+
"sandbox": sandbox,
|
|
219
|
+
"sandbox_network_access": network_access,
|
|
220
|
+
"prompt": {"path": prompt_relative, "sha256": sha256(prompt_raw)},
|
|
221
|
+
"session": {"path": session_relative, "sha256": None, "bytes": None},
|
|
222
|
+
"argv_sha256": sha256(json.dumps(argv, separators=(",", ":")).encode("utf-8")),
|
|
223
|
+
"status": "started",
|
|
224
|
+
"exit_code": None,
|
|
225
|
+
})
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def read_receipt(path: pathlib.Path) -> dict:
|
|
229
|
+
try:
|
|
230
|
+
value = loads_strict_json(path.read_text(encoding="utf-8"))
|
|
231
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
232
|
+
raise ReceiptError(f"invocation receipt is invalid: {exc}") from exc
|
|
233
|
+
if not isinstance(value, dict):
|
|
234
|
+
raise ReceiptError("invocation receipt must contain an object")
|
|
235
|
+
return value
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def finish_receipt(work: pathlib.Path, receipt_path: pathlib.Path, exit_code: int) -> None:
|
|
239
|
+
receipt = read_receipt(receipt_path)
|
|
240
|
+
if receipt.get("status") != "started" or receipt.get("exit_code") is not None:
|
|
241
|
+
raise ReceiptError("invocation receipt is not open")
|
|
242
|
+
session = receipt.get("session")
|
|
243
|
+
session_path = session.get("path") if isinstance(session, dict) else None
|
|
244
|
+
if not isinstance(session_path, str):
|
|
245
|
+
raise ReceiptError("invocation receipt session path is invalid")
|
|
246
|
+
session_file, _ = relative_file(work, session_path, "worker session", require=True)
|
|
247
|
+
raw = session_file.read_bytes()
|
|
248
|
+
receipt["session"] = {
|
|
249
|
+
"path": session_path,
|
|
250
|
+
"sha256": sha256(raw),
|
|
251
|
+
"bytes": len(raw),
|
|
252
|
+
}
|
|
253
|
+
receipt["status"] = "completed"
|
|
254
|
+
receipt["exit_code"] = exit_code
|
|
255
|
+
atomic_write(receipt_path, receipt)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def validate_receipt_artifacts(
|
|
259
|
+
work: pathlib.Path,
|
|
260
|
+
receipt_path: pathlib.Path,
|
|
261
|
+
*,
|
|
262
|
+
run_id: str,
|
|
263
|
+
phase: str,
|
|
264
|
+
) -> tuple[dict, pathlib.Path, pathlib.Path, pathlib.Path]:
|
|
265
|
+
"""Validate self-contained receipt identity and every referenced byte carrier."""
|
|
266
|
+
receipt_file, receipt_relative = relative_file(
|
|
267
|
+
work, str(receipt_path), "invocation receipt", require=True,
|
|
268
|
+
)
|
|
269
|
+
receipt = read_receipt(receipt_file)
|
|
270
|
+
if set(receipt) != {
|
|
271
|
+
"schema_version", "run_id", "phase", "round", "engine", "model",
|
|
272
|
+
"sandbox", "sandbox_network_access", "prompt", "session", "argv_sha256",
|
|
273
|
+
"status", "exit_code",
|
|
274
|
+
}:
|
|
275
|
+
raise ReceiptError("invocation receipt has an invalid shape")
|
|
276
|
+
round_ = receipt.get("round")
|
|
277
|
+
if isinstance(round_, bool) or not isinstance(round_, int) or round_ < 0:
|
|
278
|
+
raise ReceiptError("invocation receipt round is invalid")
|
|
279
|
+
expected_path = f".devlyn/{phase}.invocation.{round_}.json"
|
|
280
|
+
if receipt_relative != expected_path:
|
|
281
|
+
raise ReceiptError(f"invocation receipt path expected {expected_path}")
|
|
282
|
+
expected_identity = (SCHEMA_VERSION, run_id, phase, round_, "codex", "completed")
|
|
283
|
+
actual_identity = (
|
|
284
|
+
receipt["schema_version"], receipt["run_id"], receipt["phase"], receipt["round"],
|
|
285
|
+
receipt["engine"], receipt["status"],
|
|
286
|
+
)
|
|
287
|
+
if actual_identity != expected_identity:
|
|
288
|
+
raise ReceiptError("invocation receipt identity/status mismatch")
|
|
289
|
+
if not isinstance(receipt["model"], str) or not receipt["model"]:
|
|
290
|
+
raise ReceiptError("invocation receipt model is invalid")
|
|
291
|
+
if receipt["sandbox"] != "workspace-write":
|
|
292
|
+
raise ReceiptError("invocation receipt sandbox must be workspace-write")
|
|
293
|
+
expected_network_access = phase == "build_gate"
|
|
294
|
+
if receipt["sandbox_network_access"] is not expected_network_access:
|
|
295
|
+
raise ReceiptError("invocation receipt sandbox network-access capability mismatch")
|
|
296
|
+
if not isinstance(receipt["exit_code"], int) or isinstance(receipt["exit_code"], bool):
|
|
297
|
+
raise ReceiptError("invocation receipt exit_code is invalid")
|
|
298
|
+
if not isinstance(receipt["argv_sha256"], str) or SHA256_RE.fullmatch(receipt["argv_sha256"]) is None:
|
|
299
|
+
raise ReceiptError("invocation receipt argv digest is invalid")
|
|
300
|
+
prompt = receipt["prompt"]
|
|
301
|
+
if not isinstance(prompt, dict) or set(prompt) != {"path", "sha256"}:
|
|
302
|
+
raise ReceiptError("invocation receipt prompt binding is invalid")
|
|
303
|
+
prompt_file, _ = relative_file(work, prompt["path"], "invocation prompt", require=True)
|
|
304
|
+
if prompt["path"] != f".devlyn/{phase}.prompt.{round_}":
|
|
305
|
+
raise ReceiptError("invocation receipt prompt path mismatch")
|
|
306
|
+
if (
|
|
307
|
+
not isinstance(prompt["sha256"], str)
|
|
308
|
+
or SHA256_RE.fullmatch(prompt["sha256"]) is None
|
|
309
|
+
or sha256(prompt_file.read_bytes()) != prompt["sha256"]
|
|
310
|
+
):
|
|
311
|
+
raise ReceiptError("invocation prompt digest mismatch")
|
|
312
|
+
session = receipt["session"]
|
|
313
|
+
if not isinstance(session, dict) or set(session) != {"path", "sha256", "bytes"}:
|
|
314
|
+
raise ReceiptError("invocation receipt session binding is invalid")
|
|
315
|
+
if not isinstance(session["path"], str):
|
|
316
|
+
raise ReceiptError("invocation receipt session path is invalid")
|
|
317
|
+
session_file, relative_session = relative_file(
|
|
318
|
+
work, session["path"], "worker session", require=True,
|
|
319
|
+
)
|
|
320
|
+
if relative_session != f".devlyn/{phase}.worker-session.{round_}.jsonl":
|
|
321
|
+
raise ReceiptError("invocation receipt session path mismatch")
|
|
322
|
+
raw = session_file.read_bytes()
|
|
323
|
+
if (
|
|
324
|
+
not isinstance(session["sha256"], str)
|
|
325
|
+
or SHA256_RE.fullmatch(session["sha256"]) is None
|
|
326
|
+
or isinstance(session["bytes"], bool)
|
|
327
|
+
or not isinstance(session["bytes"], int)
|
|
328
|
+
or session["bytes"] < 0
|
|
329
|
+
or session["sha256"] != sha256(raw)
|
|
330
|
+
or session["bytes"] != len(raw)
|
|
331
|
+
):
|
|
332
|
+
raise ReceiptError("invocation worker-session digest mismatch")
|
|
333
|
+
return receipt, receipt_file, prompt_file, session_file
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def validate_receipt(
|
|
337
|
+
work: pathlib.Path,
|
|
338
|
+
receipt_path: pathlib.Path,
|
|
339
|
+
*,
|
|
340
|
+
run_id: str,
|
|
341
|
+
phase: str,
|
|
342
|
+
round_: int,
|
|
343
|
+
model: str,
|
|
344
|
+
prompt_sha256: str,
|
|
345
|
+
session_path: pathlib.Path,
|
|
346
|
+
) -> dict:
|
|
347
|
+
receipt, receipt_file, _prompt_file, receipt_session = validate_receipt_artifacts(
|
|
348
|
+
work, receipt_path, run_id=run_id, phase=phase,
|
|
349
|
+
)
|
|
350
|
+
if receipt["round"] != round_ or receipt["model"] != model:
|
|
351
|
+
raise ReceiptError("invocation receipt round/model mismatch")
|
|
352
|
+
if receipt["prompt"]["sha256"] != prompt_sha256:
|
|
353
|
+
raise ReceiptError("invocation prompt digest mismatch")
|
|
354
|
+
supplied_session, _ = relative_file(
|
|
355
|
+
work, str(session_path), "worker session", require=True,
|
|
356
|
+
)
|
|
357
|
+
if supplied_session != receipt_session:
|
|
358
|
+
raise ReceiptError("invocation receipt session path mismatch")
|
|
359
|
+
raw = receipt_session.read_bytes()
|
|
360
|
+
if sha256(raw) != receipt["session"]["sha256"]:
|
|
361
|
+
raise ReceiptError("invocation worker-session digest mismatch")
|
|
362
|
+
for number, line in enumerate(raw.decode("utf-8").split("\n"), start=1):
|
|
363
|
+
if not line.strip():
|
|
364
|
+
continue
|
|
365
|
+
try:
|
|
366
|
+
event = loads_strict_json(line)
|
|
367
|
+
except ValueError as exc:
|
|
368
|
+
raise ReceiptError(f"invalid worker-session JSONL at line {number}: {exc}") from exc
|
|
369
|
+
if not isinstance(event, dict):
|
|
370
|
+
raise ReceiptError(f"worker-session event at line {number} must be an object")
|
|
371
|
+
item = event.get("item")
|
|
372
|
+
if (event.get("type") == "item.completed" and isinstance(item, dict)
|
|
373
|
+
and item.get("type") == "error" and isinstance(item.get("message"), str)
|
|
374
|
+
and item["message"].startswith("model rerouted: ")):
|
|
375
|
+
raise ReceiptError(
|
|
376
|
+
f"Codex reported model reroute at worker-session line {number}; "
|
|
377
|
+
"requested model was not preserved; inspect the retained session"
|
|
378
|
+
)
|
|
379
|
+
return {
|
|
380
|
+
"path": receipt_file.relative_to(work.resolve()).as_posix(),
|
|
381
|
+
"sha256": sha256(receipt_file.read_bytes()),
|
|
382
|
+
"sandbox": receipt["sandbox"],
|
|
383
|
+
"sandbox_network_access": receipt["sandbox_network_access"],
|
|
384
|
+
"argv_sha256": receipt["argv_sha256"],
|
|
385
|
+
"exit_code": receipt["exit_code"],
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def monitor_descendant_regression() -> None:
|
|
390
|
+
with tempfile.TemporaryDirectory() as raw_tmp:
|
|
391
|
+
work = pathlib.Path(raw_tmp)
|
|
392
|
+
fixture_bin = work / "bin"
|
|
393
|
+
fixture_bin.mkdir()
|
|
394
|
+
release_marker = work / "release"
|
|
395
|
+
codex_pid_marker = work / "codex.pid"
|
|
396
|
+
sleep_marker = work / "sleep.tsv"
|
|
397
|
+
real_bash = shutil.which("bash")
|
|
398
|
+
real_sleep = shutil.which("sleep")
|
|
399
|
+
if real_bash is None or real_sleep is None:
|
|
400
|
+
raise AssertionError("monitor regression requires bash and sleep")
|
|
401
|
+
real_bash = str(pathlib.Path(real_bash).resolve(strict=True))
|
|
402
|
+
real_sleep = str(pathlib.Path(real_sleep).resolve(strict=True))
|
|
403
|
+
|
|
404
|
+
fake_codex = work / "fake-codex"
|
|
405
|
+
fake_codex.write_text(
|
|
406
|
+
f"#!{real_bash}\n"
|
|
407
|
+
"printf '%s\\n' \"$$\" > \"$DEVLYN_WATCHDOG_TEST_CODEX_PID\"\n"
|
|
408
|
+
"while [ ! -e \"$DEVLYN_WATCHDOG_TEST_RELEASE\" ]; do\n"
|
|
409
|
+
" \"$DEVLYN_WATCHDOG_TEST_REAL_SLEEP\" 0.01\n"
|
|
410
|
+
"done\n",
|
|
411
|
+
encoding="utf-8",
|
|
412
|
+
)
|
|
413
|
+
fake_codex.chmod(0o755)
|
|
414
|
+
sleep_shim = fixture_bin / "sleep"
|
|
415
|
+
sleep_shim.write_text(
|
|
416
|
+
f"#!{real_bash}\n"
|
|
417
|
+
"printf '%s\\t%s\\n' \"$$\" \"${1-}\" >> "
|
|
418
|
+
"\"$DEVLYN_WATCHDOG_TEST_SLEEP_MARKER\"\n"
|
|
419
|
+
"exec \"$DEVLYN_WATCHDOG_TEST_REAL_SLEEP\" \"$@\"\n",
|
|
420
|
+
encoding="utf-8",
|
|
421
|
+
)
|
|
422
|
+
sleep_shim.chmod(0o755)
|
|
423
|
+
|
|
424
|
+
wrapper = pathlib.Path(__file__).resolve().with_name("codex-monitored.sh")
|
|
425
|
+
env = os.environ.copy()
|
|
426
|
+
for key in tuple(env):
|
|
427
|
+
if (
|
|
428
|
+
key.startswith("DEVLYN_INVOCATION_")
|
|
429
|
+
or key.startswith("DEVLYN_WATCHDOG_TEST_")
|
|
430
|
+
or key.startswith("CODEX_MONITORED_")
|
|
431
|
+
or key in {"CODEX_BLOCKED", "CODEX_REAL_BIN"}
|
|
432
|
+
):
|
|
433
|
+
env.pop(key)
|
|
434
|
+
env.update({
|
|
435
|
+
"CODEX_BIN": str(fake_codex),
|
|
436
|
+
"CODEX_MONITORED_HEARTBEAT": "1",
|
|
437
|
+
"CODEX_MONITORED_TIMEOUT_SEC": "30",
|
|
438
|
+
"DEVLYN_WATCHDOG_TEST_CODEX_PID": str(codex_pid_marker),
|
|
439
|
+
"DEVLYN_WATCHDOG_TEST_RELEASE": str(release_marker),
|
|
440
|
+
"DEVLYN_WATCHDOG_TEST_SLEEP_MARKER": str(sleep_marker),
|
|
441
|
+
"DEVLYN_WATCHDOG_TEST_REAL_SLEEP": real_sleep,
|
|
442
|
+
"PATH": str(fixture_bin) + os.pathsep + env.get("PATH", ""),
|
|
443
|
+
})
|
|
444
|
+
|
|
445
|
+
wrapped: subprocess.Popen | None = None
|
|
446
|
+
fixture_groups: dict[int, set[int]] = {}
|
|
447
|
+
wrapper_pgid: int | None = None
|
|
448
|
+
|
|
449
|
+
def record_owned_process(pid: int, session_id: int) -> int | None:
|
|
450
|
+
try:
|
|
451
|
+
process_session = os.getsid(pid)
|
|
452
|
+
process_group = os.getpgid(pid)
|
|
453
|
+
except ProcessLookupError:
|
|
454
|
+
return None
|
|
455
|
+
except PermissionError as exc:
|
|
456
|
+
raise AssertionError(f"cannot inspect fixture pid {pid}: {exc}") from exc
|
|
457
|
+
if process_session != session_id:
|
|
458
|
+
raise AssertionError(
|
|
459
|
+
f"fixture pid {pid} escaped session {session_id}: {process_session}"
|
|
460
|
+
)
|
|
461
|
+
fixture_groups.setdefault(process_group, set()).update({pid, process_group})
|
|
462
|
+
return process_group
|
|
463
|
+
|
|
464
|
+
def read_announced_groups(session_id: int) -> tuple[
|
|
465
|
+
tuple[int, int] | None, dict[str, tuple[int, int]]
|
|
466
|
+
]:
|
|
467
|
+
codex_info = None
|
|
468
|
+
if codex_pid_marker.is_file():
|
|
469
|
+
raw_pid = codex_pid_marker.read_text(encoding="utf-8")
|
|
470
|
+
if raw_pid.endswith("\n") and re.fullmatch(r"[1-9][0-9]*\n", raw_pid):
|
|
471
|
+
codex_pid = int(raw_pid.strip())
|
|
472
|
+
codex_pgid = record_owned_process(codex_pid, session_id)
|
|
473
|
+
if codex_pgid is not None:
|
|
474
|
+
codex_info = (codex_pid, codex_pgid)
|
|
475
|
+
elif raw_pid.endswith("\n"):
|
|
476
|
+
raise AssertionError(f"invalid fake Codex PID marker: {raw_pid!r}")
|
|
477
|
+
|
|
478
|
+
sleeps: dict[str, tuple[int, int]] = {}
|
|
479
|
+
if sleep_marker.is_file():
|
|
480
|
+
raw_sleeps = sleep_marker.read_text(encoding="utf-8")
|
|
481
|
+
complete = raw_sleeps if raw_sleeps.endswith("\n") else raw_sleeps.rsplit("\n", 1)[0]
|
|
482
|
+
for line in complete.splitlines():
|
|
483
|
+
match = re.fullmatch(r"([1-9][0-9]*)\t([^\t]+)", line)
|
|
484
|
+
if match is None:
|
|
485
|
+
raise AssertionError(f"invalid monitor sleep marker: {line!r}")
|
|
486
|
+
sleep_pid = int(match.group(1))
|
|
487
|
+
sleep_pgid = record_owned_process(sleep_pid, session_id)
|
|
488
|
+
if sleep_pgid is not None:
|
|
489
|
+
sleeps[match.group(2)] = (sleep_pid, sleep_pgid)
|
|
490
|
+
return codex_info, sleeps
|
|
491
|
+
|
|
492
|
+
def owned_group_alive(pgid: int, session_id: int) -> bool:
|
|
493
|
+
for pid in fixture_groups.get(pgid, ()):
|
|
494
|
+
try:
|
|
495
|
+
if os.getsid(pid) == session_id and os.getpgid(pid) == pgid:
|
|
496
|
+
return True
|
|
497
|
+
except ProcessLookupError:
|
|
498
|
+
continue
|
|
499
|
+
except PermissionError as exc:
|
|
500
|
+
raise AssertionError(
|
|
501
|
+
f"cannot inspect fixture process group {pgid}: {exc}"
|
|
502
|
+
) from exc
|
|
503
|
+
return False
|
|
504
|
+
|
|
505
|
+
def cleanup_fixture(session_id: int) -> list[str]:
|
|
506
|
+
failures: list[str] = []
|
|
507
|
+
if wrapped is None:
|
|
508
|
+
return failures
|
|
509
|
+
try:
|
|
510
|
+
read_announced_groups(session_id)
|
|
511
|
+
except (OSError, UnicodeError, AssertionError) as exc:
|
|
512
|
+
failures.append(f"announcement discovery failed: {exc}")
|
|
513
|
+
|
|
514
|
+
def live_groups() -> list[int]:
|
|
515
|
+
result = []
|
|
516
|
+
for pgid in sorted(fixture_groups):
|
|
517
|
+
if pgid == os.getpgrp():
|
|
518
|
+
failures.append(f"refused to signal current process group {pgid}")
|
|
519
|
+
continue
|
|
520
|
+
try:
|
|
521
|
+
if owned_group_alive(pgid, session_id):
|
|
522
|
+
result.append(pgid)
|
|
523
|
+
except AssertionError as exc:
|
|
524
|
+
failures.append(str(exc))
|
|
525
|
+
return result
|
|
526
|
+
|
|
527
|
+
for pgid in live_groups():
|
|
528
|
+
try:
|
|
529
|
+
os.killpg(pgid, signal.SIGTERM)
|
|
530
|
+
except ProcessLookupError:
|
|
531
|
+
pass
|
|
532
|
+
except PermissionError as exc:
|
|
533
|
+
failures.append(f"TERM fixture process group {pgid} failed: {exc}")
|
|
534
|
+
|
|
535
|
+
term_deadline = time.monotonic() + 1.0
|
|
536
|
+
survivors = live_groups()
|
|
537
|
+
while survivors and time.monotonic() < term_deadline:
|
|
538
|
+
time.sleep(0.01)
|
|
539
|
+
survivors = live_groups()
|
|
540
|
+
for pgid in survivors:
|
|
541
|
+
try:
|
|
542
|
+
os.killpg(pgid, signal.SIGKILL)
|
|
543
|
+
except ProcessLookupError:
|
|
544
|
+
pass
|
|
545
|
+
except PermissionError as exc:
|
|
546
|
+
failures.append(f"KILL fixture process group {pgid} failed: {exc}")
|
|
547
|
+
|
|
548
|
+
kill_deadline = time.monotonic() + 1.0
|
|
549
|
+
survivors = live_groups()
|
|
550
|
+
while survivors and time.monotonic() < kill_deadline:
|
|
551
|
+
time.sleep(0.01)
|
|
552
|
+
survivors = live_groups()
|
|
553
|
+
if survivors:
|
|
554
|
+
failures.append(f"fixture process groups survived cleanup: {survivors}")
|
|
555
|
+
|
|
556
|
+
try:
|
|
557
|
+
wrapped.wait(timeout=1)
|
|
558
|
+
except subprocess.TimeoutExpired:
|
|
559
|
+
failures.append("wrapper wait exceeded cleanup deadline")
|
|
560
|
+
try:
|
|
561
|
+
wrapped.kill()
|
|
562
|
+
except ProcessLookupError:
|
|
563
|
+
pass
|
|
564
|
+
except PermissionError as exc:
|
|
565
|
+
failures.append(f"direct wrapper KILL failed: {exc}")
|
|
566
|
+
try:
|
|
567
|
+
wrapped.wait(timeout=1)
|
|
568
|
+
except subprocess.TimeoutExpired:
|
|
569
|
+
failures.append("wrapper remained alive after direct KILL")
|
|
570
|
+
|
|
571
|
+
try:
|
|
572
|
+
wrapped.communicate(timeout=1)
|
|
573
|
+
except subprocess.TimeoutExpired:
|
|
574
|
+
failures.append("captured stderr did not reach EOF during cleanup")
|
|
575
|
+
if wrapped.stderr is not None:
|
|
576
|
+
wrapped.stderr.close()
|
|
577
|
+
return failures
|
|
578
|
+
|
|
579
|
+
try:
|
|
580
|
+
wrapped = subprocess.Popen(
|
|
581
|
+
[
|
|
582
|
+
real_bash, str(wrapper), "-C", str(work), "-s", "workspace-write",
|
|
583
|
+
"-m", "gpt-wrapper", "verify monitor cleanup",
|
|
584
|
+
],
|
|
585
|
+
cwd=work,
|
|
586
|
+
env=env,
|
|
587
|
+
stdout=subprocess.DEVNULL,
|
|
588
|
+
stderr=subprocess.PIPE,
|
|
589
|
+
start_new_session=True,
|
|
590
|
+
)
|
|
591
|
+
session_id = wrapped.pid
|
|
592
|
+
wrapper_pgid = record_owned_process(wrapped.pid, session_id)
|
|
593
|
+
if wrapper_pgid != wrapped.pid:
|
|
594
|
+
raise AssertionError(
|
|
595
|
+
f"wrapper did not own its fixture session/process group: {wrapper_pgid}"
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
setup_deadline = time.monotonic() + 5.0
|
|
599
|
+
ready = None
|
|
600
|
+
while time.monotonic() < setup_deadline:
|
|
601
|
+
codex_info, sleeps = read_announced_groups(session_id)
|
|
602
|
+
heartbeat = sleeps.get("1")
|
|
603
|
+
watchdog = sleeps.get("30")
|
|
604
|
+
if codex_info is not None and heartbeat is not None and watchdog is not None:
|
|
605
|
+
groups = {
|
|
606
|
+
wrapper_pgid,
|
|
607
|
+
codex_info[1],
|
|
608
|
+
heartbeat[1],
|
|
609
|
+
watchdog[1],
|
|
610
|
+
}
|
|
611
|
+
if len(groups) != 4:
|
|
612
|
+
raise AssertionError(
|
|
613
|
+
"wrapper, Codex, heartbeat, and watchdog did not own distinct groups: "
|
|
614
|
+
f"{sorted(groups)}"
|
|
615
|
+
)
|
|
616
|
+
if all(owned_group_alive(pgid, session_id) for pgid in groups):
|
|
617
|
+
ready = (codex_info, heartbeat, watchdog)
|
|
618
|
+
break
|
|
619
|
+
if wrapped.poll() is not None:
|
|
620
|
+
raise AssertionError(
|
|
621
|
+
f"wrapper exited during monitor setup with code {wrapped.returncode}"
|
|
622
|
+
)
|
|
623
|
+
time.sleep(0.01)
|
|
624
|
+
if ready is None:
|
|
625
|
+
sleep_detail = sleep_marker.read_text(encoding="utf-8") if sleep_marker.exists() else ""
|
|
626
|
+
raise AssertionError(
|
|
627
|
+
"monitor setup did not announce live Codex/heartbeat=1/watchdog=30 "
|
|
628
|
+
f"within 5 seconds: {sleep_detail!r}"
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
release_marker.write_bytes(b"release\n")
|
|
632
|
+
try:
|
|
633
|
+
_stdout, stderr = wrapped.communicate(timeout=5)
|
|
634
|
+
except subprocess.TimeoutExpired as exc:
|
|
635
|
+
raise AssertionError(
|
|
636
|
+
"wrapper completion or captured stderr EOF exceeded 5 seconds "
|
|
637
|
+
f"after release (returncode={wrapped.poll()})"
|
|
638
|
+
) from exc
|
|
639
|
+
if wrapped.returncode != 0:
|
|
640
|
+
raise AssertionError(stderr.decode("utf-8", errors="replace"))
|
|
641
|
+
read_announced_groups(session_id)
|
|
642
|
+
survivors = [
|
|
643
|
+
pgid
|
|
644
|
+
for pgid in sorted(fixture_groups)
|
|
645
|
+
if owned_group_alive(pgid, session_id)
|
|
646
|
+
]
|
|
647
|
+
if survivors:
|
|
648
|
+
raise AssertionError(
|
|
649
|
+
f"wrapper returned while fixture process groups remained alive: {survivors}"
|
|
650
|
+
)
|
|
651
|
+
except BaseException as exc:
|
|
652
|
+
session_id = wrapped.pid if wrapped is not None else -1
|
|
653
|
+
cleanup_failures = cleanup_fixture(session_id)
|
|
654
|
+
if cleanup_failures:
|
|
655
|
+
raise AssertionError(
|
|
656
|
+
f"monitor regression failed ({exc}); cleanup failed: "
|
|
657
|
+
+ "; ".join(cleanup_failures)
|
|
658
|
+
) from exc
|
|
659
|
+
raise
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def self_test() -> int:
|
|
663
|
+
monitor_descendant_regression()
|
|
664
|
+
with tempfile.TemporaryDirectory() as raw_tmp:
|
|
665
|
+
work = pathlib.Path(raw_tmp)
|
|
666
|
+
devlyn = work / ".devlyn"
|
|
667
|
+
devlyn.mkdir()
|
|
668
|
+
prompt = devlyn / "implement.prompt.0"
|
|
669
|
+
prompt.write_text("do the task\n", encoding="utf-8")
|
|
670
|
+
session = devlyn / "implement.worker-session.0.jsonl"
|
|
671
|
+
session.write_text('{"type":"thread.started"}\n', encoding="utf-8")
|
|
672
|
+
receipt = devlyn / "implement.invocation.0.json"
|
|
673
|
+
argv = [
|
|
674
|
+
"--json", "-C", str(work), "-s", "workspace-write", "-m", "gpt-test",
|
|
675
|
+
"-c", "sandbox_workspace_write.network_access=false", "do the task",
|
|
676
|
+
]
|
|
677
|
+
start_receipt(work, receipt, "rs-receipt", "implement", 0, str(prompt), str(session), argv)
|
|
678
|
+
finish_receipt(work, receipt, 0)
|
|
679
|
+
bound = validate_receipt(
|
|
680
|
+
work, receipt, run_id="rs-receipt", phase="implement", round_=0,
|
|
681
|
+
model="gpt-test", prompt_sha256=sha256(prompt.read_bytes()), session_path=session,
|
|
682
|
+
)
|
|
683
|
+
assert bound["sandbox"] == "workspace-write" and bound["exit_code"] == 0
|
|
684
|
+
original_session = session.read_bytes()
|
|
685
|
+
original_receipt = receipt.read_bytes()
|
|
686
|
+
reroute = "model rerouted: gpt-test -> another-model (Policy)"
|
|
687
|
+
cases = [
|
|
688
|
+
({"type": "item.completed", "item": {"type": "error", "message": reroute}}, True),
|
|
689
|
+
({"type": "item.completed", "item": {"type": "agent_message", "text": reroute}}, False),
|
|
690
|
+
({"type": "item.completed", "item": {"type": "command_execution", "aggregated_output": reroute}}, False),
|
|
691
|
+
({"type": "item.completed", "item": {"type": "error", "message": "recoverable tool error"}}, False),
|
|
692
|
+
]
|
|
693
|
+
cases.append(({"type": "item.completed", "item": {
|
|
694
|
+
"type": "agent_message", "text": "ordinary \u2028 \u2029 \u0085 message",
|
|
695
|
+
}}, False))
|
|
696
|
+
for event, blocked in cases:
|
|
697
|
+
raw = original_session + (json.dumps(event, ensure_ascii=False) + '\n{"type":"turn.completed"}\n').encode()
|
|
698
|
+
session.write_bytes(raw)
|
|
699
|
+
receipt.unlink()
|
|
700
|
+
start_receipt(work, receipt, "rs-receipt", "implement", 0, str(prompt), str(session), argv)
|
|
701
|
+
finish_receipt(work, receipt, 0)
|
|
702
|
+
sealed = receipt.read_bytes()
|
|
703
|
+
try:
|
|
704
|
+
validate_receipt(work, receipt, run_id="rs-receipt", phase="implement", round_=0,
|
|
705
|
+
model="gpt-test", prompt_sha256=sha256(prompt.read_bytes()), session_path=session)
|
|
706
|
+
except ReceiptError as exc:
|
|
707
|
+
assert blocked and "reported model reroute" in str(exc), str(exc)
|
|
708
|
+
else:
|
|
709
|
+
assert not blocked, "native reroute followed by success was accepted"
|
|
710
|
+
validate_receipt_artifacts(work, receipt, run_id="rs-receipt", phase="implement")
|
|
711
|
+
assert receipt.read_bytes() == sealed and session.read_bytes() == raw
|
|
712
|
+
for malformed in (b'{broken\n', b'null\n', b'{"type":"item.completed","type":"ignored"}\n'):
|
|
713
|
+
session.write_bytes(malformed)
|
|
714
|
+
receipt.unlink()
|
|
715
|
+
start_receipt(work, receipt, "rs-receipt", "implement", 0, str(prompt), str(session), argv)
|
|
716
|
+
finish_receipt(work, receipt, 0)
|
|
717
|
+
try:
|
|
718
|
+
validate_receipt(work, receipt, run_id="rs-receipt", phase="implement", round_=0,
|
|
719
|
+
model="gpt-test", prompt_sha256=sha256(prompt.read_bytes()), session_path=session)
|
|
720
|
+
except ReceiptError:
|
|
721
|
+
pass
|
|
722
|
+
else:
|
|
723
|
+
raise AssertionError("malformed worker event was accepted")
|
|
724
|
+
validate_receipt_artifacts(work, receipt, run_id="rs-receipt", phase="implement")
|
|
725
|
+
session.write_bytes(original_session)
|
|
726
|
+
receipt.write_bytes(original_receipt)
|
|
727
|
+
plan_prompt = devlyn / "plan.prompt.0"
|
|
728
|
+
plan_prompt.write_text("plan exactly\n", encoding="utf-8")
|
|
729
|
+
plan_session = devlyn / "plan.worker-session.0.jsonl"
|
|
730
|
+
plan_session.write_text('{"type":"thread.started"}\n', encoding="utf-8")
|
|
731
|
+
plan_receipt = devlyn / "plan.invocation.0.json"
|
|
732
|
+
plan_argv = [
|
|
733
|
+
"--json", "-C", str(work), "-s", "workspace-write",
|
|
734
|
+
"-m", "gpt-plan", "-c",
|
|
735
|
+
"sandbox_workspace_write.network_access=false", "plan exactly",
|
|
736
|
+
]
|
|
737
|
+
start_receipt(
|
|
738
|
+
work, plan_receipt, "rs-receipt", "plan", 0,
|
|
739
|
+
str(plan_prompt), str(plan_session), plan_argv,
|
|
740
|
+
)
|
|
741
|
+
finish_receipt(work, plan_receipt, 0)
|
|
742
|
+
plan_bound = validate_receipt(
|
|
743
|
+
work, plan_receipt, run_id="rs-receipt", phase="plan", round_=0,
|
|
744
|
+
model="gpt-plan", prompt_sha256=sha256(plan_prompt.read_bytes()),
|
|
745
|
+
session_path=plan_session,
|
|
746
|
+
)
|
|
747
|
+
assert plan_bound["sandbox"] == "workspace-write"
|
|
748
|
+
try:
|
|
749
|
+
loads_strict_json('{"run_id":"a","run_id":"b"}')
|
|
750
|
+
except ValueError as exc:
|
|
751
|
+
assert "duplicate JSON key" in str(exc)
|
|
752
|
+
else:
|
|
753
|
+
raise AssertionError("duplicate invocation receipt key was accepted")
|
|
754
|
+
try:
|
|
755
|
+
start_receipt(
|
|
756
|
+
work, devlyn / "cleanup.invocation.0.json", "rs-receipt", "cleanup", 0,
|
|
757
|
+
str(prompt), str(devlyn / "cleanup.worker-session.0.jsonl"),
|
|
758
|
+
["--json", "--dangerously-bypass-approvals-and-sandbox", "-s", "danger-full-access",
|
|
759
|
+
"-m", "gpt-test", "do the task"],
|
|
760
|
+
)
|
|
761
|
+
except ReceiptError as exc:
|
|
762
|
+
assert "forbidden Codex bypass flag" in str(exc)
|
|
763
|
+
else:
|
|
764
|
+
raise AssertionError("Codex bypass flag was accepted")
|
|
765
|
+
cleanup_prompt = devlyn / "cleanup.prompt.0"
|
|
766
|
+
cleanup_prompt.write_text("do the task\n", encoding="utf-8")
|
|
767
|
+
try:
|
|
768
|
+
start_receipt(
|
|
769
|
+
work, devlyn / "cleanup.invocation.0.json", "rs-receipt", "cleanup", 0,
|
|
770
|
+
str(cleanup_prompt), str(devlyn / "cleanup.worker-session.0.jsonl"),
|
|
771
|
+
["--json", "-C", str(work), "-s", "workspace-write", "-m", "gpt-test",
|
|
772
|
+
"do the task"],
|
|
773
|
+
)
|
|
774
|
+
except ReceiptError as exc:
|
|
775
|
+
assert "requires exactly one -c sandbox_workspace_write.network_access=false" in str(exc)
|
|
776
|
+
else:
|
|
777
|
+
raise AssertionError("Codex cleanup accepted implicit network capability")
|
|
778
|
+
try:
|
|
779
|
+
start_receipt(
|
|
780
|
+
work, devlyn / "cleanup.invocation.0.json", "rs-receipt", "cleanup", 0,
|
|
781
|
+
str(cleanup_prompt), str(devlyn / "cleanup.worker-session.0.jsonl"),
|
|
782
|
+
["--json", "-C", str(work), "-s", "workspace-write", "-m", "gpt-test",
|
|
783
|
+
"-c", "sandbox_workspace_write.network_access=true", "do the task"],
|
|
784
|
+
)
|
|
785
|
+
except ReceiptError as exc:
|
|
786
|
+
assert "requires exactly one -c sandbox_workspace_write.network_access=false" in str(exc)
|
|
787
|
+
else:
|
|
788
|
+
raise AssertionError("Codex cleanup accepted enabled network capability")
|
|
789
|
+
try:
|
|
790
|
+
start_receipt(
|
|
791
|
+
work, devlyn / "cleanup.invocation.0.json", "rs-receipt", "cleanup", 0,
|
|
792
|
+
str(cleanup_prompt), str(devlyn / "cleanup.worker-session.0.jsonl"),
|
|
793
|
+
["--json", "-s", "danger-full-access", "-m", "gpt-test", "do the task"],
|
|
794
|
+
)
|
|
795
|
+
except ReceiptError as exc:
|
|
796
|
+
assert "must remain workspace-write" in str(exc)
|
|
797
|
+
else:
|
|
798
|
+
raise AssertionError("widened Codex sandbox was accepted")
|
|
799
|
+
try:
|
|
800
|
+
start_receipt(
|
|
801
|
+
work, devlyn / "cleanup.invocation.0.json", "rs-receipt", "cleanup", 0,
|
|
802
|
+
str(cleanup_prompt), str(devlyn / "cleanup.worker-session.0.jsonl"),
|
|
803
|
+
["--json", "-C", str(devlyn), "-s", "workspace-write", "-m", "gpt-test",
|
|
804
|
+
"-c", "sandbox_workspace_write.network_access=false", "do the task"],
|
|
805
|
+
)
|
|
806
|
+
except ReceiptError as exc:
|
|
807
|
+
assert "workdir does not match" in str(exc)
|
|
808
|
+
else:
|
|
809
|
+
raise AssertionError("Codex invocation accepted a different working directory")
|
|
810
|
+
session.write_text('{"type":"thread.started","mutated":true}\n', encoding="utf-8")
|
|
811
|
+
try:
|
|
812
|
+
validate_receipt(
|
|
813
|
+
work, receipt, run_id="rs-receipt", phase="implement", round_=0,
|
|
814
|
+
model="gpt-test", prompt_sha256=sha256(prompt.read_bytes()), session_path=session,
|
|
815
|
+
)
|
|
816
|
+
except ReceiptError as exc:
|
|
817
|
+
assert "worker-session digest mismatch" in str(exc)
|
|
818
|
+
else:
|
|
819
|
+
raise AssertionError("mutated worker session was accepted")
|
|
820
|
+
|
|
821
|
+
build_prompt = devlyn / "build_gate.prompt.0"
|
|
822
|
+
build_prompt.write_text("verify the task\n", encoding="utf-8")
|
|
823
|
+
build_session = devlyn / "build_gate.worker-session.0.jsonl"
|
|
824
|
+
build_receipt = devlyn / "build_gate.invocation.0.json"
|
|
825
|
+
try:
|
|
826
|
+
start_receipt(
|
|
827
|
+
work, build_receipt, "rs-wrapper", "build_gate", 0,
|
|
828
|
+
str(build_prompt), str(build_session),
|
|
829
|
+
["--json", "-C", str(work), "-s", "workspace-write", "-m", "gpt-wrapper",
|
|
830
|
+
"verify the task"],
|
|
831
|
+
)
|
|
832
|
+
except ReceiptError as exc:
|
|
833
|
+
assert "requires exactly one -c sandbox_workspace_write.network_access=true" in str(exc)
|
|
834
|
+
else:
|
|
835
|
+
raise AssertionError("Codex build_gate accepted missing network capability")
|
|
836
|
+
try:
|
|
837
|
+
start_receipt(
|
|
838
|
+
work, build_receipt, "rs-wrapper", "build_gate", 0,
|
|
839
|
+
str(build_prompt), str(build_session),
|
|
840
|
+
["--json", "-C", str(work), "-s", "workspace-write", "-m", "gpt-wrapper",
|
|
841
|
+
"-c", "sandbox_workspace_write.network_access=true",
|
|
842
|
+
"-c", "sandbox_workspace_write.network_access=false",
|
|
843
|
+
"verify the task"],
|
|
844
|
+
)
|
|
845
|
+
except ReceiptError as exc:
|
|
846
|
+
assert "duplicate network overrides are forbidden" in str(exc)
|
|
847
|
+
else:
|
|
848
|
+
raise AssertionError("Codex build_gate accepted duplicate network capabilities")
|
|
849
|
+
for alternate in (
|
|
850
|
+
"sandbox_workspace_write={network_access=true}",
|
|
851
|
+
'"sandbox_workspace_write".network_access=true',
|
|
852
|
+
):
|
|
853
|
+
try:
|
|
854
|
+
start_receipt(
|
|
855
|
+
work, build_receipt, "rs-wrapper", "build_gate", 0,
|
|
856
|
+
str(build_prompt), str(build_session),
|
|
857
|
+
["--json", "-C", str(work), "-s", "workspace-write", "-m", "gpt-wrapper",
|
|
858
|
+
"-c", "sandbox_workspace_write.network_access=true",
|
|
859
|
+
"--config=" + alternate, "verify the task"],
|
|
860
|
+
)
|
|
861
|
+
except ReceiptError as exc:
|
|
862
|
+
assert "alternate, table, missing, and duplicate" in str(exc)
|
|
863
|
+
else:
|
|
864
|
+
raise AssertionError(f"Codex build_gate accepted alternate override: {alternate}")
|
|
865
|
+
glued_prompt = devlyn / "build_gate.prompt.1"
|
|
866
|
+
glued_prompt.write_text("verify glued config\n", encoding="utf-8")
|
|
867
|
+
glued_session = devlyn / "build_gate.worker-session.1.jsonl"
|
|
868
|
+
glued_session.write_text('{"type":"thread.started"}\n', encoding="utf-8")
|
|
869
|
+
glued_receipt = devlyn / "build_gate.invocation.1.json"
|
|
870
|
+
start_receipt(
|
|
871
|
+
work, glued_receipt, "rs-glued", "build_gate", 1,
|
|
872
|
+
str(glued_prompt), str(glued_session),
|
|
873
|
+
["--json", "-C", str(work), "-s", "workspace-write", "-m", "gpt-wrapper",
|
|
874
|
+
"-c=sandbox_workspace_write.network_access=true", "verify glued config"],
|
|
875
|
+
)
|
|
876
|
+
finish_receipt(work, glued_receipt, 0)
|
|
877
|
+
assert validate_receipt(
|
|
878
|
+
work, glued_receipt, run_id="rs-glued", phase="build_gate", round_=1,
|
|
879
|
+
model="gpt-wrapper", prompt_sha256=sha256(glued_prompt.read_bytes()),
|
|
880
|
+
session_path=glued_session,
|
|
881
|
+
)["sandbox_network_access"] is True
|
|
882
|
+
fake_codex = work / "fake-codex"
|
|
883
|
+
fake_codex.write_text(
|
|
884
|
+
"#!/usr/bin/env bash\n"
|
|
885
|
+
"touch native-started\n"
|
|
886
|
+
"printf '%s\\n' '{\"type\":\"thread.started\"}'\n",
|
|
887
|
+
encoding="utf-8",
|
|
888
|
+
)
|
|
889
|
+
fake_codex.chmod(0o755)
|
|
890
|
+
wrapper = pathlib.Path(__file__).with_name("codex-monitored.sh")
|
|
891
|
+
widened = subprocess.run(
|
|
892
|
+
[
|
|
893
|
+
"bash", str(wrapper), "-C", str(work), "-s", "danger-full-access",
|
|
894
|
+
"-m", "gpt-wrapper", "verify the task",
|
|
895
|
+
],
|
|
896
|
+
cwd=work,
|
|
897
|
+
env={**os.environ, "CODEX_BIN": str(fake_codex)},
|
|
898
|
+
stdout=subprocess.PIPE,
|
|
899
|
+
stderr=subprocess.PIPE,
|
|
900
|
+
check=False,
|
|
901
|
+
)
|
|
902
|
+
assert widened.returncode == 64
|
|
903
|
+
assert b"forbidden Codex sandbox" in widened.stderr
|
|
904
|
+
assert widened.stdout == b""
|
|
905
|
+
env = os.environ.copy()
|
|
906
|
+
env.update({
|
|
907
|
+
"CODEX_BIN": str(fake_codex),
|
|
908
|
+
"CODEX_MONITORED_HEARTBEAT": "1",
|
|
909
|
+
"DEVLYN_INVOCATION_RUN_ID": "rs-wrapper",
|
|
910
|
+
"DEVLYN_INVOCATION_PHASE": "build_gate",
|
|
911
|
+
"DEVLYN_INVOCATION_ROUND": "0",
|
|
912
|
+
"DEVLYN_INVOCATION_WORKDIR": str(work),
|
|
913
|
+
"DEVLYN_INVOCATION_PROMPT_FILE": str(build_prompt),
|
|
914
|
+
"DEVLYN_INVOCATION_SESSION_FILE": str(build_session),
|
|
915
|
+
"DEVLYN_INVOCATION_RECEIPT": str(build_receipt),
|
|
916
|
+
})
|
|
917
|
+
with build_session.open("wb") as stdout:
|
|
918
|
+
missing_json = subprocess.run(
|
|
919
|
+
["bash", str(wrapper), "-C", str(work), "-s", "workspace-write",
|
|
920
|
+
"-m", "gpt-wrapper", "-c",
|
|
921
|
+
"sandbox_workspace_write.network_access=true", "verify the task"],
|
|
922
|
+
cwd=work, env=env, stdout=stdout, stderr=subprocess.PIPE,
|
|
923
|
+
check=False,
|
|
924
|
+
)
|
|
925
|
+
assert missing_json.returncode == 64 and b"requires --json" in missing_json.stderr, missing_json.stderr
|
|
926
|
+
assert build_session.read_bytes() == b"" and not (work / "native-started").exists()
|
|
927
|
+
assert not build_receipt.exists()
|
|
928
|
+
with build_session.open("wb") as stdout:
|
|
929
|
+
wrapped = subprocess.run(
|
|
930
|
+
[
|
|
931
|
+
"bash", str(wrapper), "--json", "-C", str(work), "-s", "workspace-write",
|
|
932
|
+
"-m", "gpt-wrapper", "-c",
|
|
933
|
+
"sandbox_workspace_write.network_access=true", "verify the task",
|
|
934
|
+
],
|
|
935
|
+
cwd=work,
|
|
936
|
+
env=env,
|
|
937
|
+
stdout=stdout,
|
|
938
|
+
stderr=subprocess.PIPE,
|
|
939
|
+
check=False,
|
|
940
|
+
)
|
|
941
|
+
assert wrapped.returncode == 0, wrapped.stderr.decode("utf-8", errors="replace")
|
|
942
|
+
wrapper_bound = validate_receipt(
|
|
943
|
+
work, build_receipt, run_id="rs-wrapper", phase="build_gate", round_=0,
|
|
944
|
+
model="gpt-wrapper", prompt_sha256=sha256(build_prompt.read_bytes()),
|
|
945
|
+
session_path=build_session,
|
|
946
|
+
)
|
|
947
|
+
assert wrapper_bound["exit_code"] == 0
|
|
948
|
+
assert wrapper_bound["sandbox_network_access"] is True
|
|
949
|
+
|
|
950
|
+
wrapped_plan_prompt = devlyn / "plan.prompt.1"
|
|
951
|
+
wrapped_plan_prompt.write_text("plan through wrapper\n", encoding="utf-8")
|
|
952
|
+
wrapped_plan_session = devlyn / "plan.worker-session.1.jsonl"
|
|
953
|
+
wrapped_plan_receipt = devlyn / "plan.invocation.1.json"
|
|
954
|
+
plan_env = os.environ.copy()
|
|
955
|
+
plan_env.update({
|
|
956
|
+
"CODEX_BIN": str(fake_codex),
|
|
957
|
+
"CODEX_MONITORED_HEARTBEAT": "1",
|
|
958
|
+
"DEVLYN_INVOCATION_RUN_ID": "rs-plan-wrapper",
|
|
959
|
+
"DEVLYN_INVOCATION_PHASE": "plan",
|
|
960
|
+
"DEVLYN_INVOCATION_ROUND": "1",
|
|
961
|
+
"DEVLYN_INVOCATION_WORKDIR": str(work),
|
|
962
|
+
"DEVLYN_INVOCATION_PROMPT_FILE": str(wrapped_plan_prompt),
|
|
963
|
+
"DEVLYN_INVOCATION_SESSION_FILE": str(wrapped_plan_session),
|
|
964
|
+
"DEVLYN_INVOCATION_RECEIPT": str(wrapped_plan_receipt),
|
|
965
|
+
})
|
|
966
|
+
with wrapped_plan_session.open("wb") as stdout:
|
|
967
|
+
wrapped_plan = subprocess.run(
|
|
968
|
+
[
|
|
969
|
+
"bash", str(wrapper), "--json", "-C", str(work),
|
|
970
|
+
"-s", "workspace-write", "-m", "gpt-plan-wrapper",
|
|
971
|
+
"-c", "sandbox_workspace_write.network_access=false",
|
|
972
|
+
"plan through wrapper",
|
|
973
|
+
],
|
|
974
|
+
cwd=work,
|
|
975
|
+
env=plan_env,
|
|
976
|
+
stdout=stdout,
|
|
977
|
+
stderr=subprocess.PIPE,
|
|
978
|
+
check=False,
|
|
979
|
+
)
|
|
980
|
+
assert wrapped_plan.returncode == 0, wrapped_plan.stderr.decode(
|
|
981
|
+
"utf-8", errors="replace",
|
|
982
|
+
)
|
|
983
|
+
plan_wrapper_bound = validate_receipt(
|
|
984
|
+
work, wrapped_plan_receipt, run_id="rs-plan-wrapper", phase="plan",
|
|
985
|
+
round_=1, model="gpt-plan-wrapper",
|
|
986
|
+
prompt_sha256=sha256(wrapped_plan_prompt.read_bytes()),
|
|
987
|
+
session_path=wrapped_plan_session,
|
|
988
|
+
)
|
|
989
|
+
assert plan_wrapper_bound["exit_code"] == 0
|
|
990
|
+
print(
|
|
991
|
+
"PASS invocation receipt identity, prompt/session digest, bypass guard, "
|
|
992
|
+
"phase-scoped BUILD_GATE network capability, and monitored-wrapper "
|
|
993
|
+
"integration including PLAN"
|
|
994
|
+
)
|
|
995
|
+
return 0
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def main() -> int:
|
|
999
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
1000
|
+
parser.add_argument("--self-test", action="store_true")
|
|
1001
|
+
subparsers = parser.add_subparsers(dest="action")
|
|
1002
|
+
start = subparsers.add_parser("start")
|
|
1003
|
+
start.add_argument("--workdir", required=True)
|
|
1004
|
+
start.add_argument("--receipt", required=True)
|
|
1005
|
+
start.add_argument("--run-id", required=True)
|
|
1006
|
+
start.add_argument("--phase", choices=sorted(PHASES), required=True)
|
|
1007
|
+
start.add_argument("--round", type=int, required=True)
|
|
1008
|
+
start.add_argument("--prompt-file", required=True)
|
|
1009
|
+
start.add_argument("--session-file", required=True)
|
|
1010
|
+
start.add_argument("argv", nargs=argparse.REMAINDER)
|
|
1011
|
+
finish = subparsers.add_parser("finish")
|
|
1012
|
+
finish.add_argument("--workdir", required=True)
|
|
1013
|
+
finish.add_argument("--receipt", required=True)
|
|
1014
|
+
finish.add_argument("--exit-code", type=int, required=True)
|
|
1015
|
+
args = parser.parse_args()
|
|
1016
|
+
if args.self_test:
|
|
1017
|
+
return self_test()
|
|
1018
|
+
try:
|
|
1019
|
+
if args.action == "start":
|
|
1020
|
+
argv = args.argv[1:] if args.argv[:1] == ["--"] else args.argv
|
|
1021
|
+
start_receipt(
|
|
1022
|
+
pathlib.Path(args.workdir).resolve(), pathlib.Path(args.receipt), args.run_id,
|
|
1023
|
+
args.phase, args.round, args.prompt_file, args.session_file, argv,
|
|
1024
|
+
)
|
|
1025
|
+
elif args.action == "finish":
|
|
1026
|
+
work = pathlib.Path(args.workdir).resolve()
|
|
1027
|
+
receipt_path = pathlib.Path(args.receipt)
|
|
1028
|
+
if not receipt_path.is_absolute():
|
|
1029
|
+
receipt_path = work / receipt_path
|
|
1030
|
+
finish_receipt(
|
|
1031
|
+
work, receipt_path, args.exit_code,
|
|
1032
|
+
)
|
|
1033
|
+
else:
|
|
1034
|
+
parser.error("an action is required")
|
|
1035
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
1036
|
+
parser.error(str(exc))
|
|
1037
|
+
return 0
|
|
1038
|
+
|
|
1039
|
+
|
|
1040
|
+
if __name__ == "__main__":
|
|
1041
|
+
raise SystemExit(main())
|