devlyn-cli 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +10 -94
- package/CLAUDE.md +7 -160
- package/LICENSE +21 -0
- package/README.md +65 -110
- package/bin/devlyn.js +95 -116
- package/bin/instruction-templates.json +43 -2
- package/bin/instructions.js +29 -8
- package/config/skills/_shared/adapters/claude.md +8 -85
- package/config/skills/_shared/adapters/codex.md +14 -34
- package/config/skills/_shared/adapters/omp.md +1 -11
- package/config/skills/_shared/codex-monitored.sh +4 -54
- package/config/skills/_shared/engine-doctor.sh +18 -58
- package/config/skills/_shared/engine-preflight.md +15 -47
- package/config/skills/_shared/expected-contract.py +242 -0
- package/config/skills/_shared/expected.schema.json +10 -95
- package/config/skills/_shared/invocation-receipt.py +26 -757
- package/config/skills/_shared/platform-support.py +37 -28
- package/config/skills/_shared/role-config.py +100 -421
- package/config/skills/_shared/runtime-principles.md +17 -93
- package/config/skills/_shared/task-complete.py +490 -285
- package/config/skills/_shared/task-completion.md +235 -0
- package/config/skills/devlyn-engines/SKILL.md +9 -11
- package/config/skills/devlyn-ideate/SKILL.md +57 -107
- package/config/skills/devlyn-ideate/references/elicitation.md +18 -99
- package/config/skills/devlyn-ideate/references/loop.md +110 -0
- package/config/skills/devlyn-ideate/references/package-format.md +128 -0
- package/config/skills/devlyn-ideate/scripts/acceptance.py +795 -0
- package/config/skills/devlyn-ideate/scripts/queue.py +1780 -0
- package/package.json +2 -3
- package/config/commit-conventions.md +0 -28
- package/config/skills/_shared/adapters/README.md +0 -74
- package/config/skills/_shared/adapters/grok.md +0 -100
- package/config/skills/_shared/archive_run.py +0 -969
- package/config/skills/_shared/codex-config.md +0 -81
- package/config/skills/_shared/collect-codex-findings.py +0 -505
- package/config/skills/_shared/finish-gate.py +0 -521
- package/config/skills/_shared/grok-anchor-guard.py +0 -78
- package/config/skills/_shared/judge-output-parser.py +0 -200
- package/config/skills/_shared/judge-role-evidence.py +0 -320
- package/config/skills/_shared/phase-prompt-render.py +0 -183
- package/config/skills/_shared/process-evidence.py +0 -926
- package/config/skills/_shared/resolve-bootstrap.py +0 -1410
- package/config/skills/_shared/resolve-stop-hook.py +0 -359
- package/config/skills/_shared/run-bounded.py +0 -58
- package/config/skills/_shared/spec-verify-check.py +0 -5083
- package/config/skills/_shared/state-phase-write.py +0 -5989
- package/config/skills/_shared/terminal-claim-check.py +0 -777
- package/config/skills/_shared/verify-merge-findings.py +0 -3528
- package/config/skills/devlyn-design-ui/SKILL.md +0 -650
- package/config/skills/devlyn-ideate/references/from-spec-mode.md +0 -61
- package/config/skills/devlyn-ideate/references/project-mode.md +0 -80
- package/config/skills/devlyn-ideate/references/spec-template.md +0 -121
- package/config/skills/devlyn-ideate/references/templates/decision.md +0 -48
- package/config/skills/devlyn-ideate/references/templates/roadmap.md +0 -50
- package/config/skills/devlyn-ideate/references/templates/vision.md +0 -44
- package/config/skills/devlyn-queue/SKILL.md +0 -66
- package/config/skills/devlyn-queue/scripts/append.py +0 -94
- package/config/skills/devlyn-resolve/SKILL.md +0 -379
- package/config/skills/devlyn-resolve/references/free-form-mode.md +0 -71
- package/config/skills/devlyn-resolve/references/outer-loop.md +0 -23
- package/config/skills/devlyn-resolve/references/phases/build-gate.md +0 -75
- package/config/skills/devlyn-resolve/references/phases/cleanup.md +0 -33
- package/config/skills/devlyn-resolve/references/phases/implement.md +0 -44
- package/config/skills/devlyn-resolve/references/phases/plan.md +0 -39
- package/config/skills/devlyn-resolve/references/phases/probe-derive.md +0 -261
- package/config/skills/devlyn-resolve/references/phases/surface-close.md +0 -34
- package/config/skills/devlyn-resolve/references/phases/verify.md +0 -310
- package/config/skills/devlyn-resolve/references/state-schema.md +0 -152
- package/config/skills/devlyn-resolve/references/task-completion.md +0 -160
- package/config/templates/prompt-templates.md +0 -71
- package/config/templates/template-feature.spec.md +0 -255
- package/config/templates/template-product-spec.md +0 -680
|
@@ -0,0 +1,795 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Loop acceptance: derive a task result from its committed source and evidence.
|
|
3
|
+
|
|
4
|
+
`run` executes the declared checks on the committed candidate; the executor may
|
|
5
|
+
call it before review. `accept` derives the drain result: it binds the contract,
|
|
6
|
+
requires the owned committed source, reuses the intact outcomes of a wholly clean
|
|
7
|
+
runner result for the same source and contract, executes the rest, evaluates
|
|
8
|
+
file/diff guards, confirms the source is unchanged and checks review records. A
|
|
9
|
+
malformed executor record fails acceptance with its reason. An executor's summary
|
|
10
|
+
verdict is never evidence. Protocol: ../references/loop.md.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import functools
|
|
16
|
+
import hashlib
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
import re
|
|
21
|
+
import runpy
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
import tempfile
|
|
25
|
+
import unittest
|
|
26
|
+
|
|
27
|
+
SKILLS = Path(__file__).resolve().parents[2]
|
|
28
|
+
LOOP_DIR = ".devlyn/loop"
|
|
29
|
+
QUEUE = "docs/specs/queue.md"
|
|
30
|
+
SHA_RE = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}")
|
|
31
|
+
BLOCKERS = ("failed", "needs-review", "blocked-infrastructure")
|
|
32
|
+
REVIEW_KEYS = {"schema_version", "kind", "task", "engine", "model", "source_sha", "contract_sha256",
|
|
33
|
+
"expected_sha256", "requirements", "findings"}
|
|
34
|
+
SUBMISSION_KEYS = {"schema_version", "task", "source_sha", "runner_results", "reviews", "findings", "cleanup",
|
|
35
|
+
"handoff", "assumptions", "blockers", "summary"}
|
|
36
|
+
PACKET_KEYS = {"task", "worktree", "branch", "inputs_sha", "allocation_base", "contract", "expected", "meta",
|
|
37
|
+
"requirements", "review_requirements"}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class AcceptanceError(Exception):
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@functools.lru_cache(maxsize=None)
|
|
45
|
+
def shared(name):
|
|
46
|
+
return runpy.run_path(str(SKILLS / "_shared" / f"{name}.py"))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def git(work, *args, ok=(0,)):
|
|
50
|
+
result = subprocess.run(["git", "-C", str(work), *args], capture_output=True)
|
|
51
|
+
if result.returncode not in ok:
|
|
52
|
+
raise AcceptanceError(f"git {' '.join(args[:2])}: {result.stderr.decode('utf-8', 'replace').strip()}")
|
|
53
|
+
return result
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def text(work, *args):
|
|
57
|
+
return git(work, *args).stdout.decode("utf-8").strip()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def sha256(raw):
|
|
61
|
+
return hashlib.sha256(raw).hexdigest()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def read_json(path):
|
|
65
|
+
try:
|
|
66
|
+
return shared("expected-contract")["loads_strict_json"](Path(path).read_text(encoding="utf-8"))
|
|
67
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
68
|
+
raise AcceptanceError(f"{path}: {exc}") from exc
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def atomic_write(path, data):
|
|
72
|
+
path = Path(path)
|
|
73
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
74
|
+
fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.")
|
|
75
|
+
try:
|
|
76
|
+
with os.fdopen(fd, "wb") as stream:
|
|
77
|
+
stream.write(data)
|
|
78
|
+
os.replace(temporary, path)
|
|
79
|
+
finally:
|
|
80
|
+
Path(temporary).unlink(missing_ok=True)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def write_json(path, value):
|
|
84
|
+
atomic_write(path, (json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8"))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def load_packet(path):
|
|
88
|
+
packet = read_json(path)
|
|
89
|
+
if not isinstance(packet, dict) or PACKET_KEYS - set(packet):
|
|
90
|
+
raise AcceptanceError(f"{path}: task packet lacks {sorted(PACKET_KEYS - set(packet if isinstance(packet, dict) else {}))}")
|
|
91
|
+
return packet
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def contract_digests(packet):
|
|
95
|
+
return {key: packet[key]["sha256"] for key in ("contract", "expected", "meta")}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def relative(work, path):
|
|
99
|
+
return Path(path).resolve().relative_to(Path(work).resolve()).as_posix()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def evidence_file(work, path, under=LOOP_DIR):
|
|
103
|
+
candidate = Path(path) if Path(path).is_absolute() else Path(work) / path
|
|
104
|
+
resolved = candidate.resolve()
|
|
105
|
+
if not resolved.is_relative_to((Path(work) / under).resolve()) or not resolved.is_file():
|
|
106
|
+
raise AcceptanceError(f"evidence must be a file under {under} in the task worktree: {path}")
|
|
107
|
+
return resolved
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def bound_contract(packet):
|
|
111
|
+
"""Return the committed expected contract after checking every input against the packet digests."""
|
|
112
|
+
work = Path(packet["worktree"])
|
|
113
|
+
raw = {}
|
|
114
|
+
for key in ("contract", "expected", "meta"):
|
|
115
|
+
rel = relative(work, packet[key]["path"])
|
|
116
|
+
raw[key] = git(work, "show", f"{packet['inputs_sha']}:{rel}").stdout
|
|
117
|
+
if sha256(raw[key]) != packet[key]["sha256"]:
|
|
118
|
+
raise AcceptanceError(f"{rel} at the inputs commit differs from the task packet")
|
|
119
|
+
contract = shared("expected-contract")
|
|
120
|
+
try:
|
|
121
|
+
expected = contract["loads_strict_json"](raw["expected"].decode("utf-8"))
|
|
122
|
+
except (UnicodeError, ValueError) as exc:
|
|
123
|
+
raise AcceptanceError(f"expected contract: {exc}") from exc
|
|
124
|
+
error = contract["validate_expected_shape"](expected)
|
|
125
|
+
if error:
|
|
126
|
+
raise AcceptanceError(f"expected contract: {error}")
|
|
127
|
+
return expected
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def source_problems(packet, source):
|
|
131
|
+
"""The candidate must be the owned task commit, descend from the inputs, leave them untouched and be clean."""
|
|
132
|
+
work = Path(packet["worktree"])
|
|
133
|
+
if not SHA_RE.fullmatch(source) or git(work, "cat-file", "-e", source + "^{commit}", ok=(0, 1, 128)).returncode:
|
|
134
|
+
return [f"candidate {source!r} is not a commit"]
|
|
135
|
+
problems = []
|
|
136
|
+
head = git(work, "symbolic-ref", "-q", "HEAD", ok=(0, 1)).stdout.decode("utf-8").strip()
|
|
137
|
+
branch = text(work, "rev-parse", "refs/heads/" + packet["branch"])
|
|
138
|
+
if (source, head) != (branch, "refs/heads/" + packet["branch"]):
|
|
139
|
+
problems.append(f"candidate {source} is not the owned task commit checked out on its branch (branch {branch}, HEAD {head or 'detached'})")
|
|
140
|
+
if git(work, "merge-base", "--is-ancestor", packet["inputs_sha"], source, ok=(0, 1)).returncode:
|
|
141
|
+
problems.append(f"candidate {source} does not descend from the committed inputs")
|
|
142
|
+
else:
|
|
143
|
+
package = relative(work, packet["meta"]["path"]).rpartition("/")[0] + "/"
|
|
144
|
+
edits = [path for path in changed_paths(work, packet["inputs_sha"], source) if path == QUEUE or path.startswith(package)]
|
|
145
|
+
if edits:
|
|
146
|
+
problems.append("candidate edits loop inputs: " + ", ".join(edits))
|
|
147
|
+
if dirty := git(work, "status", "--porcelain", "--untracked-files=all").stdout.decode("utf-8", "replace").splitlines():
|
|
148
|
+
problems.append("worktree has uncommitted or untracked changes (undeclared source delta): " + ", ".join(line[3:] for line in dirty))
|
|
149
|
+
return problems
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def changed_paths(work, old, new):
|
|
153
|
+
raw = git(work, "diff", "--name-only", "-z", "--no-renames", old, new).stdout
|
|
154
|
+
return [path.decode("utf-8", "surrogateescape") for path in raw.split(b"\0") if path]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def evaluate(command, outcome, combined):
|
|
158
|
+
if outcome["kind"] != "exit":
|
|
159
|
+
return False, outcome["kind"]
|
|
160
|
+
expected_exit = command.get("exit_code", 0)
|
|
161
|
+
if outcome["exit_code"] != expected_exit:
|
|
162
|
+
return False, f"exit {outcome['exit_code']}, expected {expected_exit}"
|
|
163
|
+
missing = [value for value in command.get("stdout_contains", []) if value.encode("utf-8") not in combined]
|
|
164
|
+
if missing:
|
|
165
|
+
return False, f"output lacks {missing!r}"
|
|
166
|
+
found = [value for value in command.get("stdout_not_contains", []) if value.encode("utf-8") in combined]
|
|
167
|
+
if found:
|
|
168
|
+
return False, f"output contains forbidden {found!r}"
|
|
169
|
+
return True, None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def execute(work, command, index, run_dir):
|
|
173
|
+
timeout = shared("expected-contract")["verification_timeout_sec"](command)
|
|
174
|
+
env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} # A Python check would leave __pycache__/ in the source it checks.
|
|
175
|
+
try:
|
|
176
|
+
if "argv" in command:
|
|
177
|
+
proc = subprocess.run(shared("platform-support")["native_argv"](command["argv"], cwd=work), cwd=work,
|
|
178
|
+
capture_output=True, timeout=timeout, env=env)
|
|
179
|
+
else:
|
|
180
|
+
proc = subprocess.run(command["cmd"], cwd=work, shell=True, capture_output=True, timeout=timeout, env=env)
|
|
181
|
+
out, err = proc.stdout, proc.stderr
|
|
182
|
+
outcome = ({"kind": "exit", "exit_code": proc.returncode} if proc.returncode >= 0
|
|
183
|
+
else {"kind": "signal", "signal": -proc.returncode})
|
|
184
|
+
except subprocess.TimeoutExpired as exc:
|
|
185
|
+
out, err, outcome = exc.stdout or b"", exc.stderr or b"", {"kind": "timeout", "timeout_sec": timeout}
|
|
186
|
+
except OSError as exc:
|
|
187
|
+
out, err, outcome = b"", str(exc).encode("utf-8"), {"kind": "spawn_error", "error": str(exc)}
|
|
188
|
+
record = {"index": index, "outcome": outcome}
|
|
189
|
+
for name, raw in (("stdout", out), ("stderr", err)):
|
|
190
|
+
path = run_dir / f"cmd-{index}.{name}"
|
|
191
|
+
path.write_bytes(raw)
|
|
192
|
+
record[name] = {"path": relative(work, path), "sha256": sha256(raw), "bytes": len(raw)}
|
|
193
|
+
record["passed"], record["reason"] = evaluate(command, outcome, out + err)
|
|
194
|
+
return record
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def new_run_dir(work):
|
|
198
|
+
runs = Path(work) / LOOP_DIR / "runs"
|
|
199
|
+
runs.mkdir(parents=True, exist_ok=True)
|
|
200
|
+
number = 1 + max((int(path.name) for path in runs.iterdir() if path.name.isdigit()), default=0)
|
|
201
|
+
run_dir = runs / str(number)
|
|
202
|
+
run_dir.mkdir()
|
|
203
|
+
return run_dir
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def intact(work, record):
|
|
207
|
+
try:
|
|
208
|
+
return all(sha256(evidence_file(work, record[name]["path"]).read_bytes()) == record[name]["sha256"]
|
|
209
|
+
for name in ("stdout", "stderr"))
|
|
210
|
+
except (AcceptanceError, KeyError, TypeError, OSError):
|
|
211
|
+
return False
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def reusable(packet, source, paths):
|
|
215
|
+
"""Passed outcomes of wholly clean runner results (no reasons, so no failed check or source change during
|
|
216
|
+
the checks) for this exact source and contract, with intact raw streams."""
|
|
217
|
+
work = Path(packet["worktree"])
|
|
218
|
+
reused = {}
|
|
219
|
+
for path in paths:
|
|
220
|
+
result_path = evidence_file(work, path)
|
|
221
|
+
result = read_json(result_path)
|
|
222
|
+
if not isinstance(result, dict) or (result.get("kind"), result.get("task"), result.get("source_sha"), result.get("contract"),
|
|
223
|
+
result.get("reasons")) != ("loop-checks", packet["task"], source, contract_digests(packet), []):
|
|
224
|
+
continue
|
|
225
|
+
records = result.get("commands")
|
|
226
|
+
if not isinstance(records, list) or not all(isinstance(record, dict) and type(record.get("index")) is int for record in records):
|
|
227
|
+
raise AcceptanceError(f"runner result {relative(work, result_path)} has malformed command records")
|
|
228
|
+
for record in records:
|
|
229
|
+
if record.get("passed") is True and intact(work, record):
|
|
230
|
+
reused.setdefault(record["index"], dict(record, reused_from=relative(work, result_path)))
|
|
231
|
+
return reused
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def run_checks(packet, expected, reused):
|
|
235
|
+
work = Path(packet["worktree"])
|
|
236
|
+
records, run_dir = [], None
|
|
237
|
+
for index, command in enumerate(expected.get("verification_commands", [])):
|
|
238
|
+
if index in reused:
|
|
239
|
+
records.append(reused[index])
|
|
240
|
+
continue
|
|
241
|
+
run_dir = run_dir or new_run_dir(work)
|
|
242
|
+
records.append(execute(work, command, index, run_dir))
|
|
243
|
+
for record, command in zip(records, expected.get("verification_commands", [])):
|
|
244
|
+
record["command"] = {key: command[key] for key in ("cmd", "argv") if key in command}
|
|
245
|
+
record["contract_refs"] = command.get("contract_refs", [])
|
|
246
|
+
return records, run_dir
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def guard_results(packet, expected, source):
|
|
250
|
+
work, inputs = Path(packet["worktree"]), packet["inputs_sha"]
|
|
251
|
+
contract = shared("expected-contract")
|
|
252
|
+
diff = git(work, "diff", "--no-color", "--no-ext-diff", inputs, source).stdout.decode("utf-8", "surrogateescape")
|
|
253
|
+
changed = set(changed_paths(work, inputs, source))
|
|
254
|
+
results = []
|
|
255
|
+
for index, pattern in enumerate(expected.get("forbidden_patterns", [])):
|
|
256
|
+
try:
|
|
257
|
+
matched = re.search(pattern["pattern"], contract["slice_diff_to_files"](diff, pattern.get("files") or [])) is not None
|
|
258
|
+
except re.error as exc:
|
|
259
|
+
raise AcceptanceError(f"forbidden_patterns/{index} is not a valid regex: {exc}") from exc
|
|
260
|
+
results.append({"rule": f"forbidden_patterns/{index}", "subject": pattern["description"], "passed": not matched,
|
|
261
|
+
"blocking": pattern["severity"] == "disqualifier"})
|
|
262
|
+
for path in expected.get("required_files", []):
|
|
263
|
+
present = git(work, "cat-file", "-e", f"{source}:{path}", ok=(0, 1, 128)).returncode == 0
|
|
264
|
+
results.append({"rule": "required_files", "subject": path, "passed": present, "blocking": True})
|
|
265
|
+
for path in expected.get("forbidden_files", []):
|
|
266
|
+
results.append({"rule": "forbidden_files", "subject": path, "passed": path not in changed, "blocking": True})
|
|
267
|
+
limit = expected.get("max_deps_added", 0)
|
|
268
|
+
try:
|
|
269
|
+
added = len(dependency_names(work, source) - dependency_names(work, inputs))
|
|
270
|
+
subject, passed = f"{added} added, limit {limit}", added <= limit
|
|
271
|
+
except ValueError as exc:
|
|
272
|
+
subject, passed = str(exc), False
|
|
273
|
+
results.append({"rule": "max_deps_added", "subject": subject, "passed": passed, "blocking": True})
|
|
274
|
+
return results
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def dependency_names(work, commit):
|
|
278
|
+
"""Package names under dependencies and devDependencies of the root package.json at `commit`; a missing file has none."""
|
|
279
|
+
if not git(work, "ls-tree", commit, "--", "package.json").stdout:
|
|
280
|
+
return set()
|
|
281
|
+
try:
|
|
282
|
+
# Read as npm reads it: a leading BOM is ignored, a duplicate key's last value wins, a null section is empty and
|
|
283
|
+
# NaN or Infinity is invalid.
|
|
284
|
+
data = json.loads(git(work, "show", f"{commit}:package.json").stdout.decode("utf-8-sig"),
|
|
285
|
+
parse_constant=shared("expected-contract")["reject_json_constant"])
|
|
286
|
+
sections = [data.get(key) or {} for key in ("dependencies", "devDependencies")] if isinstance(data, dict) else [None]
|
|
287
|
+
if not all(isinstance(section, dict) for section in sections):
|
|
288
|
+
raise ValueError("it must be an object whose dependencies and devDependencies are objects")
|
|
289
|
+
except ValueError as exc:
|
|
290
|
+
raise ValueError(f"package.json at {commit} is invalid: {exc}") from exc
|
|
291
|
+
return {name for section in sections for name in section}
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def review_problem(packet, record, source):
|
|
295
|
+
"""(problem, malformed): a malformed record fails acceptance; a well-formed record bound elsewhere does not count.
|
|
296
|
+
Keys acceptance does not read are ignored: every key a blocking decision reads is required."""
|
|
297
|
+
if not isinstance(record, dict) or not REVIEW_KEYS <= set(record):
|
|
298
|
+
return f"review record keys must include {sorted(REVIEW_KEYS)}", True
|
|
299
|
+
if (record["schema_version"], record["kind"]) != (1, "devlyn-review"):
|
|
300
|
+
return "review record is not a schema 1 devlyn-review", True
|
|
301
|
+
if not all(isinstance(record[key], str) and record[key].strip() for key in ("engine", "model")):
|
|
302
|
+
return "review record must name its engine and model", True
|
|
303
|
+
if not isinstance(record["requirements"], list) or not all(req in packet["requirements"] for req in record["requirements"]):
|
|
304
|
+
return "review requirements must name task requirement IDs", True
|
|
305
|
+
findings = record["findings"]
|
|
306
|
+
if not isinstance(findings, list) or not all(
|
|
307
|
+
isinstance(f, dict) and {"id", "binding", "disposition"} <= set(f) and isinstance(f["id"], str)
|
|
308
|
+
and isinstance(f["binding"], bool) and f["disposition"] in ("open", "resolved", "rejected")
|
|
309
|
+
and (f["disposition"] != "rejected" or (isinstance(f.get("reason"), str) and f["reason"].strip()))
|
|
310
|
+
for f in findings):
|
|
311
|
+
return "findings need id, boolean binding and disposition open|resolved|rejected (rejected needs a reason)", True
|
|
312
|
+
if record["task"] != packet["task"]:
|
|
313
|
+
return f"review is for task {record['task']!r}, not {packet['task']}", False
|
|
314
|
+
if record["source_sha"] != source:
|
|
315
|
+
return f"review is bound to source {record['source_sha']}, not {source}", False
|
|
316
|
+
if (record["contract_sha256"], record["expected_sha256"]) != (packet["contract"]["sha256"], packet["expected"]["sha256"]):
|
|
317
|
+
return "review is bound to a different contract", False
|
|
318
|
+
return None, False
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def review_results(packet, paths, source):
|
|
322
|
+
work = Path(packet["worktree"])
|
|
323
|
+
counted, ignored = [], []
|
|
324
|
+
for path in paths:
|
|
325
|
+
try:
|
|
326
|
+
record_path = evidence_file(work, path, ".devlyn")
|
|
327
|
+
raw = record_path.read_bytes()
|
|
328
|
+
record = shared("expected-contract")["loads_strict_json"](raw.decode("utf-8"))
|
|
329
|
+
problem, malformed = review_problem(packet, record, source)
|
|
330
|
+
except (AcceptanceError, OSError, UnicodeError, ValueError) as exc:
|
|
331
|
+
problem, malformed = str(exc), True
|
|
332
|
+
if malformed:
|
|
333
|
+
raise AcceptanceError(f"malformed review record {path}: {problem}")
|
|
334
|
+
if problem:
|
|
335
|
+
ignored.append({"path": str(path), "reason": problem})
|
|
336
|
+
continue
|
|
337
|
+
counted.append({"path": relative(work, record_path), "sha256": sha256(raw), "engine": record["engine"],
|
|
338
|
+
"model": record["model"], "requirements": record["requirements"],
|
|
339
|
+
"open_binding_findings": [f["id"] for f in record["findings"] if f["binding"] and f["disposition"] == "open"]})
|
|
340
|
+
return counted, ignored
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def check_submission(packet, submission):
|
|
344
|
+
if not isinstance(submission, dict) or not {"schema_version", "task", "source_sha"} <= set(submission) <= SUBMISSION_KEYS:
|
|
345
|
+
raise AcceptanceError(f"submission keys must include schema_version, task, source_sha and stay within {sorted(SUBMISSION_KEYS)}")
|
|
346
|
+
if (submission["schema_version"], submission["task"]) != (1, packet["task"]) or not isinstance(submission["source_sha"], str):
|
|
347
|
+
raise AcceptanceError("submission is not schema 1 for this task with a string source_sha")
|
|
348
|
+
for key in ("runner_results", "reviews", "assumptions"):
|
|
349
|
+
value = submission.get(key, [])
|
|
350
|
+
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
|
351
|
+
raise AcceptanceError(f"submission {key} must be a list of strings")
|
|
352
|
+
blockers = submission.get("blockers", [])
|
|
353
|
+
if not isinstance(blockers, list) or not all(
|
|
354
|
+
isinstance(b, dict) and {"kind", "detail"} <= set(b) and b["kind"] in BLOCKERS and isinstance(b["detail"], str) and b["detail"].strip()
|
|
355
|
+
for b in blockers):
|
|
356
|
+
raise AcceptanceError(f"submission blockers must include {{kind, detail}} with kind in {sorted(BLOCKERS)}")
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def result_shell(packet, kind, source):
|
|
360
|
+
return {"schema_version": 1, "kind": kind, "task": packet["task"], "source_sha": source,
|
|
361
|
+
"inputs_sha": packet["inputs_sha"], "allocation_base": packet["allocation_base"],
|
|
362
|
+
"contract": contract_digests(packet), "commands": [], "guards": []}
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def check_reasons(result):
|
|
366
|
+
reasons = [f"failed: command {r['index']} ({', '.join(r['contract_refs']) or 'no refs'}) {r['reason']}"
|
|
367
|
+
for r in result["commands"] if not r["passed"]]
|
|
368
|
+
return reasons + [f"failed: guard {g['rule']} {g['subject']}" for g in result["guards"] if g["blocking"] and not g["passed"]]
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def run(packet_path, source=None):
|
|
372
|
+
"""Executor-side runner: checks on the committed candidate, before review."""
|
|
373
|
+
packet = load_packet(packet_path)
|
|
374
|
+
work = Path(packet["worktree"])
|
|
375
|
+
source = source or text(work, "rev-parse", "HEAD")
|
|
376
|
+
expected = bound_contract(packet)
|
|
377
|
+
problems = source_problems(packet, source)
|
|
378
|
+
if problems:
|
|
379
|
+
raise AcceptanceError("; ".join(problems))
|
|
380
|
+
result = result_shell(packet, "loop-checks", source)
|
|
381
|
+
result["commands"], run_dir = run_checks(packet, expected, {})
|
|
382
|
+
result["guards"] = guard_results(packet, expected, source)
|
|
383
|
+
result["reasons"] = check_reasons(result) + [f"failed: source changed during checks: {p}" for p in source_problems(packet, source)]
|
|
384
|
+
path = (run_dir or new_run_dir(work)) / "result.json"
|
|
385
|
+
write_json(path, result)
|
|
386
|
+
return path, result
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def accept(packet_path, submission_path, failure=None):
|
|
390
|
+
"""Drain-side derivation; writes .devlyn/loop/acceptance.json for task-complete custody. A drain-recorded
|
|
391
|
+
`failure` (changed inputs, an unobservable interrupted execution) fails the task without its submission."""
|
|
392
|
+
packet = load_packet(packet_path)
|
|
393
|
+
work = Path(packet["worktree"])
|
|
394
|
+
result = result_shell(packet, "loop", text(work, "rev-parse", "refs/heads/" + packet["branch"]))
|
|
395
|
+
result.update(reviews=[], ignored_reviews=[], assumptions=[], reasons=[failure] if failure else [])
|
|
396
|
+
try:
|
|
397
|
+
if not failure:
|
|
398
|
+
submission = read_json(submission_path)
|
|
399
|
+
check_submission(packet, submission)
|
|
400
|
+
result["assumptions"] = submission.get("assumptions", [])
|
|
401
|
+
if submission.get("blockers"):
|
|
402
|
+
result["reasons"] = [f"{b['kind']}: {b['detail']}" for b in submission["blockers"]]
|
|
403
|
+
else:
|
|
404
|
+
derive(packet, submission, result)
|
|
405
|
+
except AcceptanceError as exc:
|
|
406
|
+
result["reasons"].append(f"failed: {exc}")
|
|
407
|
+
result["verdict"] = "FAILED" if result["reasons"] else "ACCEPTED"
|
|
408
|
+
result["evidence"] = sorted({r[name]["path"] for r in result["commands"] for name in ("stdout", "stderr")}
|
|
409
|
+
| {review["path"] for review in result["reviews"]})
|
|
410
|
+
write_json(work / LOOP_DIR / "acceptance.json", result)
|
|
411
|
+
return result
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def derive(packet, submission, result):
|
|
415
|
+
source = submission["source_sha"]
|
|
416
|
+
expected = bound_contract(packet)
|
|
417
|
+
problems = source_problems(packet, source)
|
|
418
|
+
if problems:
|
|
419
|
+
result["reasons"] += [f"failed: {problem}" for problem in problems]
|
|
420
|
+
return
|
|
421
|
+
result["source_sha"] = source
|
|
422
|
+
result["commands"], _ = run_checks(packet, expected, reusable(packet, source, submission.get("runner_results", [])))
|
|
423
|
+
result["guards"] = guard_results(packet, expected, source)
|
|
424
|
+
result["reasons"] += check_reasons(result)
|
|
425
|
+
result["reasons"] += [f"failed: source changed during checks: {p}" for p in source_problems(packet, source)]
|
|
426
|
+
result["reviews"], result["ignored_reviews"] = review_results(packet, submission.get("reviews", []), source)
|
|
427
|
+
covered = {req for review in result["reviews"] for req in review["requirements"]}
|
|
428
|
+
missing = [req for req in packet["review_requirements"] if req not in covered]
|
|
429
|
+
if missing:
|
|
430
|
+
result["reasons"].append("failed: required review coverage missing for " + ", ".join(missing))
|
|
431
|
+
unresolved = [finding for review in result["reviews"] for finding in review["open_binding_findings"]]
|
|
432
|
+
if unresolved:
|
|
433
|
+
result["reasons"].append("failed: unresolved binding review findings " + ", ".join(unresolved))
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
class AcceptanceTests(unittest.TestCase):
|
|
437
|
+
def setUp(self):
|
|
438
|
+
self.temp = tempfile.TemporaryDirectory(prefix="devlyn-acceptance ")
|
|
439
|
+
self.addCleanup(self.temp.cleanup)
|
|
440
|
+
self.root = Path(self.temp.name).resolve()
|
|
441
|
+
self.work = self.root / "work tree"
|
|
442
|
+
env = {"GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": str(self.root / "gitconfig"), "GIT_AUTHOR_NAME": "Fixture",
|
|
443
|
+
"GIT_AUTHOR_EMAIL": "fixture@example.invalid", "GIT_COMMITTER_NAME": "Fixture", "GIT_COMMITTER_EMAIL": "fixture@example.invalid"}
|
|
444
|
+
self.saved = {key: os.environ.get(key) for key in env}
|
|
445
|
+
os.environ.update(env)
|
|
446
|
+
self.addCleanup(lambda: [os.environ.pop(k, None) if v is None else os.environ.__setitem__(k, v) for k, v in self.saved.items()])
|
|
447
|
+
subprocess.run(["git", "init", "-q", "--initial-branch=main", str(self.work)], check=True)
|
|
448
|
+
self.write(".gitignore", ".devlyn/\n")
|
|
449
|
+
self.commit("base")
|
|
450
|
+
self.g("switch", "-q", "-c", "devlyn/l/t")
|
|
451
|
+
self.commands = [
|
|
452
|
+
{"argv": [sys.executable, "-c", "import pathlib; print(pathlib.Path('product.txt').read_text())"],
|
|
453
|
+
"stdout_contains": ["ready"], "contract_refs": ["R1"]},
|
|
454
|
+
{"cmd": "echo shell-ok", "stdout_contains": ["shell-ok"], "contract_refs": ["R1"]},
|
|
455
|
+
]
|
|
456
|
+
|
|
457
|
+
def g(self, *args):
|
|
458
|
+
return text(self.work, *args)
|
|
459
|
+
|
|
460
|
+
def write(self, rel, data):
|
|
461
|
+
path = self.work / rel
|
|
462
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
463
|
+
path.write_text(data, encoding="utf-8")
|
|
464
|
+
|
|
465
|
+
def commit(self, message):
|
|
466
|
+
self.g("add", "-A")
|
|
467
|
+
self.g("commit", "-q", "-m", message)
|
|
468
|
+
return self.g("rev-parse", "HEAD")
|
|
469
|
+
|
|
470
|
+
def inputs(self, expected=None, review=()):
|
|
471
|
+
base = self.g("rev-parse", "HEAD")
|
|
472
|
+
files = {"docs/specs/l/meta.md": "# meta\n", "docs/specs/l/t/spec.md": "# task R1 R2\n",
|
|
473
|
+
"docs/specs/l/t/spec.expected.json": json.dumps(expected or {"verification_commands": self.commands}),
|
|
474
|
+
QUEUE: "- [ ] l.t [T](docs/specs/l/t/spec.md)\n"}
|
|
475
|
+
for rel, data in files.items():
|
|
476
|
+
self.write(rel, data)
|
|
477
|
+
inputs = self.commit("inputs")
|
|
478
|
+
digest = {rel: sha256((self.work / rel).read_bytes()) for rel in files}
|
|
479
|
+
self.packet = {"task": "l.t", "worktree": str(self.work), "branch": "devlyn/l/t", "inputs_sha": inputs, "allocation_base": base,
|
|
480
|
+
"contract": {"path": str(self.work / "docs/specs/l/t/spec.md"), "sha256": digest["docs/specs/l/t/spec.md"]},
|
|
481
|
+
"expected": {"path": str(self.work / "docs/specs/l/t/spec.expected.json"), "sha256": digest["docs/specs/l/t/spec.expected.json"]},
|
|
482
|
+
"meta": {"path": str(self.work / "docs/specs/l/meta.md"), "sha256": digest["docs/specs/l/meta.md"]},
|
|
483
|
+
"requirements": ["R1", "R2"], "review_requirements": list(review)}
|
|
484
|
+
self.packet_path = self.root / "packet.json"
|
|
485
|
+
self.packet_path.write_text(json.dumps(self.packet), encoding="utf-8")
|
|
486
|
+
|
|
487
|
+
def product(self, body="ready"):
|
|
488
|
+
self.write("product.txt", body)
|
|
489
|
+
return self.commit("product")
|
|
490
|
+
|
|
491
|
+
def review(self, source, name="review.json", **changes):
|
|
492
|
+
record = {"schema_version": 1, "kind": "devlyn-review", "task": "l.t", "engine": "codex", "model": "fixture-model",
|
|
493
|
+
"source_sha": source, "contract_sha256": self.packet["contract"]["sha256"],
|
|
494
|
+
"expected_sha256": self.packet["expected"]["sha256"], "requirements": ["R2"], "findings": [], **changes}
|
|
495
|
+
self.write(f".devlyn/reviews/{name}", json.dumps(record))
|
|
496
|
+
return f".devlyn/reviews/{name}"
|
|
497
|
+
|
|
498
|
+
def submit(self, source, **fields):
|
|
499
|
+
path = self.root / "submission.json"
|
|
500
|
+
path.write_text(json.dumps({"schema_version": 1, "task": "l.t", "source_sha": source, **fields}), encoding="utf-8")
|
|
501
|
+
return accept(self.packet_path, path)
|
|
502
|
+
|
|
503
|
+
def test_accepts_owned_source_with_checks_and_bound_review(self):
|
|
504
|
+
self.inputs(review=["R2"])
|
|
505
|
+
source = self.product()
|
|
506
|
+
result = self.submit(source, reviews=[self.review(source)])
|
|
507
|
+
self.assertEqual((result["verdict"], result["reasons"]), ("ACCEPTED", []))
|
|
508
|
+
self.assertEqual(json.loads((self.work / LOOP_DIR / "acceptance.json").read_text(encoding="utf-8"))["source_sha"], source)
|
|
509
|
+
self.assertIn(".devlyn/reviews/review.json", result["evidence"])
|
|
510
|
+
for path in result["evidence"]:
|
|
511
|
+
self.assertTrue((self.work / path).is_file(), path)
|
|
512
|
+
|
|
513
|
+
def test_command_assertions_timeout_and_spawn_failure(self):
|
|
514
|
+
python = sys.executable
|
|
515
|
+
self.inputs({"verification_commands": [
|
|
516
|
+
{"argv": [python, "-c", "raise SystemExit(3)"], "contract_refs": ["R1"]},
|
|
517
|
+
{"argv": [python, "-c", "print('nope')"], "stdout_contains": ["yes"], "contract_refs": ["R1"]},
|
|
518
|
+
{"argv": [python, "-c", "print('secret')"], "stdout_not_contains": ["secret"], "contract_refs": ["R1"]},
|
|
519
|
+
{"argv": [python, "-c", "import time; time.sleep(5)"], "timeout_sec": 1, "contract_refs": ["R2"]},
|
|
520
|
+
{"argv": ["devlyn-missing-binary-for-test"], "contract_refs": ["R2"]},
|
|
521
|
+
]})
|
|
522
|
+
result = self.submit(self.product())
|
|
523
|
+
self.assertEqual(result["verdict"], "FAILED")
|
|
524
|
+
self.assertEqual([r["outcome"]["kind"] for r in result["commands"]], ["exit", "exit", "exit", "timeout", "spawn_error"])
|
|
525
|
+
self.assertEqual([bool(r["reason"]) for r in result["commands"]], [True] * 5)
|
|
526
|
+
self.assertIn("exit 3, expected 0", " ".join(result["reasons"]))
|
|
527
|
+
self.assertTrue(all((self.work / r["stdout"]["path"]).is_file() for r in result["commands"]))
|
|
528
|
+
|
|
529
|
+
def test_source_and_contract_binding(self):
|
|
530
|
+
self.inputs()
|
|
531
|
+
source = self.product()
|
|
532
|
+
self.write("untracked.txt", "delta")
|
|
533
|
+
self.assertIn("undeclared source delta", " ".join(self.submit(source)["reasons"]))
|
|
534
|
+
(self.work / "untracked.txt").unlink()
|
|
535
|
+
self.g("checkout", "-q", "--detach", source)
|
|
536
|
+
self.assertIn("is not the owned task commit", " ".join(self.submit(source)["reasons"]))
|
|
537
|
+
self.g("switch", "-q", "devlyn/l/t")
|
|
538
|
+
self.assertIn("is not the owned task commit", " ".join(self.submit(self.packet["inputs_sha"])["reasons"]))
|
|
539
|
+
self.write("docs/specs/l/t/spec.md", "# weakened\n")
|
|
540
|
+
weakened = self.commit("weaken")
|
|
541
|
+
self.assertIn("candidate edits loop inputs: docs/specs/l/t/spec.md", " ".join(self.submit(weakened)["reasons"]))
|
|
542
|
+
self.packet["contract"]["sha256"] = "0" * 64
|
|
543
|
+
self.packet_path.write_text(json.dumps(self.packet), encoding="utf-8")
|
|
544
|
+
self.assertIn("differs from the task packet", " ".join(self.submit(weakened)["reasons"]))
|
|
545
|
+
|
|
546
|
+
def test_review_binding_and_findings(self):
|
|
547
|
+
self.inputs(review=["R2"])
|
|
548
|
+
first = self.product()
|
|
549
|
+
stale = self.review(first, "stale.json")
|
|
550
|
+
source = self.product("ready again")
|
|
551
|
+
for reviews, message in (
|
|
552
|
+
([stale], "required review coverage missing for R2"),
|
|
553
|
+
([self.review(source, "contract.json", expected_sha256="0" * 64)], "required review coverage missing for R2"),
|
|
554
|
+
([self.review(source, "rejected.json", findings=[{"id": "F1", "binding": True, "disposition": "rejected"}])], "malformed review record"),
|
|
555
|
+
([self.review(source, "open.json", findings=[{"id": "F2", "binding": True, "disposition": "open"}])], "unresolved binding review findings F2"),
|
|
556
|
+
):
|
|
557
|
+
result = self.submit(source, reviews=reviews)
|
|
558
|
+
self.assertEqual(result["verdict"], "FAILED")
|
|
559
|
+
self.assertIn(message, " ".join(result["reasons"]))
|
|
560
|
+
result = self.submit(source, reviews=[stale, self.review(source, findings=[{"id": "F3", "binding": False, "disposition": "open"}])])
|
|
561
|
+
self.assertEqual(result["verdict"], "ACCEPTED")
|
|
562
|
+
self.assertEqual([item["path"] for item in result["ignored_reviews"]], [stale])
|
|
563
|
+
|
|
564
|
+
def test_review_records_and_findings_ignore_other_keys(self):
|
|
565
|
+
"""E2. Prediction: a review record with an extra top-level key and an extra key in a finding counts, and an open
|
|
566
|
+
binding finding carrying an extra key still fails acceptance with its id named. Before: both records failed as
|
|
567
|
+
"malformed review record", so the outcome depended on which engine wrote the record."""
|
|
568
|
+
self.inputs(review=["R2"])
|
|
569
|
+
source = self.product()
|
|
570
|
+
extra = {"evidence": ["checks pass"]}
|
|
571
|
+
counted = self.review(source, "extra.json", **extra, findings=[{"id": "F1", "binding": True, "disposition": "resolved", **extra}])
|
|
572
|
+
result = self.submit(source, reviews=[counted])
|
|
573
|
+
self.assertEqual((result["verdict"], [review["path"] for review in result["reviews"]]), ("ACCEPTED", [counted]))
|
|
574
|
+
blocking = self.review(source, "open.json", findings=[{"id": "F2", "binding": True, "disposition": "open", **extra}])
|
|
575
|
+
result = self.submit(source, reviews=[blocking])
|
|
576
|
+
self.assertEqual((result["verdict"], result["reasons"]), ("FAILED", ["failed: unresolved binding review findings F2"]))
|
|
577
|
+
|
|
578
|
+
def test_blockers_ignore_keys_beyond_kind_and_detail(self):
|
|
579
|
+
"""T6. Prediction: a needs-review blocker carrying a key beyond kind and detail records its question as the reason,
|
|
580
|
+
as one without it does, so the report and queue row keep the question. Before: the extra key failed the submission
|
|
581
|
+
"submission blockers must be {kind, detail} ...", and the question was lost."""
|
|
582
|
+
self.inputs()
|
|
583
|
+
result = self.submit(self.product(), blockers=[{"kind": "needs-review", "detail": "Which store wins?", "requirement": "R1"}])
|
|
584
|
+
self.assertEqual((result["verdict"], result["reasons"]), ("FAILED", ["needs-review: Which store wins?"]))
|
|
585
|
+
|
|
586
|
+
def test_python_checks_leave_no_bytecode_and_a_dirty_worktree_names_its_paths(self):
|
|
587
|
+
"""H7. Prediction: a check that imports a module committed in the task worktree, where nothing ignores
|
|
588
|
+
__pycache__/, writes no bytecode there, so the runner and acceptance both pass; a file left untracked fails
|
|
589
|
+
acceptance with its path named. Before: the import wrote __pycache__/ into the worktree, so the correct task failed
|
|
590
|
+
"source changed during checks: worktree has uncommitted or untracked changes (undeclared source delta)", naming
|
|
591
|
+
no path."""
|
|
592
|
+
self.inputs({"verification_commands": [{"argv": [sys.executable, "-c", "import greeting; print(greeting.greet())"],
|
|
593
|
+
"stdout_contains": ["hello"], "contract_refs": ["R1", "R2"]}]})
|
|
594
|
+
self.write("greeting.py", "def greet():\n return 'hello'\n")
|
|
595
|
+
source = self.commit("product")
|
|
596
|
+
_, checks = run(self.packet_path)
|
|
597
|
+
result = self.submit(source)
|
|
598
|
+
self.assertEqual((checks["reasons"], result["verdict"], result["reasons"], (self.work / "__pycache__").exists()), ([], "ACCEPTED", [], False))
|
|
599
|
+
self.write("notes.txt", "left behind\n")
|
|
600
|
+
self.assertEqual(self.submit(source)["reasons"], ["failed: worktree has uncommitted or untracked changes (undeclared source delta): notes.txt"])
|
|
601
|
+
|
|
602
|
+
def test_runner_result_with_a_source_change_is_never_reused(self):
|
|
603
|
+
mutate = [sys.executable, "-c", "import pathlib; p = pathlib.Path('product.txt'); p.write_text(p.read_text() + '!'); print('ok')"]
|
|
604
|
+
self.inputs({"verification_commands": [{"argv": mutate, "stdout_contains": ["ok"], "contract_refs": ["R1", "R2"]}]})
|
|
605
|
+
source = self.product()
|
|
606
|
+
path, checks = run(self.packet_path)
|
|
607
|
+
self.assertTrue(checks["commands"][0]["passed"])
|
|
608
|
+
self.assertIn("failed: source changed during checks", " ".join(checks["reasons"]))
|
|
609
|
+
self.g("checkout", "--", "product.txt")
|
|
610
|
+
result = self.submit(source, runner_results=[relative(self.work, path)])
|
|
611
|
+
self.assertEqual(result["verdict"], "FAILED")
|
|
612
|
+
self.assertNotIn("reused_from", result["commands"][0])
|
|
613
|
+
self.assertIn("failed: source changed during checks", " ".join(result["reasons"]))
|
|
614
|
+
|
|
615
|
+
def test_malformed_executor_records_fail_with_a_named_reason(self):
|
|
616
|
+
self.inputs(review=["R2"])
|
|
617
|
+
source = self.product()
|
|
618
|
+
malformed = self.review(source, "malformed.json", requirements=[{"id": "R2"}])
|
|
619
|
+
result = self.submit(source, reviews=[malformed])
|
|
620
|
+
self.assertEqual(result["verdict"], "FAILED")
|
|
621
|
+
self.assertIn(f"malformed review record {malformed}", " ".join(result["reasons"]))
|
|
622
|
+
path, _ = run(self.packet_path)
|
|
623
|
+
checks = json.loads(path.read_text(encoding="utf-8"))
|
|
624
|
+
del checks["commands"][0]["index"]
|
|
625
|
+
path.write_text(json.dumps(checks), encoding="utf-8")
|
|
626
|
+
result = self.submit(source, runner_results=[relative(self.work, path)], reviews=[self.review(source)])
|
|
627
|
+
self.assertEqual(result["verdict"], "FAILED")
|
|
628
|
+
self.assertIn("malformed command records", " ".join(result["reasons"]))
|
|
629
|
+
blocked = self.submit(source, blockers=[{"kind": ["failed"], "detail": "unhashable kind"}])
|
|
630
|
+
self.assertEqual(blocked["verdict"], "FAILED")
|
|
631
|
+
self.assertIn("submission blockers must include", " ".join(blocked["reasons"]))
|
|
632
|
+
|
|
633
|
+
def test_reuses_intact_runner_results_and_reexecutes_altered_evidence(self):
|
|
634
|
+
counter = [sys.executable, "-c", "import pathlib; p = pathlib.Path('.devlyn/count'); p.parent.mkdir(exist_ok=True); "
|
|
635
|
+
"p.write_text(str(int(p.read_text()) + 1) if p.exists() else '1'); print('counted')"]
|
|
636
|
+
self.inputs({"verification_commands": [{"argv": counter, "stdout_contains": ["counted"], "contract_refs": ["R1", "R2"]}]})
|
|
637
|
+
source = self.product()
|
|
638
|
+
path, checks = run(self.packet_path)
|
|
639
|
+
self.assertEqual(checks["reasons"], [])
|
|
640
|
+
runner = relative(self.work, path)
|
|
641
|
+
self.assertEqual(self.submit(source, runner_results=[runner])["commands"][0]["reused_from"], runner)
|
|
642
|
+
self.assertEqual((self.work / ".devlyn/count").read_text(), "1")
|
|
643
|
+
(self.work / checks["commands"][0]["stdout"]["path"]).write_text("altered", encoding="utf-8")
|
|
644
|
+
result = self.submit(source, runner_results=[runner])
|
|
645
|
+
self.assertNotIn("reused_from", result["commands"][0])
|
|
646
|
+
self.assertEqual(((self.work / ".devlyn/count").read_text(), result["verdict"]), ("2", "ACCEPTED"))
|
|
647
|
+
|
|
648
|
+
def test_windows_runs_node_command_shims_without_cmd(self):
|
|
649
|
+
"""Windows resolution mocked: verification argv such as ["npm", "test"] runs the Node and script its command
|
|
650
|
+
shim names, as cmd.exe would resolve it; any other batch file is refused with an actionable message."""
|
|
651
|
+
from types import SimpleNamespace
|
|
652
|
+
from unittest import mock
|
|
653
|
+
npm_cmd = "\r\n".join(( # npm 10 and 11 ship this npm.cmd byte for byte.
|
|
654
|
+
":: Created by npm, please don't edit manually.", "@ECHO OFF", "", "SETLOCAL", "",
|
|
655
|
+
'SET "NODE_EXE=%~dp0\\node.exe"', 'IF NOT EXIST "%NODE_EXE%" (', ' SET "NODE_EXE=node"', ")", "",
|
|
656
|
+
'SET "NPM_PREFIX_JS=%~dp0\\node_modules\\npm\\bin\\npm-prefix.js"', 'SET "NPM_CLI_JS=%~dp0\\node_modules\\npm\\bin\\npm-cli.js"',
|
|
657
|
+
'FOR /F "delims=" %%F IN (\'CALL "%NODE_EXE%" "%NPM_PREFIX_JS%"\') DO (',
|
|
658
|
+
' SET "NPM_PREFIX_NPM_CLI_JS=%%F\\node_modules\\npm\\bin\\npm-cli.js"', ")",
|
|
659
|
+
'IF EXIST "%NPM_PREFIX_NPM_CLI_JS%" (', ' SET "NPM_CLI_JS=%NPM_PREFIX_NPM_CLI_JS%"', ")", "", '"%NODE_EXE%" "%NPM_CLI_JS%" %*', ""))
|
|
660
|
+
cmd_shim = "\r\n".join(( # npm's cmd-shim 6 (npm install -g, node_modules/.bin).
|
|
661
|
+
"@ECHO off", "GOTO start", ":find_dp0", "SET dp0=%~dp0", "EXIT /b", ":start", "SETLOCAL", "CALL :find_dp0", "",
|
|
662
|
+
'IF EXIST "%dp0%\\node.exe" (', ' SET "_prog=%dp0%\\node.exe"', ") ELSE (", ' SET "_prog=node"',
|
|
663
|
+
" SET PATHEXT=%PATHEXT:;.JS;=;%", ")", "",
|
|
664
|
+
'endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%" "%dp0%\\node_modules\\pnpm\\bin\\pnpm.cjs" %*', ""))
|
|
665
|
+
corepack = "\r\n".join(( # @zkochan/cmd-shim, as corepack enable and pnpm write it.
|
|
666
|
+
"@SETLOCAL", '@IF EXIST "%~dp0\\node.exe" (', ' "%~dp0\\node.exe" "%~dp0\\node_modules\\corepack\\dist\\yarn.js" %*', ") ELSE (",
|
|
667
|
+
" @SET PATHEXT=%PATHEXT:;.JS;=;%", ' node "%~dp0\\node_modules\\corepack\\dist\\yarn.js" %*', ")", ""))
|
|
668
|
+
node_path = corepack.replace("@SETLOCAL\r\n", '@SETLOCAL\r\n@SET "NODE_PATH=C:\\proj\\node_modules\\.pnpm\\node_modules"\r\n')
|
|
669
|
+
nodejs, tools, prefix = self.root / "nodejs", self.root / "tools", self.root / "prefix"
|
|
670
|
+
node = nodejs / "node.exe"
|
|
671
|
+
files = {nodejs / "npm.cmd": npm_cmd, node: "", tools / "pnpm.cmd": cmd_shim, tools / "yarn.cmd": corepack,
|
|
672
|
+
tools / "vitest.cmd": node_path, tools / "build.bat": "@echo off\r\necho built\r\n"}
|
|
673
|
+
for path in (nodejs / "node_modules/npm/bin/npm-cli.js", nodejs / "node_modules/npm/bin/npm-prefix.js",
|
|
674
|
+
tools / "node_modules/pnpm/bin/pnpm.cjs", tools / "node_modules/corepack/dist/yarn.js"):
|
|
675
|
+
files[path] = ""
|
|
676
|
+
for path, body in files.items():
|
|
677
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
678
|
+
path.write_bytes(body.encode("utf-8"))
|
|
679
|
+
found = {"npm": nodejs / "npm.cmd", "node.exe": node, **{path.stem: path for path in tools.glob("*.*")}}
|
|
680
|
+
platform = shared("platform-support")
|
|
681
|
+
lookups = []
|
|
682
|
+
|
|
683
|
+
def npm_prefix(argv, cwd=None, **kwargs): # what node npm-prefix.js prints, recorded with the directory it ran in
|
|
684
|
+
lookups.append((argv, cwd))
|
|
685
|
+
return SimpleNamespace(stdout=f"{prefix}\n")
|
|
686
|
+
windows = {"os": SimpleNamespace(name="nt"), "shutil": SimpleNamespace(which=lambda name: str(found[name]) if name in found else None),
|
|
687
|
+
"subprocess": SimpleNamespace(run=npm_prefix)}
|
|
688
|
+
with mock.patch.dict(platform["native_argv"].__globals__, windows):
|
|
689
|
+
resolve = platform["native_argv"]
|
|
690
|
+
self.assertEqual(resolve(["npm", "test"]), [str(node), str(nodejs / "node_modules/npm/bin/npm-cli.js"), "test"])
|
|
691
|
+
upgraded = prefix / "node_modules/npm/bin/npm-cli.js" # npm install -g npm: npm.cmd defers to the global prefix's npm.
|
|
692
|
+
upgraded.parent.mkdir(parents=True)
|
|
693
|
+
upgraded.write_bytes(b"")
|
|
694
|
+
self.assertEqual(resolve(["npm", "test"], cwd=self.root), [str(node), str(upgraded), "test"])
|
|
695
|
+
self.assertEqual(lookups[-1], ([str(node), str(nodejs / "node_modules/npm/bin/npm-prefix.js")], self.root))
|
|
696
|
+
self.assertEqual(resolve(["pnpm", "test"]), [str(node), str(tools / "node_modules/pnpm/bin/pnpm.cjs"), "test"])
|
|
697
|
+
self.assertEqual(resolve(["yarn", "test"]), [str(node), str(tools / "node_modules/corepack/dist/yarn.js"), "test"])
|
|
698
|
+
for name in ("vitest", "build"):
|
|
699
|
+
with self.subTest(name=name), self.assertRaisesRegex(OSError, "use argv with an explicit interpreter"):
|
|
700
|
+
resolve([name, "run"])
|
|
701
|
+
|
|
702
|
+
def test_guards_and_blockers(self):
|
|
703
|
+
self.inputs({"verification_commands": self.commands, "required_files": ["missing.txt"], "forbidden_files": ["product.txt"],
|
|
704
|
+
"forbidden_patterns": [{"pattern": "rea+dy", "description": "no ready", "severity": "disqualifier"},
|
|
705
|
+
{"pattern": "ready", "description": "warn only", "severity": "warning"}]})
|
|
706
|
+
source = self.product()
|
|
707
|
+
result = self.submit(source)
|
|
708
|
+
failed = {g["rule"] for g in result["guards"] if not g["passed"]}
|
|
709
|
+
self.assertEqual(failed, {"forbidden_patterns/0", "forbidden_patterns/1", "required_files", "forbidden_files"})
|
|
710
|
+
self.assertNotIn("warn only", " ".join(result["reasons"]))
|
|
711
|
+
blocked = self.submit(source, blockers=[{"kind": "needs-review", "detail": "Which store wins?"}])
|
|
712
|
+
self.assertEqual((blocked["verdict"], blocked["reasons"], blocked["commands"]), ("FAILED", ["needs-review: Which store wins?"], []))
|
|
713
|
+
|
|
714
|
+
def test_max_deps_added_counts_the_names_the_root_package_json_gains(self):
|
|
715
|
+
"""P4-3. Prediction: under the default limit 0 with a passing check, one dependency added to a one-line
|
|
716
|
+
package.json (none at the inputs) counts 1 and fails acceptance; two appended to a multi-line one count 2; a name
|
|
717
|
+
moved from dependencies to devDependencies counts 0 and is accepted; an invalid package.json fails the guard,
|
|
718
|
+
naming it. A leading UTF-8 BOM, which npm and Node accept, is ignored: with one at both commits, one added
|
|
719
|
+
counts 1. A duplicate key (its last value wins) and a null dependencies section, which npm also reads, count 0
|
|
720
|
+
and are accepted; NaN, which npm rejects, fails the guard."""
|
|
721
|
+
passing = {"verification_commands": [{"argv": [sys.executable, "-c", "pass"], "contract_refs": ["R1"]}]}
|
|
722
|
+
|
|
723
|
+
def guard(inputs, source):
|
|
724
|
+
if inputs is not None:
|
|
725
|
+
self.write("package.json", json.dumps(inputs, indent=2))
|
|
726
|
+
self.inputs(passing)
|
|
727
|
+
self.write("package.json", source)
|
|
728
|
+
commit = self.commit("product")
|
|
729
|
+
result = self.submit(commit)
|
|
730
|
+
return commit, result, next(g for g in result["guards"] if g["rule"] == "max_deps_added")
|
|
731
|
+
|
|
732
|
+
with self.subTest("one-line add"):
|
|
733
|
+
_, result, deps = guard(None, json.dumps({"name": "app", "dependencies": {"left-pad": "1.0.0"}}))
|
|
734
|
+
self.assertEqual((deps["subject"], result["reasons"]), ("1 added, limit 0", ["failed: guard max_deps_added 1 added, limit 0"]))
|
|
735
|
+
with self.subTest("multi-line add of 2"):
|
|
736
|
+
_, _, deps = guard({"dependencies": {"a": "1"}}, json.dumps({"dependencies": {"a": "1", "b": "1", "c": "1"}}, indent=2))
|
|
737
|
+
self.assertEqual((deps["subject"], deps["passed"]), ("2 added, limit 0", False))
|
|
738
|
+
with self.subTest("move"):
|
|
739
|
+
_, result, deps = guard({"dependencies": {"a": "1", "b": "1"}, "devDependencies": {"c": "1"}},
|
|
740
|
+
json.dumps({"dependencies": {"b": "1"}, "devDependencies": {"a": "1", "c": "1"}}, indent=2))
|
|
741
|
+
self.assertEqual((deps["subject"], result["verdict"]), ("0 added, limit 0", "ACCEPTED"))
|
|
742
|
+
with self.subTest("invalid"):
|
|
743
|
+
commit, result, deps = guard({"dependencies": {"a": "1"}}, '{"dependencies": {"a": "1",}}\n')
|
|
744
|
+
self.assertEqual((deps["passed"], result["verdict"]), (False, "FAILED"))
|
|
745
|
+
self.assertIn(f"package.json at {commit} is invalid", deps["subject"])
|
|
746
|
+
with self.subTest("BOM"):
|
|
747
|
+
self.write("package.json", "\ufeff" + json.dumps({"dependencies": {"a": "1"}}))
|
|
748
|
+
_, _, deps = guard(None, "\ufeff" + json.dumps({"dependencies": {"a": "1", "b": "1"}}))
|
|
749
|
+
self.assertEqual(deps["subject"], "1 added, limit 0")
|
|
750
|
+
with self.subTest("duplicate key and null"):
|
|
751
|
+
self.write("package.json", '{"dependencies": null, "scripts": {"a": "1"}, "scripts": {"b": "2"}}')
|
|
752
|
+
_, result, deps = guard(None, '{"dependencies": {"a": "1"}, "dependencies": null}')
|
|
753
|
+
self.assertEqual((deps["subject"], result["verdict"]), ("0 added, limit 0", "ACCEPTED"))
|
|
754
|
+
with self.subTest("NaN"):
|
|
755
|
+
commit, result, deps = guard({"dependencies": {"a": "1"}}, '{"config": {"threshold": NaN}}')
|
|
756
|
+
self.assertEqual((deps["passed"], result["verdict"]), (False, "FAILED"))
|
|
757
|
+
self.assertIn(f"package.json at {commit} is invalid", deps["subject"])
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def main():
|
|
761
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
762
|
+
parser.add_argument("--self-test", action="store_true")
|
|
763
|
+
actions = parser.add_subparsers(dest="action")
|
|
764
|
+
runner = actions.add_parser("run")
|
|
765
|
+
runner.add_argument("--packet", required=True)
|
|
766
|
+
runner.add_argument("--source")
|
|
767
|
+
acceptance = actions.add_parser("accept")
|
|
768
|
+
acceptance.add_argument("--packet", required=True)
|
|
769
|
+
acceptance.add_argument("--submission", required=True)
|
|
770
|
+
args = parser.parse_args()
|
|
771
|
+
if args.self_test:
|
|
772
|
+
result = unittest.TextTestRunner(verbosity=1).run(unittest.defaultTestLoader.loadTestsFromTestCase(AcceptanceTests))
|
|
773
|
+
if result.wasSuccessful():
|
|
774
|
+
print(f"acceptance self-test: PASS ({result.testsRun} tests)")
|
|
775
|
+
return 0 if result.wasSuccessful() else 1
|
|
776
|
+
if args.action is None:
|
|
777
|
+
parser.error("run or accept is required")
|
|
778
|
+
try:
|
|
779
|
+
if args.action == "run":
|
|
780
|
+
path, result = run(args.packet, args.source)
|
|
781
|
+
output = {"status": "CHECKED", "result": str(path), "passed": not result["reasons"], "reasons": result["reasons"]}
|
|
782
|
+
else:
|
|
783
|
+
result = accept(args.packet, args.submission)
|
|
784
|
+
output = {"status": result["verdict"], "result": str(Path(load_packet(args.packet)["worktree"]) / LOOP_DIR / "acceptance.json"),
|
|
785
|
+
"reasons": result["reasons"]}
|
|
786
|
+
print(json.dumps(output, sort_keys=True))
|
|
787
|
+
return 0
|
|
788
|
+
except (AcceptanceError, OSError, KeyError, TypeError, ValueError) as exc:
|
|
789
|
+
print(json.dumps({"status": "BLOCKED", "reason": str(exc)}, sort_keys=True))
|
|
790
|
+
return 1
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
if __name__ == "__main__":
|
|
794
|
+
shared("platform-support")["configure_utf8"]()
|
|
795
|
+
sys.exit(main())
|