devlyn-cli 4.3.27 → 4.3.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CLAUDE.md +1 -1
- package/README.md +10 -10
- package/bin/devlyn.js +92 -58
- package/bin/engines.js +5 -3
- package/bin/harness-activation.js +212 -19
- package/bin/instruction-templates.json +2 -0
- package/bin/skill-history.json +11 -0
- package/config/skills/_shared/engine-doctor.sh +2 -2
- package/config/skills/_shared/engine-preflight.md +2 -2
- package/config/skills/_shared/managed-storage.md +80 -0
- package/config/skills/_shared/platform-support.py +67 -28
- package/config/skills/_shared/role-config.py +16 -7
- package/config/skills/_shared/runtime-principles.md +1 -1
- package/config/skills/_shared/storage-client.py +31 -7
- package/config/skills/_shared/task-complete.py +1 -1
- package/config/skills/_shared/task-completion.md +2 -76
- package/config/skills/devlyn-engines/SKILL.md +1 -1
- package/config/skills/devlyn-ideate/SKILL.md +2 -2
- package/config/skills/devlyn-ideate/references/executor.md +62 -0
- package/config/skills/devlyn-ideate/references/loop.md +2 -59
- package/config/skills/devlyn-ideate/references/package-format.md +1 -1
- package/config/skills/devlyn-ideate/scripts/acceptance.py +442 -21
- package/config/skills/devlyn-ideate/scripts/queue.py +58 -10
- package/harness/AGENTS.md +2 -2
- package/harness/CLAUDE.md +2 -2
- package/native/claude/.claude-plugin/plugin.json +1 -1
- package/native/claude/skills/_shared/engine-doctor.sh +2 -2
- package/native/claude/skills/_shared/engine-preflight.md +2 -2
- package/native/claude/skills/_shared/managed-storage.md +80 -0
- package/native/claude/skills/_shared/platform-support.py +67 -28
- package/native/claude/skills/_shared/role-config.py +16 -7
- package/native/claude/skills/_shared/runtime-principles.md +1 -1
- package/native/claude/skills/_shared/storage-client.py +31 -7
- package/native/claude/skills/_shared/task-complete.py +1 -1
- package/native/claude/skills/_shared/task-completion.md +2 -76
- package/native/claude/skills/devlyn-engines/SKILL.md +1 -1
- package/native/claude/skills/devlyn-ideate/SKILL.md +2 -2
- package/native/claude/skills/devlyn-ideate/references/executor.md +62 -0
- package/native/claude/skills/devlyn-ideate/references/loop.md +2 -59
- package/native/claude/skills/devlyn-ideate/references/package-format.md +1 -1
- package/native/claude/skills/devlyn-ideate/scripts/acceptance.py +442 -21
- package/native/claude/skills/devlyn-ideate/scripts/queue.py +58 -10
- package/native/codex/.agents/plugins/marketplace.json +1 -1
- package/native/codex/plugins/devlyn/.codex-plugin/plugin.json +1 -1
- package/native/codex/plugins/devlyn/skills/_shared/engine-doctor.sh +2 -2
- package/native/codex/plugins/devlyn/skills/_shared/engine-preflight.md +2 -2
- package/native/codex/plugins/devlyn/skills/_shared/managed-storage.md +80 -0
- package/native/codex/plugins/devlyn/skills/_shared/platform-support.py +67 -28
- package/native/codex/plugins/devlyn/skills/_shared/role-config.py +16 -7
- package/native/codex/plugins/devlyn/skills/_shared/runtime-principles.md +1 -1
- package/native/codex/plugins/devlyn/skills/_shared/storage-client.py +31 -7
- package/native/codex/plugins/devlyn/skills/_shared/task-complete.py +1 -1
- package/native/codex/plugins/devlyn/skills/_shared/task-completion.md +2 -76
- package/native/codex/plugins/devlyn/skills/devlyn-engines/SKILL.md +1 -1
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/SKILL.md +2 -2
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/executor.md +62 -0
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/loop.md +2 -59
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/package-format.md +1 -1
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/acceptance.py +442 -21
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/queue.py +58 -10
- package/package.json +1 -11
- package/release.json +98 -68
- /package/config/skills/_shared/adapters/{claude.md → claude.adapter.md} +0 -0
- /package/config/skills/_shared/adapters/{codex.md → codex.adapter.md} +0 -0
- /package/config/skills/_shared/adapters/{omp.md → omp.adapter.md} +0 -0
- /package/native/claude/skills/_shared/adapters/{claude.md → claude.adapter.md} +0 -0
- /package/native/claude/skills/_shared/adapters/{codex.md → codex.adapter.md} +0 -0
- /package/native/claude/skills/_shared/adapters/{omp.md → omp.adapter.md} +0 -0
- /package/native/codex/plugins/devlyn/skills/_shared/adapters/{claude.md → claude.adapter.md} +0 -0
- /package/native/codex/plugins/devlyn/skills/_shared/adapters/{codex.md → codex.adapter.md} +0 -0
- /package/native/codex/plugins/devlyn/skills/_shared/adapters/{omp.md → omp.adapter.md} +0 -0
|
@@ -103,7 +103,7 @@ Coverage proves that every obligation has evidence. Whether the checks express t
|
|
|
103
103
|
```
|
|
104
104
|
|
|
105
105
|
- Links are repository-relative and exactly `docs/specs/<loop-id>/<task-id>/spec.md`; titles escape `\`, `[` and `]`. Packets and reports carry absolute paths.
|
|
106
|
-
- `[ ]` is pending or active (the receipt tells which). `[x]` is an accepted product. `[F]` is failed,
|
|
106
|
+
- `[ ]` is pending or active (the receipt tells which). `[x]` is an accepted product. `[F]` is failed, prerequisite-blocked or needs-review and ends with ` — <reason> (receipt <id>)`. Delivery state lives in receipts and the drain report; a pending PR never turns `[x]` into `[F]`.
|
|
107
107
|
- The only legal transition turns one pending row into `[x]` or `[F]` with every other byte unchanged, so a terminal commit changes one line of its loop's file. Terminal rows are never reinterpreted or rerun. The blank lines let transitions of adjacent rows merge, and loops never share a file.
|
|
108
108
|
- Identities are unique; equal titles are fine.
|
|
109
109
|
- `docs/specs/queue.md` is the legacy queue. Its raw rows are intents: readable, never executed, never written (a row there that fully matches this grammar is ignored). `add --materialize <line>` plans one pending legacy row as a loop whose `## Intent` reproduces it verbatim (whitespace may reflow); that loop's queue file starts with `<!-- replaces docs/specs/queue.md row (occurrence <n>): <row> -->`, the row's exact text and its occurrence among equal rows, line endings aside. Every checkout, a fresh clone included, then shows the loop's rows in that row's place. A claim matching no row replaces nothing; a row two loops claim (two clones each materializing it) is hidden, neither loop taking its place, so it is never planned again.
|
|
@@ -7,7 +7,8 @@ requires the owned committed source, reuses the intact outcomes of a wholly clea
|
|
|
7
7
|
runner result for the same source and contract, executes the rest, evaluates
|
|
8
8
|
file/diff guards, confirms the source is unchanged and checks review records. A
|
|
9
9
|
malformed executor record fails acceptance with its reason. An executor's summary
|
|
10
|
-
verdict is never evidence. Protocol: ../references/loop.md
|
|
10
|
+
verdict is never evidence. Protocol: ../references/loop.md; submission and review
|
|
11
|
+
records: ../references/executor.md.
|
|
11
12
|
"""
|
|
12
13
|
from __future__ import annotations
|
|
13
14
|
|
|
@@ -29,7 +30,8 @@ SKILLS = Path(__file__).resolve().parents[2]
|
|
|
29
30
|
LOOP_DIR = ".devlyn/loop"
|
|
30
31
|
QUEUE = "docs/specs/queue.md"
|
|
31
32
|
SHA_RE = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}")
|
|
32
|
-
BLOCKERS = ("failed", "needs-review"
|
|
33
|
+
BLOCKERS = ("failed", "needs-review")
|
|
34
|
+
INFRASTRUCTURE = "blocked-infrastructure" # Earlier instructions offered it; see infrastructure_causes.
|
|
33
35
|
REVIEW_KEYS = {"schema_version", "kind", "task", "engine", "model", "source_sha", "contract_sha256",
|
|
34
36
|
"expected_sha256", "requirements", "findings"}
|
|
35
37
|
# findings, cleanup, handoff and summary are unread; allow them so earlier-template submissions validate.
|
|
@@ -177,27 +179,54 @@ def evaluate(command, outcome, combined):
|
|
|
177
179
|
|
|
178
180
|
|
|
179
181
|
def execute(work, command, index, run_dir):
|
|
182
|
+
helper = shared("platform-support")
|
|
180
183
|
timeout = shared("expected-contract")["verification_timeout_sec"](command)
|
|
181
184
|
env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} # A Python check would leave __pycache__/ in the source it checks.
|
|
185
|
+
paths = {name: run_dir / f"cmd-{index}.{name}" for name in ("stdout", "stderr")}
|
|
186
|
+
# The check writes live files; its evidence is a new copy made once its tree was ended. No process of the check
|
|
187
|
+
# holds an evidence file, so none that teardown could not end changes what is hashed.
|
|
188
|
+
live = {name: path.with_name(path.name + ".live") for name, path in paths.items()}
|
|
182
189
|
try:
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
190
|
+
with live["stdout"].open("xb", buffering=0) as out, live["stderr"].open("xb", buffering=0) as err:
|
|
191
|
+
try:
|
|
192
|
+
if "argv" in command:
|
|
193
|
+
argv = helper["native_argv"](command["argv"], cwd=work)
|
|
194
|
+
elif os.name == "nt": # cmd.exe as shell=True runs it, the command line passed verbatim.
|
|
195
|
+
comspec = os.environ.get("ComSpec") or os.path.join(os.environ.get("SystemRoot", ""), "System32", "cmd.exe")
|
|
196
|
+
if not os.path.isabs(comspec): # A relative shell would resolve inside the worktree under check.
|
|
197
|
+
raise FileNotFoundError("shell not found: neither %ComSpec% nor %SystemRoot% is set")
|
|
198
|
+
argv = f'"{comspec}" /c "{command["cmd"]}"'
|
|
199
|
+
else:
|
|
200
|
+
argv = ["/bin/sh", "-c", command["cmd"]]
|
|
201
|
+
# Its own session or job, ended whole before its output is copied: a process the check leaves behind
|
|
202
|
+
# could otherwise change the worktree under later checks.
|
|
203
|
+
code = helper["run_process"](argv, subprocess.DEVNULL, timeout, on_timeout=None, cwd=work, env=env, stdout=out, stderr=err)
|
|
204
|
+
outcome = ({"kind": "timeout", "timeout_sec": timeout} if code is None else {"kind": "exit", "exit_code": code}
|
|
205
|
+
if code >= 0 else {"kind": "signal", "signal": -code})
|
|
206
|
+
except OSError as exc:
|
|
207
|
+
err.write(str(exc).encode("utf-8"))
|
|
208
|
+
outcome = {"kind": "spawn_error", "error": str(exc)}
|
|
209
|
+
except SystemExit: # A signal ended the run: the live files go too, and one that cannot does not change its exit.
|
|
210
|
+
for path in live.values():
|
|
211
|
+
try:
|
|
212
|
+
path.unlink()
|
|
213
|
+
except OSError as exc:
|
|
214
|
+
print(f"cannot remove {path.name}: {exc}", file=sys.stderr)
|
|
215
|
+
raise
|
|
216
|
+
raw = {name: path.read_bytes() for name, path in live.items()}
|
|
217
|
+
for path in live.values():
|
|
218
|
+
try:
|
|
219
|
+
path.unlink()
|
|
220
|
+
except OSError as exc: # On Windows, while a process still holds it open.
|
|
221
|
+
error = f"cannot remove {path.name}: {exc}"
|
|
222
|
+
raw["stderr"] += error.encode("utf-8")
|
|
223
|
+
outcome = {"kind": "spawn_error", "error": error}
|
|
195
224
|
record = {"index": index, "outcome": outcome}
|
|
196
|
-
for name,
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
record[name] = {"path": relative(work,
|
|
200
|
-
record["passed"], record["reason"] = evaluate(command, outcome,
|
|
225
|
+
for name, data in raw.items():
|
|
226
|
+
with paths[name].open("xb") as stream:
|
|
227
|
+
stream.write(data)
|
|
228
|
+
record[name] = {"path": relative(work, paths[name]), "sha256": sha256(data), "bytes": len(data)}
|
|
229
|
+
record["passed"], record["reason"] = evaluate(command, outcome, raw["stdout"] + raw["stderr"])
|
|
201
230
|
return record
|
|
202
231
|
|
|
203
232
|
|
|
@@ -372,11 +401,27 @@ def check_submission(packet, submission):
|
|
|
372
401
|
raise AcceptanceError(f"submission {key} must be a list of strings")
|
|
373
402
|
blockers = submission.get("blockers", [])
|
|
374
403
|
if not isinstance(blockers, list) or not all(
|
|
375
|
-
isinstance(b, dict) and {"kind", "detail"} <= set(b) and b["kind"] in BLOCKERS
|
|
376
|
-
for b in blockers):
|
|
404
|
+
isinstance(b, dict) and {"kind", "detail"} <= set(b) and b["kind"] in (*BLOCKERS, INFRASTRUCTURE)
|
|
405
|
+
and isinstance(b["detail"], str) and b["detail"].strip() for b in blockers):
|
|
377
406
|
raise AcceptanceError(f"submission blockers must include {{kind, detail}} with kind in {sorted(BLOCKERS)}")
|
|
378
407
|
|
|
379
408
|
|
|
409
|
+
def infrastructure_causes(packet, path):
|
|
410
|
+
"""The details of an otherwise valid submission whose blockers are all infrastructure ones, else None. Its cause
|
|
411
|
+
is one a later run can clear, so the drain retires it once its writers have stopped and runs the task again; with any
|
|
412
|
+
other blocker it fails."""
|
|
413
|
+
try:
|
|
414
|
+
submission = read_json(path)
|
|
415
|
+
blockers = submission.get("blockers") if isinstance(submission, dict) else None
|
|
416
|
+
if not isinstance(blockers, list) or not blockers or not all(
|
|
417
|
+
isinstance(b, dict) and b.get("kind") == INFRASTRUCTURE for b in blockers):
|
|
418
|
+
return None
|
|
419
|
+
check_submission(packet, submission)
|
|
420
|
+
except AcceptanceError:
|
|
421
|
+
return None
|
|
422
|
+
return [b["detail"] for b in blockers]
|
|
423
|
+
|
|
424
|
+
|
|
380
425
|
def result_shell(packet, kind, source):
|
|
381
426
|
return {"schema_version": 1, "kind": kind, "task": packet["task"], "source_sha": source,
|
|
382
427
|
"inputs_sha": packet["inputs_sha"], "allocation_base": packet["allocation_base"],
|
|
@@ -557,6 +602,382 @@ class AcceptanceTests(unittest.TestCase):
|
|
|
557
602
|
self.assertIn("exit 3, expected 0", " ".join(result["reasons"]))
|
|
558
603
|
self.assertTrue(all((self.work / r["stdout"]["path"]).is_file() for r in result["commands"]))
|
|
559
604
|
|
|
605
|
+
def writer_tree(self, name, *, ignore_term=False, escape=False):
|
|
606
|
+
"""argv of a leader that starts a child writing ticks to a file and to its stdout (ignoring SIGTERM, or in a
|
|
607
|
+
session of its own, when asked) and returns once the first tick and a release file exist; the ticks and release
|
|
608
|
+
paths."""
|
|
609
|
+
ticks, release, stop = (self.root / f"{name}.{suffix}" for suffix in ("ticks", "release", "stop"))
|
|
610
|
+
child = ("import pathlib, signal, sys, time\n"
|
|
611
|
+
+ ("signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" if ignore_term else "")
|
|
612
|
+
+ "ticks, stop = map(pathlib.Path, sys.argv[1:])\nwhile not stop.exists():\n"
|
|
613
|
+
" ticks.write_text(str(time.monotonic_ns()))\n print('tick', flush=True)\n time.sleep(0.02)\n")
|
|
614
|
+
leader = ("import pathlib, subprocess, sys, time\n"
|
|
615
|
+
f"subprocess.Popen([sys.executable, '-c', *sys.argv[1:4]], start_new_session={escape})\n"
|
|
616
|
+
"ticks, release = map(pathlib.Path, sys.argv[2:5:2])\n"
|
|
617
|
+
"while not (ticks.exists() and release.exists()):\n time.sleep(0.01)\n")
|
|
618
|
+
self.addCleanup(self.stop_tree, ticks, stop, release)
|
|
619
|
+
return [sys.executable, "-c", leader, child, str(ticks), str(stop), str(release)], ticks, release
|
|
620
|
+
|
|
621
|
+
def stop_tree(self, ticks, *files):
|
|
622
|
+
"""End a tree a failed run left behind before the temporary directory goes: its leader returns on the release
|
|
623
|
+
file, its child on the stop file."""
|
|
624
|
+
import time
|
|
625
|
+
for path in files:
|
|
626
|
+
path.touch()
|
|
627
|
+
deadline = time.monotonic() + 5
|
|
628
|
+
while ticks.exists() and time.monotonic() < deadline:
|
|
629
|
+
before = ticks.read_bytes()
|
|
630
|
+
time.sleep(0.05)
|
|
631
|
+
if ticks.read_bytes() == before:
|
|
632
|
+
break
|
|
633
|
+
|
|
634
|
+
def assert_ended(self, ticks, record=None):
|
|
635
|
+
"""The ticks have stopped and the stdout evidence, if any, still has its recorded hash."""
|
|
636
|
+
import time
|
|
637
|
+
before = ticks.read_bytes()
|
|
638
|
+
time.sleep(0.3)
|
|
639
|
+
self.assertEqual(ticks.read_bytes(), before, "a process the check started is still writing")
|
|
640
|
+
if record:
|
|
641
|
+
stdout = (self.work / record["stdout"]["path"]).read_bytes()
|
|
642
|
+
self.assertEqual((sha256(stdout), b"tick" in stdout), (record["stdout"]["sha256"], True))
|
|
643
|
+
|
|
644
|
+
@unittest.skipIf(os.name == "nt", "a POSIX session; test-windows-portability.py covers the Windows job")
|
|
645
|
+
def test_a_check_ends_with_every_process_it_started(self):
|
|
646
|
+
"""C5. Prediction: a check whose leader leaves a child writing ticks to a file and to its stdout is recorded as it
|
|
647
|
+
ended (a timeout, or exit 0 for a leader that returns at once), and once execute returns the child is gone: the
|
|
648
|
+
ticks stop and the stdout evidence still has its recorded hash. The check's clock stands still until the first
|
|
649
|
+
tick, so a slow interpreter start cannot time it out before the child writes. `exit 124` is an exit with code
|
|
650
|
+
124. Before: a timeout killed only the leader and a normal exit nothing, so the child kept writing, and because
|
|
651
|
+
it held the pipes the leader's prompt exit 0 was recorded as a timeout."""
|
|
652
|
+
import time
|
|
653
|
+
from types import SimpleNamespace
|
|
654
|
+
from unittest import mock
|
|
655
|
+
scope = shared("platform-support")["run_process"].__globals__
|
|
656
|
+
run_dir = new_run_dir(self.work)
|
|
657
|
+
for index, (mode, seconds, outcome) in enumerate((("hang", 1, {"kind": "timeout", "timeout_sec": 1}),
|
|
658
|
+
("exit", 30, {"kind": "exit", "exit_code": 0}))):
|
|
659
|
+
with self.subTest(mode=mode):
|
|
660
|
+
argv, ticks, release = self.writer_tree(mode)
|
|
661
|
+
if mode == "exit":
|
|
662
|
+
release.touch()
|
|
663
|
+
ready = []
|
|
664
|
+
|
|
665
|
+
def monotonic():
|
|
666
|
+
if not ready and ticks.exists():
|
|
667
|
+
ready.append(time.monotonic())
|
|
668
|
+
return time.monotonic() - ready[0] if ready else 0.0
|
|
669
|
+
with mock.patch.dict(scope, time=SimpleNamespace(monotonic=monotonic, sleep=time.sleep)):
|
|
670
|
+
record = execute(self.work, {"argv": argv, "timeout_sec": seconds}, index, run_dir)
|
|
671
|
+
self.assertEqual(record["outcome"], outcome)
|
|
672
|
+
self.assert_ended(ticks, record)
|
|
673
|
+
record = execute(self.work, {"cmd": "exit 124", "exit_code": 124}, 2, run_dir)
|
|
674
|
+
self.assertEqual((record["outcome"], record["passed"]), ({"kind": "exit", "exit_code": 124}, True))
|
|
675
|
+
|
|
676
|
+
@unittest.skipIf(os.name == "nt", "a POSIX session; test-windows-portability.py covers the Windows job")
|
|
677
|
+
def test_a_check_returns_once_a_writer_that_ignores_sigterm_is_gone(self):
|
|
678
|
+
"""C5 review. Prediction: a leader that exits at once, leaving a child that ignores SIGTERM and keeps writing,
|
|
679
|
+
is recorded as exit 0 only once that child is gone (the 5 s grace, then SIGKILL): its ticks and the stdout
|
|
680
|
+
evidence are stable when execute returns, also when SIGKILL takes effect a moment after killpg returned (kill
|
|
681
|
+
only queues it; modelled here by sending it 0.5 s late). Before: execute returned as soon as SIGKILL was sent,
|
|
682
|
+
while the writer still ran."""
|
|
683
|
+
import contextlib
|
|
684
|
+
import signal
|
|
685
|
+
import threading
|
|
686
|
+
from unittest import mock
|
|
687
|
+
real, timers = os.killpg, []
|
|
688
|
+
|
|
689
|
+
def late_kill(pgid, sig):
|
|
690
|
+
if sig != signal.SIGKILL:
|
|
691
|
+
return real(pgid, sig)
|
|
692
|
+
|
|
693
|
+
def kill():
|
|
694
|
+
with contextlib.suppress(ProcessLookupError):
|
|
695
|
+
real(pgid, sig)
|
|
696
|
+
timers.append(threading.Timer(0.5, kill))
|
|
697
|
+
timers[-1].start()
|
|
698
|
+
argv, ticks, release = self.writer_tree("survivor", ignore_term=True)
|
|
699
|
+
release.touch()
|
|
700
|
+
with mock.patch.object(os, "killpg", late_kill):
|
|
701
|
+
record = execute(self.work, {"argv": argv, "timeout_sec": 30}, 0, new_run_dir(self.work))
|
|
702
|
+
self.assertEqual(record["outcome"], {"kind": "exit", "exit_code": 0})
|
|
703
|
+
self.assert_ended(ticks, record)
|
|
704
|
+
for timer in timers:
|
|
705
|
+
timer.join()
|
|
706
|
+
|
|
707
|
+
@unittest.skipIf(os.name == "nt", "a POSIX session; a Windows job has no breakaway")
|
|
708
|
+
def test_a_process_that_left_the_session_cannot_change_the_evidence(self):
|
|
709
|
+
"""C5 review 2. Prediction: a check whose child leaves the session by its own setsid and keeps writing to its
|
|
710
|
+
inherited stdout is recorded as exit 0; when execute returns, the stdout evidence has its recorded hash and keeps
|
|
711
|
+
it while that child still writes, and no live file is left. Before: the evidence was the file the check wrote
|
|
712
|
+
to, so the escaped child's writes landed in it after it was hashed."""
|
|
713
|
+
import time
|
|
714
|
+
argv, ticks, release = self.writer_tree("escaped", escape=True)
|
|
715
|
+
release.touch()
|
|
716
|
+
run_dir = new_run_dir(self.work)
|
|
717
|
+
record = execute(self.work, {"argv": argv, "timeout_sec": 30}, 0, run_dir)
|
|
718
|
+
self.assertEqual(record["outcome"], {"kind": "exit", "exit_code": 0})
|
|
719
|
+
evidence, written = self.work / record["stdout"]["path"], ticks.read_bytes()
|
|
720
|
+
self.assertEqual(sha256(evidence.read_bytes()), record["stdout"]["sha256"])
|
|
721
|
+
deadline = time.monotonic() + 30
|
|
722
|
+
while ticks.read_bytes() == written:
|
|
723
|
+
self.assertLess(time.monotonic(), deadline, "the escaped child stopped writing, so nothing was tested")
|
|
724
|
+
time.sleep(0.01)
|
|
725
|
+
self.assertEqual(sha256(evidence.read_bytes()), record["stdout"]["sha256"])
|
|
726
|
+
self.assertEqual(sorted(path.name for path in run_dir.iterdir()), ["cmd-0.stderr", "cmd-0.stdout"])
|
|
727
|
+
|
|
728
|
+
def test_a_live_file_that_cannot_be_removed_fails_its_check(self):
|
|
729
|
+
"""C5 review 2. Prediction, with removal refused as Windows refuses a file a process still holds open: the check
|
|
730
|
+
fails with an error naming that live file, which its stderr evidence also carries. Before: the check wrote its
|
|
731
|
+
evidence directly, with no live file to leave behind."""
|
|
732
|
+
from unittest import mock
|
|
733
|
+
unlink = Path.unlink
|
|
734
|
+
|
|
735
|
+
def refuse(path, *args, **kwargs):
|
|
736
|
+
if path.name.endswith(".stdout.live"):
|
|
737
|
+
raise PermissionError(13, "fixture: in use by another process", str(path))
|
|
738
|
+
return unlink(path, *args, **kwargs)
|
|
739
|
+
with mock.patch.object(Path, "unlink", refuse):
|
|
740
|
+
record = execute(self.work, {"argv": [sys.executable, "-c", "print('ok')"]}, 0, new_run_dir(self.work))
|
|
741
|
+
self.assertEqual((record["outcome"]["kind"], record["passed"]), ("spawn_error", False))
|
|
742
|
+
self.assertIn("cannot remove cmd-0.stdout.live", record["outcome"]["error"])
|
|
743
|
+
self.assertIn(record["outcome"]["error"].encode("utf-8"), (self.work / record["stderr"]["path"]).read_bytes())
|
|
744
|
+
|
|
745
|
+
@unittest.skipIf(os.name == "nt", "test-windows-portability.py covers teardown errors of the Windows job")
|
|
746
|
+
def test_a_signal_ends_the_run_whatever_teardown_raises(self):
|
|
747
|
+
"""C5 review 2. Prediction: SIGTERM during a check's run ends it with 143 even when teardown then raises OSError,
|
|
748
|
+
so execute returns no record and the run goes no further. Its live files are removed; one that cannot be is
|
|
749
|
+
named on stderr and the exit stays 143. Before: the teardown error replaced the exit, and execute recorded it as
|
|
750
|
+
a failed check and went on; then (review 2) both live files stayed in the run directory."""
|
|
751
|
+
import contextlib
|
|
752
|
+
import io
|
|
753
|
+
import signal
|
|
754
|
+
from unittest import mock
|
|
755
|
+
scope = shared("platform-support")["run_process"].__globals__
|
|
756
|
+
started, unlink = [], Path.unlink
|
|
757
|
+
|
|
758
|
+
class InterruptedRun(subprocess.Popen):
|
|
759
|
+
def wait(self, timeout=None):
|
|
760
|
+
if self not in started:
|
|
761
|
+
started.append(self)
|
|
762
|
+
signal.raise_signal(signal.SIGTERM)
|
|
763
|
+
return super().wait(timeout)
|
|
764
|
+
|
|
765
|
+
def failing_teardown(child, job=None):
|
|
766
|
+
raise OSError("fixture teardown failure")
|
|
767
|
+
|
|
768
|
+
def refuse(path, *args, **kwargs):
|
|
769
|
+
if path.name.endswith(".stdout.live"):
|
|
770
|
+
raise PermissionError(13, "fixture: in use by another process", str(path))
|
|
771
|
+
return unlink(path, *args, **kwargs)
|
|
772
|
+
self.addCleanup(lambda: [(process.kill(), process.wait()) for process in started])
|
|
773
|
+
for refused in (False, True):
|
|
774
|
+
with self.subTest(refused=refused):
|
|
775
|
+
run_dir, stderr = new_run_dir(self.work), io.StringIO()
|
|
776
|
+
with mock.patch.object(subprocess, "Popen", InterruptedRun), mock.patch.dict(scope, terminate_tree=failing_teardown), \
|
|
777
|
+
mock.patch.object(Path, "unlink", refuse if refused else unlink), contextlib.redirect_stderr(stderr), \
|
|
778
|
+
self.assertRaises(SystemExit) as stop:
|
|
779
|
+
execute(self.work, {"argv": [sys.executable, "-c", "import time; time.sleep(30)"]}, 0, run_dir)
|
|
780
|
+
self.assertEqual(stop.exception.code, 143)
|
|
781
|
+
self.assertEqual(sorted(path.name for path in run_dir.iterdir()), ["cmd-0.stdout.live"] if refused else [])
|
|
782
|
+
self.assertEqual("cannot remove cmd-0.stdout.live" in stderr.getvalue(), refused, stderr.getvalue())
|
|
783
|
+
|
|
784
|
+
@unittest.skipIf(os.name == "nt", "POSIX signals; a Windows job ends its tree however the runner stops")
|
|
785
|
+
def test_a_runner_stopped_by_a_signal_ends_the_check_tree(self):
|
|
786
|
+
"""C5 review. Prediction, with run_process driven in its own Python process: SIGHUP (a closed terminal) ends it
|
|
787
|
+
with 129 and the tree with it; SIGHUP inherited as ignored (nohup) stays ignored and the run completes with 0;
|
|
788
|
+
a second SIGTERM during the grace of a child that ignores SIGTERM waits for teardown, which still kills that
|
|
789
|
+
child, and the run ends with 143. Before: SIGHUP's default action killed the runner and the tree ran on, and a
|
|
790
|
+
second SIGTERM aborted teardown before SIGKILL."""
|
|
791
|
+
import signal
|
|
792
|
+
import time
|
|
793
|
+
# The runner touches a marker as teardown starts, so the second SIGTERM lands within the surviving child's grace.
|
|
794
|
+
runner = ("import pathlib, runpy, subprocess, sys\n"
|
|
795
|
+
"run_process = runpy.run_path(sys.argv[1])['run_process']\n"
|
|
796
|
+
"scope, marker = run_process.__globals__, pathlib.Path(sys.argv[2])\n"
|
|
797
|
+
"def teardown(*args, real=scope['terminate_tree']):\n marker.touch()\n return real(*args)\n"
|
|
798
|
+
"scope['terminate_tree'] = teardown\n"
|
|
799
|
+
"sys.exit(run_process(sys.argv[3:], subprocess.DEVNULL, 60))\n")
|
|
800
|
+
|
|
801
|
+
def wait_for(path):
|
|
802
|
+
while not path.exists():
|
|
803
|
+
self.assertLess(time.monotonic(), deadline)
|
|
804
|
+
time.sleep(0.01)
|
|
805
|
+
for case in ("hangup", "ignored hangup", "second sigterm"):
|
|
806
|
+
with self.subTest(case=case):
|
|
807
|
+
argv, ticks, release = self.writer_tree(case.replace(" ", "-"), ignore_term=case == "second sigterm")
|
|
808
|
+
tearing = self.root / f"{case.replace(' ', '-')}.teardown"
|
|
809
|
+
previous = signal.signal(signal.SIGHUP, signal.SIG_IGN if case == "ignored hangup" else signal.SIG_DFL)
|
|
810
|
+
try:
|
|
811
|
+
driver = subprocess.Popen([sys.executable, "-c", runner, str(SKILLS / "_shared/platform-support.py"), str(tearing),
|
|
812
|
+
*argv], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
|
|
813
|
+
finally:
|
|
814
|
+
signal.signal(signal.SIGHUP, previous)
|
|
815
|
+
self.addCleanup(lambda driver=driver: (driver.kill(), driver.wait()))
|
|
816
|
+
deadline = time.monotonic() + 30
|
|
817
|
+
wait_for(ticks)
|
|
818
|
+
if case == "second sigterm":
|
|
819
|
+
driver.send_signal(signal.SIGTERM)
|
|
820
|
+
wait_for(tearing)
|
|
821
|
+
driver.send_signal(signal.SIGTERM)
|
|
822
|
+
expected = 143
|
|
823
|
+
else:
|
|
824
|
+
driver.send_signal(signal.SIGHUP)
|
|
825
|
+
expected = 129
|
|
826
|
+
if case == "ignored hangup":
|
|
827
|
+
time.sleep(0.3)
|
|
828
|
+
self.assertIsNone(driver.poll())
|
|
829
|
+
release.touch()
|
|
830
|
+
expected = 0
|
|
831
|
+
self.assertEqual(driver.wait(timeout=30), expected)
|
|
832
|
+
self.assert_ended(ticks)
|
|
833
|
+
|
|
834
|
+
@unittest.skipIf(os.name == "nt", "test-windows-portability.py covers a signal while the Windows job starts")
|
|
835
|
+
def test_a_signal_while_the_check_starts_ends_its_tree(self):
|
|
836
|
+
"""C5 review. Prediction: SIGTERM raised after the fork, before Popen returns (where CPython closes the child's
|
|
837
|
+
pipe ends), waits until Popen is done, then ends the run with 143: a started session ends with it, and a child
|
|
838
|
+
whose exec failed is reaped by Popen itself. Before: the handler raised inside Popen, so the started session was
|
|
839
|
+
never ended and the failed child was left unreaped."""
|
|
840
|
+
import signal
|
|
841
|
+
from unittest import mock
|
|
842
|
+
argv, ticks, _ = self.writer_tree("start")
|
|
843
|
+
started = []
|
|
844
|
+
|
|
845
|
+
class InterruptedStart(subprocess.Popen):
|
|
846
|
+
def _close_pipe_fds(self, *args):
|
|
847
|
+
super()._close_pipe_fds(*args)
|
|
848
|
+
started.append(self)
|
|
849
|
+
signal.raise_signal(signal.SIGTERM)
|
|
850
|
+
self.addCleanup(lambda: [(process.kill(), process.wait()) for process in started])
|
|
851
|
+
for count, target in enumerate((argv, ["devlyn-missing-binary-for-test"]), 1):
|
|
852
|
+
with self.subTest(target=target[-1]):
|
|
853
|
+
with mock.patch.object(subprocess, "Popen", InterruptedStart), self.assertRaises(SystemExit) as stop:
|
|
854
|
+
shared("platform-support")["run_process"](target, subprocess.DEVNULL, 60, stdout=subprocess.DEVNULL)
|
|
855
|
+
self.assertEqual((stop.exception.code, len(started)), (143, count))
|
|
856
|
+
self.assertIsNotNone(started[-1].returncode, "the started process was neither ended nor reaped")
|
|
857
|
+
if ticks.exists():
|
|
858
|
+
self.assert_ended(ticks)
|
|
859
|
+
|
|
860
|
+
def test_a_signal_as_its_handler_is_installed_restores_every_handler(self):
|
|
861
|
+
"""C5 review 3. Prediction, with signal.signal delivering the signal right after it installs run_process's
|
|
862
|
+
handler: the run ends with 128 + that signal, and every handled signal has the handler it had before the call,
|
|
863
|
+
after the first installation and after a later one. Before: a handler was installed before its predecessor was
|
|
864
|
+
saved, so a signal in between left it installed."""
|
|
865
|
+
import signal
|
|
866
|
+
from unittest import mock
|
|
867
|
+
real = signal.signal
|
|
868
|
+
handled = [signal.SIGTERM, signal.SIGINT, *([signal.SIGHUP] if os.name != "nt" else [])]
|
|
869
|
+
before = {sig: signal.getsignal(sig) for sig in handled}
|
|
870
|
+
self.addCleanup(lambda: [real(sig, handler) for sig, handler in before.items()])
|
|
871
|
+
for target in (signal.SIGTERM, signal.SIGINT):
|
|
872
|
+
with self.subTest(signal=target.name):
|
|
873
|
+
delivered = []
|
|
874
|
+
|
|
875
|
+
def deliver(sig, handler):
|
|
876
|
+
previous = real(sig, handler)
|
|
877
|
+
if sig == target and not delivered:
|
|
878
|
+
delivered.append(sig)
|
|
879
|
+
handler(sig, None)
|
|
880
|
+
return previous
|
|
881
|
+
with mock.patch.object(signal, "signal", deliver), self.assertRaises(SystemExit) as stop:
|
|
882
|
+
shared("platform-support")["run_process"]([sys.executable, "-c", "pass"], subprocess.DEVNULL, 60)
|
|
883
|
+
self.assertEqual((stop.exception.code, delivered), (128 + target, [target]))
|
|
884
|
+
self.assertEqual({sig: signal.getsignal(sig) for sig in handled}, before)
|
|
885
|
+
|
|
886
|
+
@unittest.skipIf(os.name == "nt", "POSIX process groups")
|
|
887
|
+
def test_teardown_ends_once_the_group_is_gone_and_raises_only_for_a_leader_it_cannot_signal(self):
|
|
888
|
+
"""C5 review. Prediction, with killpg and the clock mocked: a group gone after SIGTERM and two probes ends
|
|
889
|
+
teardown at once, with no SIGKILL and none of the grace used; EPERM while the leader is alive (a leader this
|
|
890
|
+
process may not signal) is raised; EPERM once the leader has exited leaves nothing to signal, so teardown ends
|
|
891
|
+
after one SIGTERM; a group that still answers signal 0 after SIGTERM's grace and SIGKILL (dead but unreaped, or
|
|
892
|
+
not signallable) gets about one more second, then teardown returns. Before: every EPERM read as gone, followed
|
|
893
|
+
by an unbounded wait for the leader, and teardown always slept the full grace (first C5 commit); then (review 1)
|
|
894
|
+
such a group raised OSError, failing a check whose processes were all dead."""
|
|
895
|
+
import errno
|
|
896
|
+
import signal
|
|
897
|
+
from types import SimpleNamespace
|
|
898
|
+
from unittest import mock
|
|
899
|
+
scope = shared("platform-support")["terminate_tree"].__globals__
|
|
900
|
+
clock, sent = [0.0], []
|
|
901
|
+
|
|
902
|
+
class Leader:
|
|
903
|
+
pid = 4242 # killpg is mocked: nothing is signalled.
|
|
904
|
+
|
|
905
|
+
def __init__(self, code):
|
|
906
|
+
self.code = code
|
|
907
|
+
|
|
908
|
+
def poll(self):
|
|
909
|
+
return self.code
|
|
910
|
+
|
|
911
|
+
def wait(self, timeout=None): # The first C5 teardown waited for the leader.
|
|
912
|
+
return self.code
|
|
913
|
+
|
|
914
|
+
def teardown(refusal, code, answers=0): # The group answers `answers` signals, then killpg raises `refusal`, if any.
|
|
915
|
+
def killpg(pgid, sig):
|
|
916
|
+
sent.append(sig)
|
|
917
|
+
if refusal and len(sent) > answers:
|
|
918
|
+
raise refusal
|
|
919
|
+
sent.clear()
|
|
920
|
+
fake = SimpleNamespace(monotonic=lambda: clock[0], sleep=lambda seconds: clock.__setitem__(0, clock[0] + seconds))
|
|
921
|
+
with mock.patch.object(os, "killpg", killpg), mock.patch.dict(scope, time=fake):
|
|
922
|
+
scope["terminate_tree"](Leader(code))
|
|
923
|
+
teardown(ProcessLookupError(errno.ESRCH, "No such process"), 0, answers=3)
|
|
924
|
+
self.assertLess(clock[0], 0.1)
|
|
925
|
+
self.assertEqual(sent, [signal.SIGTERM, 0, 0, 0])
|
|
926
|
+
refused = PermissionError(errno.EPERM, "Operation not permitted")
|
|
927
|
+
with self.assertRaises(PermissionError):
|
|
928
|
+
teardown(refused, None)
|
|
929
|
+
teardown(refused, 0)
|
|
930
|
+
self.assertEqual(sent, [signal.SIGTERM])
|
|
931
|
+
start = clock[0]
|
|
932
|
+
teardown(None, 0)
|
|
933
|
+
self.assertEqual((sent[0], sent.count(signal.SIGKILL), set(sent)), (signal.SIGTERM, 1, {signal.SIGTERM, 0, signal.SIGKILL}))
|
|
934
|
+
self.assertAlmostEqual(clock[0] - start, 6, delta=0.1)
|
|
935
|
+
|
|
936
|
+
def test_windows_shell_commands_reach_cmd_verbatim(self):
|
|
937
|
+
"""Windows mocked: a cmd check runs ComSpec, else SystemRoot's cmd.exe, as `/c "<cmd>"` with the command untouched,
|
|
938
|
+
the line shell=True builds, so its quotes reach cmd.exe as written; a shell that is not absolute is a spawn error,
|
|
939
|
+
never a search that could find one in the worktree."""
|
|
940
|
+
import ntpath
|
|
941
|
+
from types import SimpleNamespace
|
|
942
|
+
from unittest import mock
|
|
943
|
+
launched = []
|
|
944
|
+
|
|
945
|
+
def run_process(argv, stdin, seconds, **popen):
|
|
946
|
+
launched.append(argv)
|
|
947
|
+
return 0
|
|
948
|
+
cmd = 'echo "a b" & exit 0'
|
|
949
|
+
for environ, shell in (({"ComSpec": r"C:\Windows\system32\cmd.exe", "SystemRoot": r"D:\Other"}, r"C:\Windows\system32\cmd.exe"),
|
|
950
|
+
({"SystemRoot": r"C:\Windows"}, r"C:\Windows\System32\cmd.exe"), ({"ComSpec": "cmd.exe"}, None), ({}, None)):
|
|
951
|
+
with self.subTest(environ=environ), mock.patch.dict(shared("platform-support"), run_process=run_process), \
|
|
952
|
+
mock.patch.dict(execute.__globals__, os=SimpleNamespace(name="nt", environ=environ, path=ntpath)):
|
|
953
|
+
launched.clear()
|
|
954
|
+
record = execute(self.work, {"cmd": cmd}, 0, new_run_dir(self.work))
|
|
955
|
+
if shell:
|
|
956
|
+
self.assertEqual((launched, record["outcome"]), ([f'"{shell}" /c "{cmd}"'], {"kind": "exit", "exit_code": 0}))
|
|
957
|
+
else:
|
|
958
|
+
self.assertEqual((launched, record["outcome"]["kind"]), ([], "spawn_error"))
|
|
959
|
+
self.assertIn("shell not found", record["outcome"]["error"])
|
|
960
|
+
|
|
961
|
+
def test_only_infrastructure_blockers_wait_for_another_run(self):
|
|
962
|
+
"""C5. Prediction: an otherwise valid submission blocked only by infrastructure yields its details, so the drain
|
|
963
|
+
retires it and runs the task again; with a product blocker as well, invalid fields or no readable file it yields
|
|
964
|
+
none, and the mixed one still fails with both reasons. Before: any infrastructure blocker failed the task for good."""
|
|
965
|
+
self.inputs()
|
|
966
|
+
source = self.product()
|
|
967
|
+
infrastructure = {"kind": "blocked-infrastructure", "detail": "registry unreachable"}
|
|
968
|
+
mixed = [infrastructure, {"kind": "failed", "detail": "broken"}]
|
|
969
|
+
path = self.root / "submission.json"
|
|
970
|
+
|
|
971
|
+
def causes(**fields):
|
|
972
|
+
path.write_text(json.dumps({"schema_version": 1, "task": "l.t", "source_sha": source, **fields}), encoding="utf-8")
|
|
973
|
+
return infrastructure_causes(self.packet, path)
|
|
974
|
+
self.assertEqual(causes(blockers=[infrastructure, dict(infrastructure, detail="login expired")]),
|
|
975
|
+
["registry unreachable", "login expired"])
|
|
976
|
+
self.assertEqual([causes(blockers=mixed), causes(blockers=[infrastructure], reviews="x"), causes(),
|
|
977
|
+
infrastructure_causes(self.packet, self.root / "missing.json")], [None] * 4)
|
|
978
|
+
result = self.submit(source, blockers=mixed)
|
|
979
|
+
self.assertEqual((result["verdict"], result["reasons"]), ("FAILED", ["blocked-infrastructure: registry unreachable", "failed: broken"]))
|
|
980
|
+
|
|
560
981
|
def test_source_and_contract_binding(self):
|
|
561
982
|
self.inputs()
|
|
562
983
|
source = self.product()
|