devlyn-cli 4.2.24 → 4.2.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/skill-history.json
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
"21723943c52573124ce3c74e9090a8ae7d7c9ecdeb78fdfa52b89c45ff1b3374",
|
|
24
24
|
"247904cad29c1c3ed810f848e66f982e8765e5c627ef3a18a876fd95eed9c29c",
|
|
25
25
|
"24f73bd06d359296088e43ce2e7bf5178e3f69752b559639585563ec35fc33f5",
|
|
26
|
+
"2692865ee864aebedb2b385384b2a8220bc780bd92310bc1cf2f56927de1e0b8",
|
|
26
27
|
"27d4da7863ed4652187f6a101da0e56b1d3d18e4565c720d5f8150ad0a193402",
|
|
27
28
|
"2a4361bca0a9fadc4a8e5e4578c383171cd64c65296fe882990502250df44d53",
|
|
28
29
|
"2bcac1ab5995e474c7dd46705413982876384d8b4da704126a66c10770a1a4d2",
|
|
@@ -33,6 +34,7 @@
|
|
|
33
34
|
"353b483942a62b79d1c5bf31e30060ef7ed9a67ae2d8b27450a53acb98e4e23c",
|
|
34
35
|
"3554398c2be144938e4837d9b60c0976f685cfc2c0dd0451cda714fd053f3c7d",
|
|
35
36
|
"35c7e6c069c847ff147f2bf4f3b67a296652c122aa91a3966457a00dd0597b78",
|
|
37
|
+
"364aa97ec4065d8b78188f40281dfa575120a8fb91089120363e532566ef0dcb",
|
|
36
38
|
"37acccbdc76aee15311b5f47fd26c50b1652dbe7cbcbacfc37774841ab464028",
|
|
37
39
|
"393ad728eb0906ae61110fde3820c4fce015d8f94643a2877372b74a6b6eeb02",
|
|
38
40
|
"3b6193eabc83d815804533010ca7bff1f240286e7898380a5fc2adac71dcb513",
|
|
@@ -100,6 +102,7 @@
|
|
|
100
102
|
"9162fc2fed6f4a611d299485be3a397080beb3304865ce59b95576ba7d00650e",
|
|
101
103
|
"94a14a1b87df1d7297d12949a24e47a0d6c16a8fe30495ee759fcabb2e1aa398",
|
|
102
104
|
"978e4210e49b42827c98cf6e40c868c87dc6e3533a2b779ca2fa05abfc16ad0f",
|
|
105
|
+
"97e7ad89c74c609b15dfbf6f9f7ac61df8b50e97e4bc318f72922e058ecffba4",
|
|
103
106
|
"9815b6e6963e068872a6dbeae24068143bf92042bb035f67bafd1a224d65bcb0",
|
|
104
107
|
"998343c148618b937556b58b83749c05d5aa3283280d751c118da6a542fb87f2",
|
|
105
108
|
"9dd3ec225818efe0ad3726192db94ff129154e995617a9d3c150fc58ad0f9f2c",
|
|
@@ -343,6 +346,7 @@
|
|
|
343
346
|
"a7d7d65fd5f672cb404e6069fdc42ad7077deaa80ed8500cfa8b382dab2e3e29",
|
|
344
347
|
"aba66b674683935e4c28c93333896dc116ffe67e7a35fe30b3599a8be507e94e",
|
|
345
348
|
"b2e4437fa981317362d03ee032e1cfd8b686316e966c6ba12ecd619a009c80a6",
|
|
349
|
+
"bdd0751eea04532c050334fda3bc5c61c6b429e485d97e0b8ebb4fae154da2c7",
|
|
346
350
|
"c1b8e3ebf38e2be8972aa003e9614a782cb4f195c69d20b95a475ebf9aaaab9e",
|
|
347
351
|
"c8e375aab574148542f392b093cbbf3f2e8dc623826b45e35df0a1a2c3862226",
|
|
348
352
|
"ce282a55328c2d071cacc377670fbebea4da4b9a7fee81b25293fb26763f17c9",
|
|
@@ -264,7 +264,12 @@ def allocate(args, *, exclude_receipts=()):
|
|
|
264
264
|
receipt["worktree_identity"] = workspace_identity(target, Path(receipt["worktree_gitdir"]))
|
|
265
265
|
receipt["allocation"] = "owned"
|
|
266
266
|
atomic_json(path, receipt)
|
|
267
|
+
# Owned first: an interrupted warm-up leaves a usable allocation whose build starts cold.
|
|
268
|
+
with locked_receipt(path) as owned:
|
|
269
|
+
owned["build_dirs"] = shared("warm-build-dirs")["warm"](target, work, scratch)
|
|
270
|
+
atomic_json(path, owned)
|
|
267
271
|
return {"status": "ALLOCATED", "receipt": str(path), "worktree": str(target), "scratch": str(scratch),
|
|
272
|
+
"build_dirs": owned["build_dirs"],
|
|
268
273
|
"reconciled": [] if local else reconcile(common, path, work, exclude_receipts=exclude_receipts)}
|
|
269
274
|
|
|
270
275
|
|
|
@@ -106,10 +106,21 @@ require the affected checks again.
|
|
|
106
106
|
Keep the original edits through pending or failed delivery; only reconciliation
|
|
107
107
|
removes them. Preserve all other work and report anything left behind.
|
|
108
108
|
|
|
109
|
-
|
|
110
|
-
build
|
|
111
|
-
|
|
112
|
-
|
|
109
|
+
Build in the task worktree's own default build directory, never in one
|
|
110
|
+
another checkout builds in, even one build at a time: tools such as Cargo take
|
|
111
|
+
one checkout's outputs as fresh for the other. When a Cargo workspace's
|
|
112
|
+
Cargo.toml sets `[workspace.metadata.devlyn] warm-build-dirs = true`,
|
|
113
|
+
allocation seeds that directory from a built checkout of the repository, so
|
|
114
|
+
the first build does not rebuild every dependency; `build_dirs` in its result
|
|
115
|
+
and receipt says what it did, and a skip means an ordinary cold build. The
|
|
116
|
+
opt-in is the workspace's: seeding keeps dependency outputs that may hold the
|
|
117
|
+
other checkout's paths, and those follow that checkout's later rebuilds. `warm-build-dirs.py <worktree>` does the same for a
|
|
118
|
+
worktree made without allocation. A warmed directory leaves with the worktree
|
|
119
|
+
after merge, so a pending, failed or local-only task keeps it.
|
|
120
|
+
|
|
121
|
+
Allocation also returns a receipt-owned `scratch` directory for other
|
|
122
|
+
disposable build files, such as writable tool caches. Keep source checkouts,
|
|
123
|
+
Git data, lockfiles, reports and restart evidence outside scratch. Do not create unowned
|
|
113
124
|
temporary build trees or preserve them by copying them into evidence custody.
|
|
114
125
|
Prefer `TemporaryDirectory`/`finally` for short-lived test fixtures.
|
|
115
126
|
|
|
@@ -0,0 +1,899 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Start a new worktree's build directory warm, from a built checkout (task-completion.md).
|
|
3
|
+
|
|
4
|
+
Of the common build tools only Cargo keeps a large cache inside each checkout that a new
|
|
5
|
+
worktree starts without: pnpm, bun, npm, Go and uv share content-addressed caches machine-wide,
|
|
6
|
+
Turborepo shares the main worktree's cache, and the path-keyed caches of Vite, Gradle, Xcode and
|
|
7
|
+
Flutter would miss after a copy. For a Cargo workspace whose default target directory is absent
|
|
8
|
+
and ignored, this clones the dev-profile layouts of the anchor's (else another registered
|
|
9
|
+
worktree's) built target directory into private staging, sharing its blocks (APFS clonefile,
|
|
10
|
+
Linux reflink; never a copy), removes what must not carry over, and moves the result into place:
|
|
11
|
+
|
|
12
|
+
- every unit of the recipient's path packages: Cargo judges their freshness by mtime against the
|
|
13
|
+
other tree's sources;
|
|
14
|
+
- every unit of a package the recipient does not use: nothing classified it;
|
|
15
|
+
- each build-script run whose kept results point into the donor where Cargo does not rewrite
|
|
16
|
+
them: on reuse Cargo rewrites a script's own OUT_DIR in its directives, not paths elsewhere
|
|
17
|
+
(aws-lc-rs's `cargo:include` into aws-lc-sys) nor inside the text it wrote (tauri's permission
|
|
18
|
+
lists);
|
|
19
|
+
- every unit of a package whose sources or generated Rust plainly compile OUT_DIR in (rather
|
|
20
|
+
than through include!/include_str!/include_bytes!): Cargo never recompiles that path
|
|
21
|
+
(fingerprint/dep_info.rs). This is a best-effort reading of the source, not a proof;
|
|
22
|
+
- incremental caches.
|
|
23
|
+
|
|
24
|
+
Cargo rebuilds what it then finds missing, and everything that depends on it. A path that
|
|
25
|
+
survives inside compiled output (a proc-macro's install name, debug maps, a native library or
|
|
26
|
+
a crate that compiled a path in where the reading above does not see it) still names the
|
|
27
|
+
donor's files: when the donor rebuilds or removes them, the recipient can change behavior
|
|
28
|
+
without a warning. That is why a workspace opts in, with `[workspace.metadata.devlyn]
|
|
29
|
+
warm-build-dirs = true` in its Cargo.toml, once its dependencies do not use such paths.
|
|
30
|
+
|
|
31
|
+
The donor is only read: its dev build lock is taken from an existing lock file opened read-only,
|
|
32
|
+
without waiting, while the layouts are cloned. The whole operation has a time limit. Every skip
|
|
33
|
+
is reported with its reason; the target directory then stays absent and the first build is an
|
|
34
|
+
ordinary cold one.
|
|
35
|
+
|
|
36
|
+
warm-build-dirs.py <worktree> [--from <checkout>]
|
|
37
|
+
warm-build-dirs.py --self-test
|
|
38
|
+
"""
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import argparse
|
|
42
|
+
import contextlib
|
|
43
|
+
import ctypes
|
|
44
|
+
import json
|
|
45
|
+
import os
|
|
46
|
+
from pathlib import Path
|
|
47
|
+
import re
|
|
48
|
+
import shutil
|
|
49
|
+
import stat
|
|
50
|
+
import subprocess
|
|
51
|
+
import sys
|
|
52
|
+
import tempfile
|
|
53
|
+
import time
|
|
54
|
+
import unittest
|
|
55
|
+
import uuid
|
|
56
|
+
|
|
57
|
+
PROFILE_DIR = "debug"
|
|
58
|
+
BUDGET_SECONDS = 300
|
|
59
|
+
CARGO_ENTRIES = {".fingerprint", "build", "deps", "examples", ".cargo-lock"}
|
|
60
|
+
UNIT = re.compile(r"^(?P<name>.+)-(?P<hash>[0-9a-f]{16})(?:\.[^/]*)?$")
|
|
61
|
+
SNIFF_BYTES = 8192 # a NUL among the first bytes marks a binary file
|
|
62
|
+
# Conservative on purpose. Every string literal that spells OUT_DIR, escapes included, counts as
|
|
63
|
+
# a compiled-in path, and only the plain include!(concat!(env!("OUT_DIR"), ...)) forms, which
|
|
64
|
+
# embed contents, offset one each; an env!/option_env! whose argument is not a literal counts too.
|
|
65
|
+
_SPACE = r"(?:\s|//[^\n]*|/\*[\s\S]*?\*/)*"
|
|
66
|
+
# Every quote may start a literal (a lookahead, so candidates overlap); its escapes are decoded.
|
|
67
|
+
STRING_CANDIDATE = re.compile(r'(?="((?:[^"\\]|\\[\s\S])*?)")')
|
|
68
|
+
ESCAPE = re.compile(r'\\(?:x([0-9a-fA-F]{2})|u\{([0-9a-fA-F_]+)\}|\n\s*|(.))')
|
|
69
|
+
NONLITERAL_ENV = re.compile(rf'(?<![A-Za-z0-9_])(?:option_)?env{_SPACE}!{_SPACE}[(\[{{]{_SPACE}(?!r?#*")')
|
|
70
|
+
INCLUDED_OUT_DIR = re.compile(
|
|
71
|
+
r'(?<![A-Za-z0-9_])include(?:_str|_bytes)?\s*!\s*\(\s*concat\s*!\s*\(\s*env\s*!\s*\(\s*"OUT_DIR"\s*\)')
|
|
72
|
+
REDIRECTS = ("CARGO_TARGET_DIR", "CARGO_BUILD_TARGET_DIR", "CARGO_BUILD_BUILD_DIR")
|
|
73
|
+
RUSTC_ENV = re.compile(r"^cargo::?rustc-env=[^=]+=")
|
|
74
|
+
# Runs in a child so the time limit can stop a long clone; argv: source, target.
|
|
75
|
+
CLONE_CHILD = """import ctypes, os, sys
|
|
76
|
+
if ctypes.CDLL(None, use_errno=True).clonefile(os.fsencode(sys.argv[1]), os.fsencode(sys.argv[2]), 1) != 0:
|
|
77
|
+
sys.exit(os.strerror(ctypes.get_errno()))
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class Skip(Exception):
|
|
82
|
+
"""An expected reason not to warm; the build starts cold."""
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class Unsafe(Exception):
|
|
86
|
+
"""A safety check failed; nothing was published and the donor was not written."""
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def entry(tool, status, **fields):
|
|
90
|
+
return {"tool": tool, "status": status, **fields}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class Deadline:
|
|
94
|
+
def __init__(self, seconds):
|
|
95
|
+
self.seconds = seconds
|
|
96
|
+
self.end = time.monotonic() + seconds
|
|
97
|
+
|
|
98
|
+
def left(self):
|
|
99
|
+
remaining = self.end - time.monotonic()
|
|
100
|
+
if remaining <= 0:
|
|
101
|
+
raise TimeUp(f"warming took longer than {self.seconds} s")
|
|
102
|
+
return remaining
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class TimeUp(Skip):
|
|
106
|
+
"""The time limit ran out; staging is left in scratch for its cleanup."""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def run(argv, *, cwd, deadline, env=None):
|
|
110
|
+
try:
|
|
111
|
+
result = subprocess.run(argv, cwd=cwd, env=env, capture_output=True, text=True,
|
|
112
|
+
encoding="utf-8", timeout=deadline.left())
|
|
113
|
+
except subprocess.TimeoutExpired as exc:
|
|
114
|
+
raise TimeUp(f"{' '.join(map(str, argv[:3]))} timed out") from exc
|
|
115
|
+
if result.returncode != 0:
|
|
116
|
+
raise Skip(f"{' '.join(map(str, argv[:3]))} failed: {(result.stderr or result.stdout).strip()[-400:]}")
|
|
117
|
+
return result.stdout
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def git(work, *args, deadline):
|
|
121
|
+
return run(["git", "-C", str(work), *args], cwd=work, deadline=deadline).removesuffix("\n")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def without_redirects():
|
|
125
|
+
return {key: value for key, value in os.environ.items() if key not in REDIRECTS}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def metadata(work, deadline, *, env=None, deps=False):
|
|
129
|
+
argv = ["cargo", "metadata", "--format-version", "1", "--frozen", *([] if deps else ["--no-deps"])]
|
|
130
|
+
found = json.loads(run(argv, cwd=work, deadline=deadline, env=env))
|
|
131
|
+
if "build_directory" not in found:
|
|
132
|
+
raise Skip("this Cargo does not report its build directory (Cargo 1.91 or later needed)")
|
|
133
|
+
return found
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def open_beneath(root, parts):
|
|
137
|
+
"""A descriptor for root/parts, reached without following any link below root."""
|
|
138
|
+
fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
|
|
139
|
+
try:
|
|
140
|
+
for part in parts:
|
|
141
|
+
child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
|
142
|
+
os.close(fd)
|
|
143
|
+
fd = child
|
|
144
|
+
except OSError:
|
|
145
|
+
os.close(fd)
|
|
146
|
+
raise
|
|
147
|
+
return fd
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def no_symlink_between(root, path):
|
|
151
|
+
current = root
|
|
152
|
+
for part in path.relative_to(root).parts:
|
|
153
|
+
current = current / part
|
|
154
|
+
if current.is_symlink():
|
|
155
|
+
return False
|
|
156
|
+
return True
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def destination(work, deadline, stack):
|
|
160
|
+
"""The worktree's own Cargo target directory and a descriptor for its parent, when it can be warmed."""
|
|
161
|
+
found = metadata(work, deadline)
|
|
162
|
+
if ((found.get("metadata") or {}).get("devlyn") or {}).get("warm-build-dirs") is not True:
|
|
163
|
+
raise Skip("not enabled: the workspace's Cargo.toml has no [workspace.metadata.devlyn] warm-build-dirs = true")
|
|
164
|
+
if any(key in os.environ for key in REDIRECTS) and metadata(work, deadline, env=without_redirects()) != found:
|
|
165
|
+
raise Skip("the target directory comes from this environment, which later commands may not share")
|
|
166
|
+
target = Path(found["target_directory"])
|
|
167
|
+
if Path(found["build_directory"]) != target:
|
|
168
|
+
raise Skip("a separate Cargo build-dir is configured")
|
|
169
|
+
if not target.is_absolute() or not target.is_relative_to(work) or target == work:
|
|
170
|
+
raise Skip(f"{target} is not inside the worktree")
|
|
171
|
+
relative = target.relative_to(work)
|
|
172
|
+
if relative.parts[0] in {".git", ".devlyn"} or ".." in relative.parts:
|
|
173
|
+
raise Skip(f"{relative} is reserved")
|
|
174
|
+
try:
|
|
175
|
+
parent = stack.enter_context(contextlib.closing(_Fd(open_beneath(work, relative.parts[:-1]))))
|
|
176
|
+
except OSError as exc:
|
|
177
|
+
raise Skip(f"{relative.parent} cannot be reached without a link: {exc.strerror}") from exc
|
|
178
|
+
if os.path.lexists(target):
|
|
179
|
+
raise Skip(f"{relative} already exists")
|
|
180
|
+
if git(work, "ls-files", "--", str(relative), deadline=deadline):
|
|
181
|
+
raise Skip(f"{relative} holds tracked files")
|
|
182
|
+
try:
|
|
183
|
+
# The trailing slash names a directory, which directory-only rules such as `target/` need.
|
|
184
|
+
git(work, "check-ignore", "-q", "--", f"{relative}/", deadline=deadline)
|
|
185
|
+
except TimeUp:
|
|
186
|
+
raise
|
|
187
|
+
except Skip as exc:
|
|
188
|
+
raise Skip(f"{relative} is not ignored by Git") from exc
|
|
189
|
+
return target, relative, parent
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class _Fd:
|
|
193
|
+
def __init__(self, fd):
|
|
194
|
+
self.fd = fd
|
|
195
|
+
|
|
196
|
+
def close(self):
|
|
197
|
+
os.close(self.fd)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def donors(work, anchor, relative, deadline):
|
|
201
|
+
"""Built targets at the same place in the anchor, then the other worktrees, newest first."""
|
|
202
|
+
others = []
|
|
203
|
+
for line in git(work, "worktree", "list", "--porcelain", deadline=deadline).splitlines():
|
|
204
|
+
if line.startswith("worktree "):
|
|
205
|
+
checkout = Path(line[len("worktree "):]).resolve()
|
|
206
|
+
if checkout not in {work, anchor}:
|
|
207
|
+
others.append(checkout)
|
|
208
|
+
found = []
|
|
209
|
+
for checkout in [anchor, *others]:
|
|
210
|
+
donor = checkout / relative
|
|
211
|
+
layout = donor / PROFILE_DIR
|
|
212
|
+
if not (no_symlink_between(checkout, donor) and (layout / ".fingerprint").is_dir() and not layout.is_symlink()):
|
|
213
|
+
continue
|
|
214
|
+
if donor.is_relative_to(work) or work.is_relative_to(donor):
|
|
215
|
+
continue
|
|
216
|
+
found.append((checkout, donor, (layout / ".fingerprint").stat().st_mtime))
|
|
217
|
+
first = [item for item in found if item[0] == anchor]
|
|
218
|
+
return first + sorted((item for item in found if item[0] != anchor), key=lambda item: -item[2])
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
@contextlib.contextmanager
|
|
222
|
+
def read_lock(donor):
|
|
223
|
+
"""The donor's dev build lock, held only if free now, through a read-only descriptor."""
|
|
224
|
+
import fcntl
|
|
225
|
+
path = donor / PROFILE_DIR / ".cargo-lock"
|
|
226
|
+
try:
|
|
227
|
+
# O_NONBLOCK: a FIFO in its place must not stall the open.
|
|
228
|
+
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
229
|
+
except OSError as exc:
|
|
230
|
+
raise Skip(f"{path} cannot be opened: {exc.strerror}") from exc
|
|
231
|
+
try:
|
|
232
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
233
|
+
raise Skip(f"{path} is not a regular file")
|
|
234
|
+
try:
|
|
235
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
236
|
+
except BlockingIOError as exc:
|
|
237
|
+
raise Skip(f"a build is using {donor}") from exc
|
|
238
|
+
yield fd, path
|
|
239
|
+
finally:
|
|
240
|
+
os.close(fd)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def same_file(fd, path):
|
|
244
|
+
held, current = os.fstat(fd), os.lstat(path)
|
|
245
|
+
return (held.st_dev, held.st_ino) == (current.st_dev, current.st_ino)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def layouts(target):
|
|
249
|
+
"""The dev-profile layouts under a target directory, host first, relative."""
|
|
250
|
+
found = [Path(PROFILE_DIR)] if (target / PROFILE_DIR / ".fingerprint").is_dir() else []
|
|
251
|
+
for kind in sorted(p for p in target.iterdir() if p.is_dir() and not p.is_symlink() and p.name != PROFILE_DIR):
|
|
252
|
+
if (kind / PROFILE_DIR / ".fingerprint").is_dir() and not (kind / PROFILE_DIR).is_symlink():
|
|
253
|
+
found.append(Path(kind.name, PROFILE_DIR))
|
|
254
|
+
return found
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def clone(source, target, deadline):
|
|
258
|
+
"""Share source's blocks at target (never copy them), within the time limit."""
|
|
259
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
260
|
+
argv = ([sys.executable, "-c", CLONE_CHILD, str(source), str(target)] if sys.platform == "darwin"
|
|
261
|
+
else ["cp", "-a", "--reflink=always", "--", str(source), str(target)])
|
|
262
|
+
try:
|
|
263
|
+
result = subprocess.run(argv, capture_output=True, text=True, timeout=deadline.left())
|
|
264
|
+
except subprocess.TimeoutExpired as exc:
|
|
265
|
+
raise TimeUp(f"cloning {source} timed out") from exc
|
|
266
|
+
if result.returncode != 0:
|
|
267
|
+
raise Skip(f"this filesystem cannot clone {source}: {result.stderr.strip()[-200:]}")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def publish(staging_parent, staging_name, target_parent, target_name):
|
|
271
|
+
"""Move staging into the checked parent only if nothing exists there."""
|
|
272
|
+
names = os.fsencode(staging_name), os.fsencode(target_name)
|
|
273
|
+
if sys.platform == "darwin":
|
|
274
|
+
rename = ctypes.CDLL(None, use_errno=True).renameatx_np
|
|
275
|
+
rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
|
276
|
+
failed = rename(staging_parent, names[0], target_parent, names[1], 0x4) != 0 # RENAME_EXCL
|
|
277
|
+
else:
|
|
278
|
+
rename = ctypes.CDLL(None, use_errno=True).renameat2
|
|
279
|
+
rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
|
280
|
+
failed = rename(staging_parent, names[0], target_parent, names[1], 1) != 0 # RENAME_NOREPLACE
|
|
281
|
+
if failed:
|
|
282
|
+
raise Skip(f"cannot publish {target_name}: {os.strerror(ctypes.get_errno())}")
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def check_clone(staging, deadline):
|
|
286
|
+
"""No Git data, no special file, and no link that leads out of the clone."""
|
|
287
|
+
for directory, dirs, files in os.walk(staging):
|
|
288
|
+
deadline.left()
|
|
289
|
+
for name in dirs + files:
|
|
290
|
+
path = Path(directory, name)
|
|
291
|
+
mode = path.lstat().st_mode
|
|
292
|
+
if name.casefold() == ".git":
|
|
293
|
+
raise Unsafe(f"{path} is Git data")
|
|
294
|
+
if stat.S_ISLNK(mode):
|
|
295
|
+
if not Path(os.path.realpath(path)).is_relative_to(staging):
|
|
296
|
+
raise Unsafe(f"{path} links out of the clone")
|
|
297
|
+
elif not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
|
|
298
|
+
raise Unsafe(f"{path} is not a regular file or directory")
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def mentions(data, roots):
|
|
302
|
+
"""Whether bytes name a path at or below any root (not a sibling such as `<root>.other`)."""
|
|
303
|
+
return any(re.search(re.escape(os.fsencode(root)) + rb"(?![A-Za-z0-9._-])", data) for root in roots)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def is_text(path):
|
|
307
|
+
"""Whether a regular file (never a link or special file) holds no NUL in its first bytes."""
|
|
308
|
+
if not stat.S_ISREG(path.lstat().st_mode):
|
|
309
|
+
return False
|
|
310
|
+
with open(path, "rb") as stream:
|
|
311
|
+
return b"\0" not in stream.read(SNIFF_BYTES)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def foreign_runs(staging, layout, roots, deadline):
|
|
315
|
+
"""Hashes of build-script runs whose kept results name a donor root Cargo does not rewrite."""
|
|
316
|
+
found = set()
|
|
317
|
+
for run_dir in (staging / layout / "build").iterdir():
|
|
318
|
+
deadline.left()
|
|
319
|
+
root_output = run_dir / "root-output"
|
|
320
|
+
if not root_output.is_file():
|
|
321
|
+
continue # the compiled build script, not a run of it
|
|
322
|
+
old_out = os.fsdecode(root_output.read_bytes())
|
|
323
|
+
# <old target>/<layout>/build/<unit>/out
|
|
324
|
+
tail = (*layout.parts, "build", run_dir.name, "out")
|
|
325
|
+
if not Path(old_out).is_absolute() or Path(old_out).parts[-len(tail):] != tail:
|
|
326
|
+
raise Unsafe(f"{root_output} does not name this run's OUT_DIR")
|
|
327
|
+
needles = {*roots, str(Path(old_out).parents[len(tail) - 1])}
|
|
328
|
+
output = run_dir / "output"
|
|
329
|
+
lines = output.read_text(errors="replace").splitlines() if output.is_file() else []
|
|
330
|
+
# A variable the script hands to rustc is compiled in as it is, own OUT_DIR included.
|
|
331
|
+
exported = [line.split("=", 2)[2] for line in lines if RUSTC_ENV.match(line)]
|
|
332
|
+
foreign = mentions(os.fsencode("\n".join(exported)), {*needles, old_out}) or mentions(
|
|
333
|
+
os.fsencode("\n".join(lines).replace(old_out, "")), needles)
|
|
334
|
+
for directory, _, files in os.walk(run_dir / "out") if not foreign else ():
|
|
335
|
+
deadline.left()
|
|
336
|
+
for name in files:
|
|
337
|
+
path = Path(directory, name)
|
|
338
|
+
deadline.left()
|
|
339
|
+
if is_text(path):
|
|
340
|
+
data = path.read_bytes()
|
|
341
|
+
foreign = mentions(data, needles) or (name.endswith(".rs") and compiles_path(data.decode(errors="replace")))
|
|
342
|
+
if foreign:
|
|
343
|
+
break
|
|
344
|
+
if foreign:
|
|
345
|
+
break
|
|
346
|
+
if foreign:
|
|
347
|
+
found.add(UNIT.match(run_dir.name)["hash"])
|
|
348
|
+
return found
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def decoded(literal):
|
|
352
|
+
def one(match):
|
|
353
|
+
byte, code, other = match.groups()
|
|
354
|
+
if byte or code:
|
|
355
|
+
return chr(int((byte or code).replace("_", ""), 16))
|
|
356
|
+
return "" if other is None else other # a continuation, or an escaped character
|
|
357
|
+
return ESCAPE.sub(one, literal)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def compiles_path(text):
|
|
361
|
+
"""Whether Rust source may compile its OUT_DIR in other than through an include macro."""
|
|
362
|
+
literals = sum(decoded(match) == "OUT_DIR" for match in STRING_CANDIDATE.findall(text))
|
|
363
|
+
return literals > len(INCLUDED_OUT_DIR.findall(text)) or NONLITERAL_ENV.search(text) is not None
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def compiles_out_dir(package, deadline):
|
|
367
|
+
"""Whether the package's library sources compile its OUT_DIR in."""
|
|
368
|
+
root = Path(package["manifest_path"]).parent
|
|
369
|
+
for directory, dirs, files in os.walk(root):
|
|
370
|
+
deadline.left()
|
|
371
|
+
dirs[:] = [d for d in dirs if d not in {"target", "tests", "benches", "examples", ".git"}]
|
|
372
|
+
for name in files:
|
|
373
|
+
if name.endswith(".rs") and not (Path(directory) == root and name == "build.rs"):
|
|
374
|
+
if compiles_path(Path(directory, name).read_text(encoding="utf-8", errors="replace")):
|
|
375
|
+
return True
|
|
376
|
+
return False
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def remove(item, deadline):
|
|
380
|
+
"""Delete a file or a whole directory without following links, within the time limit."""
|
|
381
|
+
deadline.left()
|
|
382
|
+
if not item.is_dir() or item.is_symlink():
|
|
383
|
+
item.unlink()
|
|
384
|
+
return
|
|
385
|
+
for directory, dirs, files in os.walk(item, topdown=False):
|
|
386
|
+
for name in files:
|
|
387
|
+
deadline.left()
|
|
388
|
+
os.unlink(os.path.join(directory, name))
|
|
389
|
+
for name in dirs:
|
|
390
|
+
deadline.left()
|
|
391
|
+
path = os.path.join(directory, name)
|
|
392
|
+
os.unlink(path) if os.path.islink(path) else os.rmdir(path)
|
|
393
|
+
deadline.left()
|
|
394
|
+
os.rmdir(item)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def remove_units(staging, layout, hashes, deadline):
|
|
398
|
+
for sub in (".fingerprint", "build", "deps", "examples"):
|
|
399
|
+
directory = staging / layout / sub
|
|
400
|
+
for item in directory.iterdir() if directory.is_dir() else ():
|
|
401
|
+
match = UNIT.match(item.name)
|
|
402
|
+
if match and match["hash"] in hashes:
|
|
403
|
+
remove(item, deadline)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def remove_unclassified(staging, layout, deadline):
|
|
407
|
+
"""Final artifacts Cargo places beside the unit directories carry no unit to classify them by."""
|
|
408
|
+
for item in (staging / layout).iterdir():
|
|
409
|
+
if item.name not in CARGO_ENTRIES:
|
|
410
|
+
remove(item, deadline)
|
|
411
|
+
for sub in ("deps", "examples"):
|
|
412
|
+
directory = staging / layout / sub
|
|
413
|
+
for item in directory.iterdir() if directory.is_dir() else ():
|
|
414
|
+
if not UNIT.match(item.name):
|
|
415
|
+
remove(item, deadline)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def sanitize(work, staging, found, roots, deadline):
|
|
419
|
+
for layout in found:
|
|
420
|
+
remove_unclassified(staging, layout, deadline)
|
|
421
|
+
env = {**without_redirects(), "CARGO_TARGET_DIR": str(staging)}
|
|
422
|
+
resolved = metadata(work, deadline, env=env, deps=True)
|
|
423
|
+
if not (Path(resolved["target_directory"]).resolve() == Path(resolved["build_directory"]).resolve() == staging.resolve()):
|
|
424
|
+
raise Unsafe("Cargo does not resolve the staging directory as both target and build directory")
|
|
425
|
+
names = {package["name"] for package in resolved["packages"]}
|
|
426
|
+
path_packages = sorted({package["name"] for package in resolved["packages"] if package["source"] is None})
|
|
427
|
+
out_dir_packages = sorted({package["name"] for package in resolved["packages"] if package["source"] is not None
|
|
428
|
+
and any("custom-build" in t["kind"] for t in package["targets"])
|
|
429
|
+
and compiles_out_dir(package, deadline)})
|
|
430
|
+
removed = set()
|
|
431
|
+
for layout in found:
|
|
432
|
+
orphans = {m["hash"] for item in (staging / layout / ".fingerprint").iterdir()
|
|
433
|
+
if (m := UNIT.match(item.name)) and m["name"] not in names}
|
|
434
|
+
foreign = foreign_runs(staging, layout, roots, deadline)
|
|
435
|
+
removed |= {m["name"] for item in (staging / layout / ".fingerprint").iterdir()
|
|
436
|
+
if (m := UNIT.match(item.name)) and m["hash"] in orphans | foreign}
|
|
437
|
+
remove_units(staging, layout, orphans | foreign, deadline)
|
|
438
|
+
clean = path_packages + out_dir_packages
|
|
439
|
+
if clean:
|
|
440
|
+
base = ["cargo", "clean", "--frozen", "--target-dir", str(staging), "--profile", "dev"]
|
|
441
|
+
for name in clean:
|
|
442
|
+
base += ["-p", name]
|
|
443
|
+
run(base, cwd=work, deadline=deadline, env=env) # Without --target only the host layout's outputs go.
|
|
444
|
+
triples = [arg for layout in found[1:] for arg in ("--target", layout.parts[0])]
|
|
445
|
+
if triples:
|
|
446
|
+
run([*base, *triples], cwd=work, deadline=deadline, env=env)
|
|
447
|
+
return sorted(removed | set(clean))
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def warm_cargo(work, anchor, staging_root, deadline):
|
|
451
|
+
if not (work / "Cargo.toml").is_file():
|
|
452
|
+
return None
|
|
453
|
+
if sys.platform not in {"darwin", "linux"}:
|
|
454
|
+
return entry("cargo", "skipped", reason=f"cloning is unsupported on {sys.platform}")
|
|
455
|
+
started = time.monotonic()
|
|
456
|
+
with contextlib.ExitStack() as stack:
|
|
457
|
+
staging = None
|
|
458
|
+
try:
|
|
459
|
+
if shutil.which("cargo") is None:
|
|
460
|
+
raise Skip("cargo is not on PATH")
|
|
461
|
+
target, relative, parent = destination(work, deadline, stack)
|
|
462
|
+
candidates = donors(work, anchor, relative, deadline)
|
|
463
|
+
if not candidates:
|
|
464
|
+
raise Skip(f"no built {relative} in this repository's checkouts")
|
|
465
|
+
if os.stat(staging_root).st_dev != os.fstat(parent.fd).st_dev:
|
|
466
|
+
raise Skip("the staging and target directories are on different volumes")
|
|
467
|
+
root = stack.enter_context(contextlib.closing(_Fd(os.open(staging_root, os.O_RDONLY | os.O_DIRECTORY))))
|
|
468
|
+
busy = []
|
|
469
|
+
for checkout, donor, _ in candidates:
|
|
470
|
+
if os.stat(donor).st_dev != os.stat(staging_root).st_dev:
|
|
471
|
+
busy.append(f"{donor} is on another volume")
|
|
472
|
+
continue
|
|
473
|
+
name = f".warm-cargo-{uuid.uuid4().hex}"
|
|
474
|
+
try:
|
|
475
|
+
with read_lock(donor) as (fd, lock):
|
|
476
|
+
identity = os.stat(donor / PROFILE_DIR)
|
|
477
|
+
found = layouts(donor)
|
|
478
|
+
os.mkdir(name, dir_fd=root.fd)
|
|
479
|
+
staging = staging_root / name
|
|
480
|
+
for layout in found:
|
|
481
|
+
clone(donor / layout, staging / layout, deadline)
|
|
482
|
+
after = os.stat(donor / PROFILE_DIR)
|
|
483
|
+
if (identity.st_dev, identity.st_ino) != (after.st_dev, after.st_ino) or not same_file(fd, lock):
|
|
484
|
+
raise Unsafe(f"{donor} was replaced while it was cloned")
|
|
485
|
+
except TimeUp:
|
|
486
|
+
raise
|
|
487
|
+
except Skip as exc:
|
|
488
|
+
busy.append(str(exc))
|
|
489
|
+
if staging is not None:
|
|
490
|
+
remove(staging, deadline)
|
|
491
|
+
staging = None
|
|
492
|
+
continue
|
|
493
|
+
check_clone(staging, deadline)
|
|
494
|
+
if (donor / "CACHEDIR.TAG").is_file():
|
|
495
|
+
(staging / "CACHEDIR.TAG").write_bytes((donor / "CACHEDIR.TAG").read_bytes())
|
|
496
|
+
removed = sanitize(work, staging, found, [str(donor), str(checkout)], deadline)
|
|
497
|
+
deadline.left()
|
|
498
|
+
try:
|
|
499
|
+
reached = open_beneath(work, relative.parts[:-1])
|
|
500
|
+
except OSError as exc:
|
|
501
|
+
raise Unsafe(f"{relative.parent} changed while warming: {exc.strerror}") from exc
|
|
502
|
+
try:
|
|
503
|
+
held, now = os.fstat(parent.fd), os.fstat(reached)
|
|
504
|
+
if (held.st_dev, held.st_ino) != (now.st_dev, now.st_ino):
|
|
505
|
+
raise Unsafe(f"{relative.parent} changed while warming")
|
|
506
|
+
finally:
|
|
507
|
+
os.close(reached)
|
|
508
|
+
publish(root.fd, name, parent.fd, relative.name)
|
|
509
|
+
staging = None
|
|
510
|
+
return entry("cargo", "seeded", path=str(target), donor=str(donor),
|
|
511
|
+
removed=removed, seconds=round(time.monotonic() - started, 1))
|
|
512
|
+
raise Skip("; ".join(busy))
|
|
513
|
+
except Skip as exc:
|
|
514
|
+
result = entry("cargo", "skipped", reason=str(exc))
|
|
515
|
+
except Unsafe as exc:
|
|
516
|
+
result = entry("cargo", "failed", reason=str(exc))
|
|
517
|
+
except FileNotFoundError as exc:
|
|
518
|
+
result = entry("cargo", "skipped", reason=f"{exc.filename} disappeared while warming")
|
|
519
|
+
except Exception as exc: # Reported, never raised: allocation goes on cold.
|
|
520
|
+
result = entry("cargo", "failed", reason=f"{type(exc).__name__}: {exc}")
|
|
521
|
+
if staging is not None and os.path.lexists(staging):
|
|
522
|
+
try:
|
|
523
|
+
remove(staging, deadline)
|
|
524
|
+
except TimeUp:
|
|
525
|
+
# Removing a large partial clone would outlast the limit; scratch cleanup owns it.
|
|
526
|
+
result["residue"] = str(staging)
|
|
527
|
+
return result
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def warm(work, anchor, staging_root, *, budget=BUDGET_SECONDS):
|
|
531
|
+
"""Warm every eligible build directory of a new worktree; never raises for a tool's failure."""
|
|
532
|
+
work, anchor, staging_root = Path(work).resolve(), Path(anchor).resolve(), Path(staging_root).resolve()
|
|
533
|
+
results = []
|
|
534
|
+
for tool, warmer in (("cargo", warm_cargo),):
|
|
535
|
+
try:
|
|
536
|
+
result = warmer(work, anchor, staging_root, Deadline(budget))
|
|
537
|
+
except Exception as exc: # Optional warming reports any failure; allocation goes on cold.
|
|
538
|
+
result = entry(tool, "failed", reason=f"{type(exc).__name__}: {exc}")
|
|
539
|
+
if result is not None:
|
|
540
|
+
results.append(result)
|
|
541
|
+
print(f"warm-build-dirs: {tool} {result['status']}: {result.get('path') or result.get('reason')}", file=sys.stderr)
|
|
542
|
+
return results
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def main():
|
|
546
|
+
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
547
|
+
parser.add_argument("worktree", nargs="?", type=Path)
|
|
548
|
+
parser.add_argument("--from", dest="anchor", type=Path, help="the checkout to warm from first (default: the main worktree)")
|
|
549
|
+
parser.add_argument("--self-test", action="store_true")
|
|
550
|
+
args = parser.parse_args()
|
|
551
|
+
if args.self_test:
|
|
552
|
+
return self_test()
|
|
553
|
+
if args.worktree is None:
|
|
554
|
+
parser.error("a worktree is required")
|
|
555
|
+
work = args.worktree.resolve()
|
|
556
|
+
anchor = args.anchor.resolve() if args.anchor else None
|
|
557
|
+
if anchor is None:
|
|
558
|
+
listing = subprocess.run(["git", "-C", str(work), "worktree", "list", "--porcelain"],
|
|
559
|
+
capture_output=True, text=True, check=True).stdout
|
|
560
|
+
anchor = Path(listing.splitlines()[0].removeprefix("worktree ")).resolve()
|
|
561
|
+
common = subprocess.run(["git", "-C", str(work), "rev-parse", "--path-format=absolute", "--git-common-dir"],
|
|
562
|
+
capture_output=True, text=True, check=True).stdout.strip()
|
|
563
|
+
staging_root = Path(tempfile.mkdtemp(prefix=".warm-build-dirs-", dir=common))
|
|
564
|
+
results = warm(work, anchor, staging_root)
|
|
565
|
+
if not any(result.get("residue") for result in results):
|
|
566
|
+
staging_root.rmdir()
|
|
567
|
+
print(json.dumps(results, sort_keys=True))
|
|
568
|
+
return 0
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def self_test():
|
|
572
|
+
result = unittest.TextTestRunner(verbosity=1).run(unittest.defaultTestLoader.loadTestsFromTestCase(WarmTests))
|
|
573
|
+
if result.wasSuccessful():
|
|
574
|
+
print(f"warm-build-dirs self-test: PASS ({result.testsRun} isolated tests)")
|
|
575
|
+
return 0 if result.wasSuccessful() else 1
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
# Fixtures use real Git and a fake `cargo` on PATH that answers `metadata` from a JSON file
|
|
579
|
+
# and performs `clean -p` on the layout as Cargo names it; a filesystem without cloning
|
|
580
|
+
# copies instead, in tests only, so every rule still runs there.
|
|
581
|
+
FAKE_CARGO = r"""#!/usr/bin/env python3
|
|
582
|
+
import json, os, pathlib, shutil, sys, time
|
|
583
|
+
argv = sys.argv[1:]
|
|
584
|
+
fixture = json.loads(pathlib.Path(os.environ["FAKE_CARGO"]).read_text())
|
|
585
|
+
with open(os.environ["FAKE_CARGO_LOG"], "a") as log:
|
|
586
|
+
log.write(json.dumps(argv) + "\n")
|
|
587
|
+
time.sleep(fixture.get("sleep", 0))
|
|
588
|
+
target = pathlib.Path(os.environ.get("CARGO_TARGET_DIR") or os.getcwd() + "/" + fixture.get("target", "target"))
|
|
589
|
+
if argv[0] == "metadata":
|
|
590
|
+
found = {"target_directory": str(target), "packages": fixture["packages"] if "--no-deps" not in argv else [],
|
|
591
|
+
"metadata": fixture.get("workspace_metadata", {"devlyn": {"warm-build-dirs": True}})}
|
|
592
|
+
if not fixture.get("old_cargo"):
|
|
593
|
+
found["build_directory"] = fixture.get("build_directory", str(target))
|
|
594
|
+
print(json.dumps(found))
|
|
595
|
+
elif argv[0] == "clean":
|
|
596
|
+
directory = pathlib.Path(argv[argv.index("--target-dir") + 1])
|
|
597
|
+
names = [argv[i + 1] for i, a in enumerate(argv) if a == "-p"]
|
|
598
|
+
triples = [argv[i + 1] for i, a in enumerate(argv) if a == "--target"]
|
|
599
|
+
for layout in [directory / t / "debug" for t in triples] or [directory / "debug"]:
|
|
600
|
+
for name in names:
|
|
601
|
+
for sub in (".fingerprint", "build", "deps"):
|
|
602
|
+
for item in (layout / sub).glob("*"):
|
|
603
|
+
stem = item.name.split(".")[0].removeprefix("lib")
|
|
604
|
+
if stem.rsplit("-", 1)[0].replace("_", "-") == name:
|
|
605
|
+
shutil.rmtree(item) if item.is_dir() else item.unlink()
|
|
606
|
+
if fixture.get("swap") and not os.path.islink("build"):
|
|
607
|
+
os.rename("build", "build-moved")
|
|
608
|
+
os.symlink(fixture["swap"], "build")
|
|
609
|
+
if fixture.get("race"):
|
|
610
|
+
(pathlib.Path(os.getcwd()) / "target").mkdir(exist_ok=True)
|
|
611
|
+
"""
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
class WarmTests(unittest.TestCase):
|
|
615
|
+
def setUp(self):
|
|
616
|
+
# Imported here: unittest.mock imports asyncio, which a caller simulating Windows cannot load.
|
|
617
|
+
from unittest import mock
|
|
618
|
+
temporary = tempfile.TemporaryDirectory(prefix="warm-build-dirs-")
|
|
619
|
+
self.addCleanup(temporary.cleanup)
|
|
620
|
+
self.root = Path(temporary.name).resolve()
|
|
621
|
+
self.anchor, self.work = self.root / "anchor", self.root / "work"
|
|
622
|
+
self.staging = self.root / "scratch"
|
|
623
|
+
self.staging.mkdir()
|
|
624
|
+
bin_dir = self.root / "bin"
|
|
625
|
+
bin_dir.mkdir()
|
|
626
|
+
(bin_dir / "cargo").write_text(FAKE_CARGO)
|
|
627
|
+
(bin_dir / "cargo").chmod(0o755)
|
|
628
|
+
self.fixture, self.log = self.root / "fixture.json", self.root / "cargo.log"
|
|
629
|
+
self.environment = mock.patch.dict(os.environ, {
|
|
630
|
+
"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", "FAKE_CARGO": str(self.fixture),
|
|
631
|
+
"FAKE_CARGO_LOG": str(self.log), "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1",
|
|
632
|
+
"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"})
|
|
633
|
+
self.environment.start()
|
|
634
|
+
self.addCleanup(self.environment.stop)
|
|
635
|
+
for key in REDIRECTS:
|
|
636
|
+
os.environ.pop(key, None)
|
|
637
|
+
self.anchor.mkdir()
|
|
638
|
+
self.git(self.anchor, "init", "-q", "-b", "main")
|
|
639
|
+
(self.anchor / "Cargo.toml").write_text("[workspace]\n")
|
|
640
|
+
(self.anchor / ".gitignore").write_text("target/\n/build/target/\n")
|
|
641
|
+
self.git(self.anchor, "add", "-A")
|
|
642
|
+
self.git(self.anchor, "commit", "-q", "-m", "init")
|
|
643
|
+
self.git(self.anchor, "worktree", "add", "-q", "-b", "task", str(self.work))
|
|
644
|
+
self.write_fixture()
|
|
645
|
+
self.donor = self.anchor / "target"
|
|
646
|
+
self.build_donor()
|
|
647
|
+
probe = self.root / "probe"
|
|
648
|
+
probe.mkdir()
|
|
649
|
+
(probe / "file").write_bytes(b"x" * 65536)
|
|
650
|
+
self.native_clone = True
|
|
651
|
+
try:
|
|
652
|
+
if os.name == "nt":
|
|
653
|
+
raise Skip("no clone primitive")
|
|
654
|
+
clone(probe, self.root / "probe-clone", Deadline(30))
|
|
655
|
+
self.native_clone = (self.root / "probe-clone" / "file").read_bytes() == b"x" * 65536
|
|
656
|
+
except (Skip, OSError):
|
|
657
|
+
self.native_clone = False
|
|
658
|
+
if not self.native_clone:
|
|
659
|
+
# A plain copy that keeps links and special files, as a clone would; tests only.
|
|
660
|
+
def copy(source, target, deadline):
|
|
661
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
662
|
+
subprocess.run(["cp", "-a", "--", str(source), str(target)], check=True)
|
|
663
|
+
patch = mock.patch(f"{__name__}.clone", copy)
|
|
664
|
+
patch.start()
|
|
665
|
+
self.addCleanup(patch.stop)
|
|
666
|
+
|
|
667
|
+
def git(self, work, *args):
|
|
668
|
+
return subprocess.run(["git", "-C", str(work), *args], check=True, capture_output=True, text=True).stdout
|
|
669
|
+
|
|
670
|
+
def write_fixture(self, **extra):
|
|
671
|
+
packages = [
|
|
672
|
+
{"name": "app", "source": None, "manifest_path": str(self.work / "Cargo.toml"), "targets": [{"kind": ["lib"]}]},
|
|
673
|
+
{"name": "serde", "source": "registry", "manifest_path": str(self.root / "registry/serde/Cargo.toml"), "targets": [{"kind": ["lib"]}]},
|
|
674
|
+
{"name": "ring", "source": "registry", "manifest_path": str(self.root / "registry/ring/Cargo.toml"), "targets": [{"kind": ["lib"]}, {"kind": ["custom-build"]}]},
|
|
675
|
+
{"name": "tauri", "source": "registry", "manifest_path": str(self.root / "registry/tauri/Cargo.toml"), "targets": [{"kind": ["lib"]}, {"kind": ["custom-build"]}]},
|
|
676
|
+
{"name": "aws-lc-rs", "source": "registry", "manifest_path": str(self.root / "registry/aws-lc-rs/Cargo.toml"), "targets": [{"kind": ["lib"]}, {"kind": ["custom-build"]}]},
|
|
677
|
+
{"name": "baked", "source": "registry", "manifest_path": str(self.root / "registry/baked/Cargo.toml"), "targets": [{"kind": ["lib"]}, {"kind": ["custom-build"]}]},
|
|
678
|
+
{"name": "exporter", "source": "registry", "manifest_path": str(self.root / "registry/exporter/Cargo.toml"), "targets": [{"kind": ["lib"]}, {"kind": ["custom-build"]}]},
|
|
679
|
+
{"name": "generator", "source": "registry", "manifest_path": str(self.root / "registry/generator/Cargo.toml"), "targets": [{"kind": ["lib"]}, {"kind": ["custom-build"]}]},
|
|
680
|
+
]
|
|
681
|
+
for package in packages[1:]:
|
|
682
|
+
source = Path(package["manifest_path"]).parent / "src"
|
|
683
|
+
source.mkdir(parents=True, exist_ok=True)
|
|
684
|
+
(source / "lib.rs").write_text('include!(concat!(\n env!("OUT_DIR"), "/gen.rs"));\n' if package["name"] != "baked"
|
|
685
|
+
else 'pub const DIR: &str = env!(/* why */ "OUT_DIR");\n')
|
|
686
|
+
self.fixture.write_text(json.dumps({"packages": packages, **extra}))
|
|
687
|
+
|
|
688
|
+
def unit(self, layout, sub, name, digest, files=None):
|
|
689
|
+
directory = self.donor / layout / sub / f"{name}-{digest}"
|
|
690
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
691
|
+
for relative, data in (files or {"stamp": b"x"}).items():
|
|
692
|
+
(directory / relative).parent.mkdir(parents=True, exist_ok=True)
|
|
693
|
+
(directory / relative).write_bytes(data)
|
|
694
|
+
|
|
695
|
+
def run_unit(self, layout, name, digest, output, out_files):
|
|
696
|
+
old_out = f"{self.donor}/{layout}/build/{name}-{digest}/out"
|
|
697
|
+
self.unit(layout, ".fingerprint", name, digest)
|
|
698
|
+
self.unit(layout, "build", name, digest, {"root-output": old_out.encode(), "output": output(old_out).encode(),
|
|
699
|
+
**{f"out/{k}": v(old_out) for k, v in out_files.items()}})
|
|
700
|
+
|
|
701
|
+
def build_donor(self):
|
|
702
|
+
for layout in ("debug", "x86_64-pc-windows-msvc/debug"):
|
|
703
|
+
deps = self.donor / layout / "deps"
|
|
704
|
+
deps.mkdir(parents=True)
|
|
705
|
+
for name, digest in (("serde", "1" * 16), ("app", "2" * 16), ("gone", "7" * 16), ("baked", "8" * 16)):
|
|
706
|
+
self.unit(layout, ".fingerprint", name, digest)
|
|
707
|
+
(deps / f"lib{name}-{digest}.rlib").write_bytes(b"rlib")
|
|
708
|
+
(self.donor / layout / ".cargo-lock").write_bytes(b"")
|
|
709
|
+
(self.donor / layout / "incremental" / "app-x").mkdir(parents=True)
|
|
710
|
+
# Final artifacts Cargo puts beside the unit directories, here of the unused `gone`.
|
|
711
|
+
(self.donor / layout / "old-tool").write_bytes(b"bin")
|
|
712
|
+
(self.donor / layout / "examples").mkdir()
|
|
713
|
+
(self.donor / layout / "examples" / "old-example").write_bytes(b"bin")
|
|
714
|
+
self.run_unit(layout, "ring", "4" * 16, lambda out: f"cargo:rustc-link-search=native={out}\n",
|
|
715
|
+
{"libring.a": lambda out: b"\0\x01" + out.encode()})
|
|
716
|
+
self.run_unit(layout, "tauri", "3" * 16, lambda out: f"cargo:rustc-link-search=native={out}\n",
|
|
717
|
+
{"permission-files": lambda out: f'["{out}/app_hide.toml"]'.encode()})
|
|
718
|
+
self.run_unit(layout, "aws-lc-rs", "5" * 16,
|
|
719
|
+
lambda out: f"cargo:include={self.donor}/{layout}/build/aws-lc-sys-{'6' * 16}/out/include\n", {})
|
|
720
|
+
# A variable handed to rustc that carries the script's own OUT_DIR is compiled in as is.
|
|
721
|
+
self.run_unit(layout, "exporter", "9" * 16, lambda out: f"cargo:rustc-env=DEBUG_OUTPUT_DIR={out}\n", {})
|
|
722
|
+
# Generated Rust that compiles its OUT_DIR in names no donor path in its text.
|
|
723
|
+
self.run_unit(layout, "generator", "a" * 16, lambda out: f"cargo:rustc-link-search=native={out}\n",
|
|
724
|
+
{"gen.rs": lambda out: b'pub const DIR: &str = env!("OUT\\x5fDIR");\n'})
|
|
725
|
+
(self.donor / "CACHEDIR.TAG").write_bytes(b"Signature: 8a477f597d28d172789f06886806bc55\n")
|
|
726
|
+
|
|
727
|
+
def snapshot(self, root):
|
|
728
|
+
return {str(p.relative_to(root)): (p.lstat().st_size, p.lstat().st_mtime_ns, p.lstat().st_mode)
|
|
729
|
+
for p in sorted(root.rglob("*"))}
|
|
730
|
+
|
|
731
|
+
def warm(self, **kwargs):
|
|
732
|
+
return warm(self.work, self.anchor, self.staging, **kwargs)
|
|
733
|
+
|
|
734
|
+
def assert_nothing_left(self):
|
|
735
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
736
|
+
self.assertEqual("", self.git(self.work, "status", "--porcelain", "--untracked-files=all"))
|
|
737
|
+
|
|
738
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
739
|
+
def test_seeds_only_what_carries_over_and_never_writes_the_donor(self):
|
|
740
|
+
before = self.snapshot(self.donor)
|
|
741
|
+
[result] = self.warm()
|
|
742
|
+
self.assertEqual("seeded", result["status"], result)
|
|
743
|
+
target = self.work / "target"
|
|
744
|
+
for layout in ("debug", "x86_64-pc-windows-msvc/debug"):
|
|
745
|
+
kept = {p.name for p in (target / layout / ".fingerprint").iterdir()}
|
|
746
|
+
self.assertEqual({f"serde-{'1' * 16}", f"ring-{'4' * 16}"}, kept, layout)
|
|
747
|
+
self.assertEqual({".fingerprint", "build", "deps", "examples", ".cargo-lock"}, {p.name for p in (target / layout).iterdir()})
|
|
748
|
+
self.assertEqual([], list((target / layout / "examples").iterdir()))
|
|
749
|
+
self.assertTrue((target / layout / "build" / f"ring-{'4' * 16}" / "out" / "libring.a").exists())
|
|
750
|
+
self.assertEqual(["app", "aws-lc-rs", "baked", "exporter", "generator", "gone", "tauri"], result["removed"])
|
|
751
|
+
self.assertEqual(str(self.donor), result["donor"])
|
|
752
|
+
self.assertEqual(before, self.snapshot(self.donor))
|
|
753
|
+
self.assert_nothing_left()
|
|
754
|
+
|
|
755
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
756
|
+
def test_a_busy_donor_is_skipped_without_waiting(self):
|
|
757
|
+
import fcntl
|
|
758
|
+
with open(self.donor / "debug" / ".cargo-lock", "rb") as lock:
|
|
759
|
+
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
760
|
+
[result] = self.warm()
|
|
761
|
+
self.assertEqual("skipped", result["status"])
|
|
762
|
+
self.assertIn("a build is using", result["reason"])
|
|
763
|
+
self.assertFalse((self.work / "target").exists())
|
|
764
|
+
self.assert_nothing_left()
|
|
765
|
+
|
|
766
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
767
|
+
def test_ineligible_destinations_are_skipped_with_their_reason(self):
|
|
768
|
+
cases = {
|
|
769
|
+
"already exists": lambda: (self.work / "target").mkdir(),
|
|
770
|
+
"not ignored": lambda: (self.work / ".gitignore").write_text(""),
|
|
771
|
+
"not enabled": lambda: self.write_fixture(workspace_metadata=None),
|
|
772
|
+
"separate Cargo build-dir": lambda: self.write_fixture(build_directory=str(self.root / "elsewhere")),
|
|
773
|
+
"build directory (Cargo 1.91": lambda: self.write_fixture(old_cargo=True),
|
|
774
|
+
"comes from this environment": lambda: os.environ.__setitem__("CARGO_TARGET_DIR", str(self.root / "t")),
|
|
775
|
+
"no built target": lambda: shutil.rmtree(self.donor),
|
|
776
|
+
}
|
|
777
|
+
for reason, arrange in cases.items():
|
|
778
|
+
with self.subTest(reason):
|
|
779
|
+
self.setUp()
|
|
780
|
+
arrange()
|
|
781
|
+
[result] = self.warm()
|
|
782
|
+
self.assertEqual("skipped", result["status"], result)
|
|
783
|
+
self.assertIn(reason, result["reason"])
|
|
784
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
785
|
+
|
|
786
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
787
|
+
def test_a_clone_with_git_data_or_a_link_out_is_refused(self):
|
|
788
|
+
for name, arrange in {
|
|
789
|
+
"links out": lambda: (self.donor / "debug" / "build" / "link").symlink_to(self.root),
|
|
790
|
+
"Git data": lambda: (self.donor / "debug" / "deps" / ".git").mkdir(),
|
|
791
|
+
}.items():
|
|
792
|
+
with self.subTest(name):
|
|
793
|
+
self.setUp()
|
|
794
|
+
arrange()
|
|
795
|
+
before = self.snapshot(self.donor)
|
|
796
|
+
[result] = self.warm()
|
|
797
|
+
self.assertEqual("failed", result["status"], result)
|
|
798
|
+
self.assertIn(name, result["reason"])
|
|
799
|
+
self.assertFalse(os.path.lexists(self.work / "target"))
|
|
800
|
+
self.assertEqual(before, self.snapshot(self.donor))
|
|
801
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
802
|
+
|
|
803
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
804
|
+
def test_running_out_of_time_publishes_nothing(self):
|
|
805
|
+
self.write_fixture(sleep=2)
|
|
806
|
+
[result] = self.warm(budget=1)
|
|
807
|
+
self.assertEqual("skipped", result["status"])
|
|
808
|
+
self.assertIn("timed out", result["reason"])
|
|
809
|
+
self.assertFalse((self.work / "target").exists())
|
|
810
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
811
|
+
|
|
812
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
813
|
+
def test_a_target_that_appears_meanwhile_is_never_replaced(self):
|
|
814
|
+
self.write_fixture(race=True)
|
|
815
|
+
[result] = self.warm()
|
|
816
|
+
self.assertEqual("skipped", result["status"])
|
|
817
|
+
self.assertIn("cannot publish", result["reason"])
|
|
818
|
+
self.assertEqual([], list((self.work / "target").iterdir()))
|
|
819
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
820
|
+
|
|
821
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
822
|
+
def test_a_parent_replaced_while_warming_receives_nothing(self):
|
|
823
|
+
(self.anchor / "build").mkdir()
|
|
824
|
+
self.donor = self.anchor / "build" / "target"
|
|
825
|
+
(self.anchor / "target").rename(self.donor)
|
|
826
|
+
(self.work / "build").mkdir()
|
|
827
|
+
elsewhere = self.root / "elsewhere"
|
|
828
|
+
elsewhere.mkdir()
|
|
829
|
+
self.write_fixture(target="build/target", swap=str(elsewhere))
|
|
830
|
+
[result] = self.warm()
|
|
831
|
+
self.assertEqual("failed", result["status"], result)
|
|
832
|
+
self.assertIn("changed while warming", result["reason"])
|
|
833
|
+
self.assertEqual([], list(elsewhere.iterdir()))
|
|
834
|
+
self.assertFalse((self.work / "build-moved" / "target").exists())
|
|
835
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
836
|
+
|
|
837
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
838
|
+
def test_a_special_file_is_never_read(self):
|
|
839
|
+
os.mkfifo(self.donor / "debug" / "build" / f"tauri-{'3' * 16}" / "out" / "pipe")
|
|
840
|
+
[result] = self.warm(budget=20)
|
|
841
|
+
self.assertEqual("failed", result["status"], result)
|
|
842
|
+
self.assertIn("not a regular file or directory", result["reason"])
|
|
843
|
+
self.assertFalse((self.work / "target").exists())
|
|
844
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
845
|
+
|
|
846
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
847
|
+
def test_a_malformed_build_record_fails_explicitly(self):
|
|
848
|
+
(self.donor / "debug" / "build" / f"tauri-{'3' * 16}" / "root-output").write_bytes(b"")
|
|
849
|
+
[result] = self.warm()
|
|
850
|
+
self.assertEqual("failed", result["status"], result)
|
|
851
|
+
self.assertIn("does not name this run's OUT_DIR", result["reason"])
|
|
852
|
+
self.assertFalse((self.work / "target").exists())
|
|
853
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
854
|
+
|
|
855
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
856
|
+
def test_allocation_warms_its_worktree_and_records_the_outcome(self):
|
|
857
|
+
tree = self.root / "allocated"
|
|
858
|
+
head = self.git(self.anchor, "rev-parse", "HEAD").strip()
|
|
859
|
+
completed = subprocess.run([sys.executable, str(Path(__file__).with_name("task-complete.py")), "allocate",
|
|
860
|
+
"--repo", str(self.anchor), "--task", "warm", "--branch", "warm-allocation",
|
|
861
|
+
"--worktree", str(tree), "--local-base", head], capture_output=True, text=True)
|
|
862
|
+
self.assertEqual(0, completed.returncode, completed.stdout + completed.stderr)
|
|
863
|
+
result = json.loads(completed.stdout)
|
|
864
|
+
[warmed] = result["build_dirs"]
|
|
865
|
+
if self.native_clone:
|
|
866
|
+
self.assertEqual(("seeded", str(tree / "target")), (warmed["status"], warmed.get("path")), warmed)
|
|
867
|
+
else: # The child cannot use the fixture's copy; a filesystem without cloning must skip.
|
|
868
|
+
self.assertEqual("skipped", warmed["status"], warmed)
|
|
869
|
+
self.assertIn("cannot clone", warmed["reason"])
|
|
870
|
+
self.assertEqual(result["build_dirs"], json.loads(Path(result["receipt"]).read_text())["build_dirs"])
|
|
871
|
+
self.assertEqual("", self.git(tree, "status", "--porcelain", "--untracked-files=all"))
|
|
872
|
+
self.assertEqual([], list(Path(result["scratch"]).iterdir()))
|
|
873
|
+
|
|
874
|
+
def test_compiled_in_out_dir_is_recognized_in_every_spelling(self):
|
|
875
|
+
compiled = ['env!("OUT_DIR")', 'env!("OUT_DIR", "missing")', 'env ! ( "OUT_DIR" )', 'std::env!(r"OUT_DIR")',
|
|
876
|
+
'option_env!("OUT_DIR")', 'myinclude!(concat!(env!("OUT_DIR"), "/a.rs"))', 'env![ "OUT_DIR" ]',
|
|
877
|
+
'env/**/!("OUT_DIR")', 'env!(/* why */ "OUT_DIR")', 'env!("OUT\\x5fDIR")', 'env!("\\u{4f}UT_DIR")',
|
|
878
|
+
'use std::env as e; e!("OUT_DIR")', 'env!(concat!("OUT", "_DIR"))', 'env!(// note\n"OUT_DIR")',
|
|
879
|
+
'env!("\\u{4_f}UT_DIR")', 'env!("OUT_\\\n DIR")', 'let a = \'"\'; env!("OUT_DIR")',
|
|
880
|
+
'include!(concat!(env!(/* c */ "OUT_DIR"), "/a.rs"))']
|
|
881
|
+
for text in compiled:
|
|
882
|
+
self.assertTrue(compiles_path(text), text)
|
|
883
|
+
for text in ['include!(concat!(env!("OUT_DIR"), "/a.rs"));', 'include_bytes!(concat!(\n env!("OUT_DIR"),\n "/b"))',
|
|
884
|
+
'env!("OUT_DIRS")', 'env!("CARGO_PKG_NAME")', 'let s = "OUT_DIR_NAME";']:
|
|
885
|
+
self.assertFalse(compiles_path(text), text)
|
|
886
|
+
|
|
887
|
+
def test_no_cargo_workspace_or_an_unsupported_platform_does_nothing(self):
|
|
888
|
+
(self.work / "Cargo.toml").unlink()
|
|
889
|
+
self.assertEqual([], self.warm())
|
|
890
|
+
(self.work / "Cargo.toml").write_text("[workspace]\n")
|
|
891
|
+
from unittest import mock
|
|
892
|
+
with mock.patch.object(sys, "platform", "win32"):
|
|
893
|
+
[result] = self.warm()
|
|
894
|
+
self.assertEqual(("skipped", "cloning is unsupported on win32"), (result["status"], result["reason"]))
|
|
895
|
+
self.assertFalse(self.log.exists())
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
if __name__ == "__main__":
|
|
899
|
+
sys.exit(main())
|
|
@@ -73,6 +73,8 @@ The packet (`<receipt dir>/packet.json`) carries everything a fresh executor nee
|
|
|
73
73
|
|
|
74
74
|
Writable tool caches go under `scratch`, for example `npm_config_cache=<scratch>/npm`: Codex's sandbox keeps caches in HOME read-only.
|
|
75
75
|
|
|
76
|
+
Build in the worktree's default build directory, never one redirected into `scratch` or shared with another checkout: drain re-runs the checks without your environment. Allocation may have warmed it (`build_dirs` in the receipt).
|
|
77
|
+
|
|
76
78
|
Run `runner` (`acceptance.py run --packet <packet>`) on the committed candidate before review. It prints `{"result": <path>, "passed": ..., "reasons": [...]}`; listing a wholly clean result (empty `reasons`) lets drain reuse its outcomes. Then write the submission:
|
|
77
79
|
|
|
78
80
|
```json
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "devlyn-cli",
|
|
3
|
-
"version": "4.2.
|
|
3
|
+
"version": "4.2.25",
|
|
4
4
|
"description": "Loop & harness engineering for AI coding agents (Claude Code, Codex, and any AGENTS.md-driven agent) — core engineering principles for CLAUDE.md/AGENTS.md and a loop designer that splits one intent into tasks agents drain hands-free",
|
|
5
5
|
"homepage": "https://github.com/fysoul17/devlyn-cli#readme",
|
|
6
6
|
"bin": {
|