devlyn-cli 4.2.24 → 4.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/codex-projection.js +129 -0
- package/bin/devlyn.js +12 -0
- package/bin/engines.js +35 -0
- package/bin/harness-activation.js +427 -0
- package/bin/harness-status.js +171 -0
- package/bin/instructions.js +13 -3
- package/bin/release.js +368 -0
- package/bin/skill-history.json +7 -0
- package/config/skills/_shared/adapters/omp.md +3 -0
- package/config/skills/_shared/runtime-principles.md +12 -2
- package/config/skills/_shared/task-complete.py +5 -0
- package/config/skills/_shared/task-completion.md +15 -4
- package/config/skills/_shared/warm-build-dirs.py +970 -0
- package/config/skills/devlyn-ideate/references/loop.md +2 -0
- package/package.json +7 -1
|
@@ -0,0 +1,970 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Start a new worktree's build directory warm, from a built checkout (task-completion.md).
|
|
3
|
+
|
|
4
|
+
Of the common build tools only Cargo keeps a large cache inside each checkout that a new
|
|
5
|
+
worktree starts without: pnpm, bun, npm, Go and uv share content-addressed caches machine-wide,
|
|
6
|
+
Turborepo shares the main worktree's cache, and the path-keyed caches of Vite, Gradle, Xcode and
|
|
7
|
+
Flutter would miss after a copy. For a Cargo workspace whose default target directory is absent
|
|
8
|
+
and ignored, this clones the dev-profile layouts of the anchor's (else another registered
|
|
9
|
+
worktree's) built target directory into private staging, sharing its blocks (APFS clonefile,
|
|
10
|
+
Linux reflink; never a copy), removes what must not carry over, and moves the result into place:
|
|
11
|
+
|
|
12
|
+
- every unit of the recipient's path packages: Cargo judges their freshness by mtime against the
|
|
13
|
+
other tree's sources;
|
|
14
|
+
- every unit of a package the recipient does not use: nothing classified it;
|
|
15
|
+
- each build-script run whose kept results point into the donor where Cargo does not rewrite
|
|
16
|
+
them: on reuse Cargo rewrites a script's own OUT_DIR in its directives, not paths elsewhere
|
|
17
|
+
(aws-lc-rs's `cargo:include` into aws-lc-sys) nor inside the text it wrote (tauri's permission
|
|
18
|
+
lists);
|
|
19
|
+
- every unit of a package whose crates read OUT_DIR while compiling, by rustc's dep-info record
|
|
20
|
+
of each env!/option_env! read in any spelling, and whose sources or generated Rust compile it
|
|
21
|
+
in other than through include!/include_str!/include_bytes!: Cargo never recompiles that path
|
|
22
|
+
(fingerprint/dep_info.rs). Telling an include from a compiled-in path is a best-effort
|
|
23
|
+
reading of the source, not a proof;
|
|
24
|
+
- incremental caches, final artifacts beside the unit directories, and outputs no fingerprint
|
|
25
|
+
claims.
|
|
26
|
+
|
|
27
|
+
Removal goes by unit hash across the layouts, so no Cargo command touches the clone. Cargo
|
|
28
|
+
rebuilds what it then finds missing, and everything that depends on it. A path that
|
|
29
|
+
survives inside compiled output (a proc-macro's install name, debug maps, a native library or
|
|
30
|
+
a crate that compiled a path in where the reading above does not see it) still names the
|
|
31
|
+
donor's files: when the donor rebuilds or removes them, the recipient can change behavior
|
|
32
|
+
without a warning. That is why a workspace opts in, with `[workspace.metadata.devlyn]
|
|
33
|
+
warm-build-dirs = true` in its Cargo.toml, once its dependencies do not use such paths.
|
|
34
|
+
|
|
35
|
+
The donor is only read: its dev build lock is taken from an existing lock file opened read-only,
|
|
36
|
+
without waiting, while the layouts are cloned. The whole operation has a time limit. Every skip
|
|
37
|
+
is reported with its reason; the target directory then stays absent and the first build is an
|
|
38
|
+
ordinary cold one.
|
|
39
|
+
|
|
40
|
+
warm-build-dirs.py <worktree> [--from <checkout>]
|
|
41
|
+
warm-build-dirs.py --self-test
|
|
42
|
+
"""
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
import argparse
|
|
46
|
+
import contextlib
|
|
47
|
+
import ctypes
|
|
48
|
+
import json
|
|
49
|
+
import os
|
|
50
|
+
from pathlib import Path
|
|
51
|
+
import re
|
|
52
|
+
import shutil
|
|
53
|
+
import stat
|
|
54
|
+
import subprocess
|
|
55
|
+
import sys
|
|
56
|
+
import tempfile
|
|
57
|
+
import time
|
|
58
|
+
import unittest
|
|
59
|
+
import uuid
|
|
60
|
+
|
|
61
|
+
PROFILE_DIR = "debug"
|
|
62
|
+
BUDGET_SECONDS = 300
|
|
63
|
+
CARGO_ENTRIES = {".fingerprint", "build", "deps", "examples", ".cargo-lock"}
|
|
64
|
+
UNIT = re.compile(r"^(?P<name>.+)-(?P<hash>[0-9a-f]{16})(?:\.[^/]*)?$")
|
|
65
|
+
SNIFF_BYTES = 8192 # a NUL among the first bytes marks a binary file
|
|
66
|
+
# Conservative on purpose. Every string literal that spells OUT_DIR, escapes included, counts as
|
|
67
|
+
# a compiled-in path, and only the plain include!(concat!(env!("OUT_DIR"), ...)) forms, which
|
|
68
|
+
# embed contents, offset one each; an env!/option_env! whose argument is not a literal counts too.
|
|
69
|
+
_SPACE = r"(?:\s|//[^\n]*|/\*[\s\S]*?\*/)*"
|
|
70
|
+
# A literal that can decode to OUT_DIR holds only its letters, \x and \u{} escapes and string
|
|
71
|
+
# continuations; every quote may start one (a lookahead, so candidates overlap).
|
|
72
|
+
STRING_CANDIDATE = re.compile(r'"(?=((?:[OUTDIR_]|\\x[0-9a-fA-F]{2}|\\u\{[0-9a-fA-F_]+\}|\\\r?\n\s*)+)")')
|
|
73
|
+
ESCAPE = re.compile(r'\\(?:x([0-9a-fA-F]{2})|u\{([0-9a-fA-F_]+)\}|\n\s*|(.))')
|
|
74
|
+
NONLITERAL_ENV = re.compile(rf'env{_SPACE}!{_SPACE}[(\[{{]{_SPACE}(?!r?#*")')
|
|
75
|
+
# Both start with a literal word, which re scans for quickly; the word's left boundary is checked
|
|
76
|
+
# on each match (a lookbehind first would make re try every position).
|
|
77
|
+
INCLUDED_OUT_DIR = re.compile(
|
|
78
|
+
r'include(?:_str|_bytes)?\s*!\s*\(\s*concat\s*!\s*\(\s*env\s*!\s*\(\s*"OUT_DIR"\s*\)')
|
|
79
|
+
REDIRECTS = ("CARGO_TARGET_DIR", "CARGO_BUILD_TARGET_DIR", "CARGO_BUILD_BUILD_DIR")
|
|
80
|
+
RUSTC_ENV = re.compile(r"^cargo::?rustc-env=[^=]+=")
|
|
81
|
+
# Runs in a child so the time limit can stop a long clone; argv: source, target.
|
|
82
|
+
CLONE_CHILD = """import ctypes, os, sys
|
|
83
|
+
if ctypes.CDLL(None, use_errno=True).clonefile(os.fsencode(sys.argv[1]), os.fsencode(sys.argv[2]), 1) != 0:
|
|
84
|
+
sys.exit(os.strerror(ctypes.get_errno()))
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class Skip(Exception):
|
|
89
|
+
"""An expected reason not to warm; the build starts cold."""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class Unsafe(Exception):
|
|
93
|
+
"""A safety check failed; nothing was published and the donor was not written."""
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def entry(tool, status, **fields):
|
|
97
|
+
return {"tool": tool, "status": status, **fields}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class Deadline:
|
|
101
|
+
def __init__(self, seconds):
|
|
102
|
+
self.seconds = seconds
|
|
103
|
+
self.end = time.monotonic() + seconds
|
|
104
|
+
|
|
105
|
+
def left(self):
|
|
106
|
+
remaining = self.end - time.monotonic()
|
|
107
|
+
if remaining <= 0:
|
|
108
|
+
raise TimeUp(f"warming took longer than {self.seconds} s")
|
|
109
|
+
return remaining
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class TimeUp(Skip):
|
|
113
|
+
"""The time limit ran out; staging is left in scratch for its cleanup."""
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def run(argv, *, cwd, deadline, env=None):
|
|
117
|
+
try:
|
|
118
|
+
result = subprocess.run(argv, cwd=cwd, env=env, capture_output=True, text=True,
|
|
119
|
+
encoding="utf-8", timeout=deadline.left())
|
|
120
|
+
except subprocess.TimeoutExpired as exc:
|
|
121
|
+
raise TimeUp(f"{' '.join(map(str, argv[:3]))} timed out") from exc
|
|
122
|
+
if result.returncode != 0:
|
|
123
|
+
raise Skip(f"{' '.join(map(str, argv[:3]))} failed: {(result.stderr or result.stdout).strip()[-400:]}")
|
|
124
|
+
return result.stdout
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def git(work, *args, deadline):
|
|
128
|
+
return run(["git", "-C", str(work), *args], cwd=work, deadline=deadline).removesuffix("\n")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def without_redirects():
|
|
132
|
+
return {key: value for key, value in os.environ.items() if key not in REDIRECTS}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def metadata(work, deadline, *, env=None, deps=False):
|
|
136
|
+
argv = ["cargo", "metadata", "--format-version", "1", "--frozen", *([] if deps else ["--no-deps"])]
|
|
137
|
+
found = json.loads(run(argv, cwd=work, deadline=deadline, env=env))
|
|
138
|
+
if "build_directory" not in found:
|
|
139
|
+
raise Skip("this Cargo does not report its build directory (Cargo 1.91 or later needed)")
|
|
140
|
+
return found
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def open_beneath(root, parts):
|
|
144
|
+
"""A descriptor for root/parts, reached without following any link below root."""
|
|
145
|
+
fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
|
|
146
|
+
try:
|
|
147
|
+
for part in parts:
|
|
148
|
+
child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
|
149
|
+
os.close(fd)
|
|
150
|
+
fd = child
|
|
151
|
+
except OSError:
|
|
152
|
+
os.close(fd)
|
|
153
|
+
raise
|
|
154
|
+
return fd
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def no_symlink_between(root, path):
|
|
158
|
+
current = root
|
|
159
|
+
for part in path.relative_to(root).parts:
|
|
160
|
+
current = current / part
|
|
161
|
+
if current.is_symlink():
|
|
162
|
+
return False
|
|
163
|
+
return True
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def destination(work, deadline, stack):
|
|
167
|
+
"""The worktree's own Cargo target directory and a descriptor for its parent, when it can be warmed."""
|
|
168
|
+
found = metadata(work, deadline)
|
|
169
|
+
if ((found.get("metadata") or {}).get("devlyn") or {}).get("warm-build-dirs") is not True:
|
|
170
|
+
raise Skip("not enabled: the workspace's Cargo.toml has no [workspace.metadata.devlyn] warm-build-dirs = true")
|
|
171
|
+
if any(key in os.environ for key in REDIRECTS) and metadata(work, deadline, env=without_redirects()) != found:
|
|
172
|
+
raise Skip("the target directory comes from this environment, which later commands may not share")
|
|
173
|
+
target = Path(found["target_directory"])
|
|
174
|
+
if Path(found["build_directory"]) != target:
|
|
175
|
+
raise Skip("a separate Cargo build-dir is configured")
|
|
176
|
+
if not target.is_absolute() or not target.is_relative_to(work) or target == work:
|
|
177
|
+
raise Skip(f"{target} is not inside the worktree")
|
|
178
|
+
relative = target.relative_to(work)
|
|
179
|
+
if relative.parts[0] in {".git", ".devlyn"} or ".." in relative.parts:
|
|
180
|
+
raise Skip(f"{relative} is reserved")
|
|
181
|
+
try:
|
|
182
|
+
parent = stack.enter_context(contextlib.closing(_Fd(open_beneath(work, relative.parts[:-1]))))
|
|
183
|
+
except OSError as exc:
|
|
184
|
+
raise Skip(f"{relative.parent} cannot be reached without a link: {exc.strerror}") from exc
|
|
185
|
+
if os.path.lexists(target):
|
|
186
|
+
raise Skip(f"{relative} already exists")
|
|
187
|
+
if git(work, "ls-files", "--", str(relative), deadline=deadline):
|
|
188
|
+
raise Skip(f"{relative} holds tracked files")
|
|
189
|
+
try:
|
|
190
|
+
# The trailing slash names a directory, which directory-only rules such as `target/` need.
|
|
191
|
+
git(work, "check-ignore", "-q", "--", f"{relative}/", deadline=deadline)
|
|
192
|
+
except TimeUp:
|
|
193
|
+
raise
|
|
194
|
+
except Skip as exc:
|
|
195
|
+
raise Skip(f"{relative} is not ignored by Git") from exc
|
|
196
|
+
return target, relative, parent
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class _Fd:
|
|
200
|
+
def __init__(self, fd):
|
|
201
|
+
self.fd = fd
|
|
202
|
+
|
|
203
|
+
def close(self):
|
|
204
|
+
os.close(self.fd)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def donors(work, anchor, relative, deadline):
|
|
208
|
+
"""Built targets at the same place in the anchor, then the other worktrees, newest first."""
|
|
209
|
+
others = []
|
|
210
|
+
for line in git(work, "worktree", "list", "--porcelain", deadline=deadline).splitlines():
|
|
211
|
+
if line.startswith("worktree "):
|
|
212
|
+
checkout = Path(line[len("worktree "):]).resolve()
|
|
213
|
+
if checkout not in {work, anchor}:
|
|
214
|
+
others.append(checkout)
|
|
215
|
+
found = []
|
|
216
|
+
for checkout in [anchor, *others]:
|
|
217
|
+
donor = checkout / relative
|
|
218
|
+
layout = donor / PROFILE_DIR
|
|
219
|
+
if not (no_symlink_between(checkout, donor) and (layout / ".fingerprint").is_dir() and not layout.is_symlink()):
|
|
220
|
+
continue
|
|
221
|
+
if donor.is_relative_to(work) or work.is_relative_to(donor):
|
|
222
|
+
continue
|
|
223
|
+
found.append((checkout, donor, (layout / ".fingerprint").stat().st_mtime))
|
|
224
|
+
first = [item for item in found if item[0] == anchor]
|
|
225
|
+
return first + sorted((item for item in found if item[0] != anchor), key=lambda item: -item[2])
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@contextlib.contextmanager
|
|
229
|
+
def read_lock(donor):
|
|
230
|
+
"""The donor's dev build lock, held only if free now, through a read-only descriptor."""
|
|
231
|
+
import fcntl
|
|
232
|
+
path = donor / PROFILE_DIR / ".cargo-lock"
|
|
233
|
+
try:
|
|
234
|
+
# O_NONBLOCK: a FIFO in its place must not stall the open.
|
|
235
|
+
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
236
|
+
except OSError as exc:
|
|
237
|
+
raise Skip(f"{path} cannot be opened: {exc.strerror}") from exc
|
|
238
|
+
try:
|
|
239
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
240
|
+
raise Skip(f"{path} is not a regular file")
|
|
241
|
+
try:
|
|
242
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
243
|
+
except BlockingIOError as exc:
|
|
244
|
+
raise Skip(f"a build is using {donor}") from exc
|
|
245
|
+
yield fd, path
|
|
246
|
+
finally:
|
|
247
|
+
os.close(fd)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def same_file(fd, path):
|
|
251
|
+
held, current = os.fstat(fd), os.lstat(path)
|
|
252
|
+
return (held.st_dev, held.st_ino) == (current.st_dev, current.st_ino)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def layouts(target):
|
|
256
|
+
"""The dev-profile layouts under a target directory, host first, relative."""
|
|
257
|
+
found = [Path(PROFILE_DIR)] if (target / PROFILE_DIR / ".fingerprint").is_dir() else []
|
|
258
|
+
for kind in sorted(p for p in target.iterdir() if p.is_dir() and not p.is_symlink() and p.name != PROFILE_DIR):
|
|
259
|
+
if (kind / PROFILE_DIR / ".fingerprint").is_dir() and not (kind / PROFILE_DIR).is_symlink():
|
|
260
|
+
found.append(Path(kind.name, PROFILE_DIR))
|
|
261
|
+
return found
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def clone(source, target, deadline):
|
|
265
|
+
"""Share source's blocks at target (never copy them), within the time limit."""
|
|
266
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
267
|
+
argv = ([sys.executable, "-c", CLONE_CHILD, str(source), str(target)] if sys.platform == "darwin"
|
|
268
|
+
else ["cp", "-a", "--reflink=always", "--", str(source), str(target)])
|
|
269
|
+
try:
|
|
270
|
+
result = subprocess.run(argv, capture_output=True, text=True, timeout=deadline.left())
|
|
271
|
+
except subprocess.TimeoutExpired as exc:
|
|
272
|
+
raise TimeUp(f"cloning {source} timed out") from exc
|
|
273
|
+
if result.returncode != 0:
|
|
274
|
+
raise Skip(f"this filesystem cannot clone {source}: {result.stderr.strip()[-200:]}")
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def publish(staging_parent, staging_name, target_parent, target_name):
|
|
278
|
+
"""Move staging into the checked parent only if nothing exists there."""
|
|
279
|
+
names = os.fsencode(staging_name), os.fsencode(target_name)
|
|
280
|
+
if sys.platform == "darwin":
|
|
281
|
+
rename = ctypes.CDLL(None, use_errno=True).renameatx_np
|
|
282
|
+
rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
|
283
|
+
failed = rename(staging_parent, names[0], target_parent, names[1], 0x4) != 0 # RENAME_EXCL
|
|
284
|
+
else:
|
|
285
|
+
rename = ctypes.CDLL(None, use_errno=True).renameat2
|
|
286
|
+
rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
|
287
|
+
failed = rename(staging_parent, names[0], target_parent, names[1], 1) != 0 # RENAME_NOREPLACE
|
|
288
|
+
if failed:
|
|
289
|
+
raise Skip(f"cannot publish {target_name}: {os.strerror(ctypes.get_errno())}")
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def check_clone(staging, deadline):
|
|
293
|
+
"""No Git data, no special file, and no link that leads out of the clone."""
|
|
294
|
+
pending = [staging]
|
|
295
|
+
while pending:
|
|
296
|
+
deadline.left()
|
|
297
|
+
with os.scandir(pending.pop()) as entries:
|
|
298
|
+
for item in entries:
|
|
299
|
+
if item.name.casefold() == ".git":
|
|
300
|
+
raise Unsafe(f"{item.path} is Git data")
|
|
301
|
+
if item.is_symlink():
|
|
302
|
+
if not Path(os.path.realpath(item.path)).is_relative_to(staging):
|
|
303
|
+
raise Unsafe(f"{item.path} links out of the clone")
|
|
304
|
+
elif item.is_dir(follow_symlinks=False):
|
|
305
|
+
pending.append(item.path)
|
|
306
|
+
elif not item.is_file(follow_symlinks=False):
|
|
307
|
+
raise Unsafe(f"{item.path} is not a regular file or directory")
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def mentions(data, roots):
|
|
311
|
+
"""Whether bytes name a path at or below any root (not a sibling such as `<root>.other`)."""
|
|
312
|
+
return any(re.search(re.escape(os.fsencode(root)) + rb"(?![A-Za-z0-9._-])", data) for root in roots)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def is_text(path):
|
|
316
|
+
"""Whether a regular file (never a link or special file) holds no NUL in its first bytes."""
|
|
317
|
+
if not stat.S_ISREG(path.lstat().st_mode):
|
|
318
|
+
return False
|
|
319
|
+
with open(path, "rb") as stream:
|
|
320
|
+
return b"\0" not in stream.read(SNIFF_BYTES)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def foreign_runs(staging, layout, roots, readers, deadline):
|
|
324
|
+
"""Hashes of build-script runs whose kept results name a donor root Cargo does not rewrite."""
|
|
325
|
+
found = set()
|
|
326
|
+
for run_dir in (staging / layout / "build").iterdir():
|
|
327
|
+
deadline.left()
|
|
328
|
+
root_output = run_dir / "root-output"
|
|
329
|
+
if not root_output.is_file():
|
|
330
|
+
continue # the compiled build script, not a run of it
|
|
331
|
+
old_out = os.fsdecode(root_output.read_bytes())
|
|
332
|
+
# <old target>/<layout>/build/<unit>/out
|
|
333
|
+
tail = (*layout.parts, "build", run_dir.name, "out")
|
|
334
|
+
if not Path(old_out).is_absolute() or Path(old_out).parts[-len(tail):] != tail:
|
|
335
|
+
raise Unsafe(f"{root_output} does not name this run's OUT_DIR")
|
|
336
|
+
needles = {*roots, str(Path(old_out).parents[len(tail) - 1])}
|
|
337
|
+
output = run_dir / "output"
|
|
338
|
+
lines = output.read_text(errors="replace").splitlines() if output.is_file() else []
|
|
339
|
+
# A variable the script hands to rustc is compiled in as it is, own OUT_DIR included.
|
|
340
|
+
exported = [line.split("=", 2)[2] for line in lines if RUSTC_ENV.match(line)]
|
|
341
|
+
foreign = mentions(os.fsencode("\n".join(exported)), {*needles, old_out}) or mentions(
|
|
342
|
+
os.fsencode("\n".join(lines).replace(old_out, "")), needles)
|
|
343
|
+
for directory, _, files in os.walk(run_dir / "out") if not foreign else ():
|
|
344
|
+
deadline.left()
|
|
345
|
+
for name in files:
|
|
346
|
+
path = Path(directory, name)
|
|
347
|
+
deadline.left()
|
|
348
|
+
if is_text(path):
|
|
349
|
+
data = path.read_bytes()
|
|
350
|
+
# Generated Rust is compiled only through an include, which reads OUT_DIR.
|
|
351
|
+
foreign = mentions(data, needles) or (
|
|
352
|
+
name.endswith(".rs") and UNIT.match(run_dir.name)["name"] in readers
|
|
353
|
+
and compiles_path(data.decode(errors="replace")))
|
|
354
|
+
if foreign:
|
|
355
|
+
break
|
|
356
|
+
if foreign:
|
|
357
|
+
break
|
|
358
|
+
if foreign:
|
|
359
|
+
found.add(UNIT.match(run_dir.name)["hash"])
|
|
360
|
+
return found
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def decoded(literal):
|
|
364
|
+
def one(match):
|
|
365
|
+
byte, code, other = match.groups()
|
|
366
|
+
if byte or code:
|
|
367
|
+
return chr(int((byte or code).replace("_", ""), 16))
|
|
368
|
+
return "" if other is None else other # a continuation, or an escaped character
|
|
369
|
+
return ESCAPE.sub(one, literal)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def at_word(text, start, prefix=""):
|
|
373
|
+
"""Whether text[start:] begins a word (or one led by `prefix`, as option_ leads env)."""
|
|
374
|
+
if prefix and text.endswith(prefix, 0, start):
|
|
375
|
+
start -= len(prefix)
|
|
376
|
+
return start == 0 or not (text[start - 1].isalnum() or text[start - 1] == "_")
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def compiles_path(text):
|
|
380
|
+
"""Whether Rust source may compile its OUT_DIR in other than through an include macro."""
|
|
381
|
+
# Without OUT_DIR spelled out or a backslash, no literal can decode to it.
|
|
382
|
+
literals = (sum(decoded(match) == "OUT_DIR" for match in STRING_CANDIDATE.findall(text))
|
|
383
|
+
if "OUT_DIR" in text or "\\" in text else 0)
|
|
384
|
+
included = sum(at_word(text, m.start()) for m in INCLUDED_OUT_DIR.finditer(text)) if "OUT_DIR" in text else 0
|
|
385
|
+
return literals > included or any(at_word(text, m.start(), "option_") for m in NONLITERAL_ENV.finditer(text))
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def compiles_out_dir(package, deadline):
|
|
389
|
+
"""Whether the package's library sources compile its OUT_DIR in."""
|
|
390
|
+
root = Path(package["manifest_path"]).parent
|
|
391
|
+
for directory, dirs, files in os.walk(root):
|
|
392
|
+
deadline.left()
|
|
393
|
+
dirs[:] = [d for d in dirs if d not in {"target", "tests", "benches", "examples", ".git"}]
|
|
394
|
+
for name in files:
|
|
395
|
+
if name.endswith(".rs") and not (Path(directory) == root and name == "build.rs"):
|
|
396
|
+
if compiles_path(Path(directory, name).read_text(encoding="utf-8", errors="replace")):
|
|
397
|
+
return True
|
|
398
|
+
return False
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def remove(item, deadline):
|
|
402
|
+
"""Delete a file or a whole directory without following links, within the time limit."""
|
|
403
|
+
deadline.left()
|
|
404
|
+
if not item.is_dir() or item.is_symlink():
|
|
405
|
+
item.unlink()
|
|
406
|
+
return
|
|
407
|
+
for directory, dirs, files in os.walk(item, topdown=False):
|
|
408
|
+
for name in files:
|
|
409
|
+
deadline.left()
|
|
410
|
+
os.unlink(os.path.join(directory, name))
|
|
411
|
+
for name in dirs:
|
|
412
|
+
deadline.left()
|
|
413
|
+
path = os.path.join(directory, name)
|
|
414
|
+
os.unlink(path) if os.path.islink(path) else os.rmdir(path)
|
|
415
|
+
deadline.left()
|
|
416
|
+
os.rmdir(item)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def remove_units(staging, layout, hashes, deadline):
|
|
420
|
+
for sub in (".fingerprint", "build", "deps", "examples"):
|
|
421
|
+
directory = staging / layout / sub
|
|
422
|
+
for item in directory.iterdir() if directory.is_dir() else ():
|
|
423
|
+
match = UNIT.match(item.name)
|
|
424
|
+
if match and match["hash"] in hashes:
|
|
425
|
+
remove(item, deadline)
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def remove_unclassified(staging, layout, deadline):
|
|
429
|
+
"""Final artifacts Cargo places beside the unit directories carry no unit to classify them by."""
|
|
430
|
+
for item in (staging / layout).iterdir():
|
|
431
|
+
if item.name not in CARGO_ENTRIES:
|
|
432
|
+
remove(item, deadline)
|
|
433
|
+
for sub in ("deps", "examples"):
|
|
434
|
+
directory = staging / layout / sub
|
|
435
|
+
for item in directory.iterdir() if directory.is_dir() else ():
|
|
436
|
+
if not UNIT.match(item.name):
|
|
437
|
+
remove(item, deadline)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def crates_reading_out_dir(staging, found, crates, deadline):
|
|
441
|
+
"""The crates among these that may have read OUT_DIR while compiling, by rustc's dep-info.
|
|
442
|
+
|
|
443
|
+
rustc writes `# env-dep:OUT_DIR=...` into a unit's deps/<crate>-<hash>.d for every
|
|
444
|
+
env!/option_env! read of it, whatever the spelling or macro that made it. A crate counts as
|
|
445
|
+
not reading it only when every kept output of it has that file and none says so. Any
|
|
446
|
+
hashed output counts, with or without the `lib` prefix: rlibs, rmeta, dylibs, Windows DLLs
|
|
447
|
+
and static libraries, test executables.
|
|
448
|
+
"""
|
|
449
|
+
reading, recorded, outputs = set(), {}, {}
|
|
450
|
+
for layout in found:
|
|
451
|
+
for item in (staging / layout / "deps").iterdir():
|
|
452
|
+
deadline.left()
|
|
453
|
+
if not (match := UNIT.match(item.name)):
|
|
454
|
+
continue
|
|
455
|
+
if item.name.endswith(".d"):
|
|
456
|
+
if match["name"] in crates:
|
|
457
|
+
recorded.setdefault(match["name"], set()).add(match["hash"])
|
|
458
|
+
if b"\n# env-dep:OUT_DIR=" in b"\n" + item.read_bytes():
|
|
459
|
+
reading.add(match["name"])
|
|
460
|
+
continue
|
|
461
|
+
stem = match["name"]
|
|
462
|
+
for crate in {stem, stem.removeprefix("lib")} & crates:
|
|
463
|
+
outputs.setdefault(crate, set()).add(match["hash"])
|
|
464
|
+
return reading | {crate for crate, hashes in outputs.items() if hashes - recorded.get(crate, set())}
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def sanitize(work, staging, found, roots, deadline):
|
|
468
|
+
for layout in found:
|
|
469
|
+
remove_unclassified(staging, layout, deadline)
|
|
470
|
+
resolved = metadata(work, deadline, env=without_redirects(), deps=True)
|
|
471
|
+
names = {package["name"] for package in resolved["packages"]}
|
|
472
|
+
path_packages = {package["name"] for package in resolved["packages"] if package["source"] is None}
|
|
473
|
+
# Every version of a package counts on its own; a name is removed if any version must be.
|
|
474
|
+
scripted = [package for package in resolved["packages"]
|
|
475
|
+
if package["source"] is not None and any("custom-build" in t["kind"] for t in package["targets"])]
|
|
476
|
+
crates = [{t["name"].replace("-", "_") for t in package["targets"] if "custom-build" not in t["kind"]}
|
|
477
|
+
for package in scripted]
|
|
478
|
+
reading = crates_reading_out_dir(staging, found, set().union(*crates), deadline)
|
|
479
|
+
reader_packages = [package for package, own in zip(scripted, crates) if own & reading]
|
|
480
|
+
readers = {package["name"] for package in reader_packages}
|
|
481
|
+
out_dir_packages = {package["name"] for package in reader_packages if compiles_out_dir(package, deadline)}
|
|
482
|
+
removed = set()
|
|
483
|
+
for layout in found:
|
|
484
|
+
units = {m["hash"]: m["name"] for item in (staging / layout / ".fingerprint").iterdir()
|
|
485
|
+
if (m := UNIT.match(item.name))}
|
|
486
|
+
drop = {digest for digest, name in units.items()
|
|
487
|
+
if name not in names or name in path_packages or name in out_dir_packages}
|
|
488
|
+
drop |= foreign_runs(staging, layout, roots, readers, deadline)
|
|
489
|
+
removed |= {units[digest] for digest in drop if digest in units}
|
|
490
|
+
# Outputs without a fingerprint belong to no unit Cargo would reuse.
|
|
491
|
+
drop |= {m["hash"] for sub in ("build", "deps", "examples") if (staging / layout / sub).is_dir()
|
|
492
|
+
for item in (staging / layout / sub).iterdir() if (m := UNIT.match(item.name)) and m["hash"] not in units}
|
|
493
|
+
remove_units(staging, layout, drop, deadline)
|
|
494
|
+
return sorted(removed)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def warm_cargo(work, anchor, staging_root, deadline):
|
|
498
|
+
if not (work / "Cargo.toml").is_file():
|
|
499
|
+
return None
|
|
500
|
+
if sys.platform not in {"darwin", "linux"}:
|
|
501
|
+
return entry("cargo", "skipped", reason=f"cloning is unsupported on {sys.platform}")
|
|
502
|
+
started = time.monotonic()
|
|
503
|
+
with contextlib.ExitStack() as stack:
|
|
504
|
+
staging = None
|
|
505
|
+
try:
|
|
506
|
+
if shutil.which("cargo") is None:
|
|
507
|
+
raise Skip("cargo is not on PATH")
|
|
508
|
+
target, relative, parent = destination(work, deadline, stack)
|
|
509
|
+
candidates = donors(work, anchor, relative, deadline)
|
|
510
|
+
if not candidates:
|
|
511
|
+
raise Skip(f"no built {relative} in this repository's checkouts")
|
|
512
|
+
if os.stat(staging_root).st_dev != os.fstat(parent.fd).st_dev:
|
|
513
|
+
raise Skip("the staging and target directories are on different volumes")
|
|
514
|
+
root = stack.enter_context(contextlib.closing(_Fd(os.open(staging_root, os.O_RDONLY | os.O_DIRECTORY))))
|
|
515
|
+
busy = []
|
|
516
|
+
for checkout, donor, _ in candidates:
|
|
517
|
+
if os.stat(donor).st_dev != os.stat(staging_root).st_dev:
|
|
518
|
+
busy.append(f"{donor} is on another volume")
|
|
519
|
+
continue
|
|
520
|
+
name = f".warm-cargo-{uuid.uuid4().hex}"
|
|
521
|
+
try:
|
|
522
|
+
with read_lock(donor) as (fd, lock):
|
|
523
|
+
identity = os.stat(donor / PROFILE_DIR)
|
|
524
|
+
found = layouts(donor)
|
|
525
|
+
os.mkdir(name, dir_fd=root.fd)
|
|
526
|
+
staging = staging_root / name
|
|
527
|
+
for layout in found:
|
|
528
|
+
clone(donor / layout, staging / layout, deadline)
|
|
529
|
+
after = os.stat(donor / PROFILE_DIR)
|
|
530
|
+
if (identity.st_dev, identity.st_ino) != (after.st_dev, after.st_ino) or not same_file(fd, lock):
|
|
531
|
+
raise Unsafe(f"{donor} was replaced while it was cloned")
|
|
532
|
+
except TimeUp:
|
|
533
|
+
raise
|
|
534
|
+
except Skip as exc:
|
|
535
|
+
busy.append(str(exc))
|
|
536
|
+
if staging is not None:
|
|
537
|
+
remove(staging, deadline)
|
|
538
|
+
staging = None
|
|
539
|
+
continue
|
|
540
|
+
check_clone(staging, deadline)
|
|
541
|
+
if (donor / "CACHEDIR.TAG").is_file():
|
|
542
|
+
(staging / "CACHEDIR.TAG").write_bytes((donor / "CACHEDIR.TAG").read_bytes())
|
|
543
|
+
removed = sanitize(work, staging, found, [str(donor), str(checkout)], deadline)
|
|
544
|
+
deadline.left()
|
|
545
|
+
try:
|
|
546
|
+
reached = open_beneath(work, relative.parts[:-1])
|
|
547
|
+
except OSError as exc:
|
|
548
|
+
raise Unsafe(f"{relative.parent} changed while warming: {exc.strerror}") from exc
|
|
549
|
+
try:
|
|
550
|
+
held, now = os.fstat(parent.fd), os.fstat(reached)
|
|
551
|
+
if (held.st_dev, held.st_ino) != (now.st_dev, now.st_ino):
|
|
552
|
+
raise Unsafe(f"{relative.parent} changed while warming")
|
|
553
|
+
finally:
|
|
554
|
+
os.close(reached)
|
|
555
|
+
publish(root.fd, name, parent.fd, relative.name)
|
|
556
|
+
staging = None
|
|
557
|
+
return entry("cargo", "seeded", path=str(target), donor=str(donor),
|
|
558
|
+
removed=removed, seconds=round(time.monotonic() - started, 1))
|
|
559
|
+
raise Skip("; ".join(busy))
|
|
560
|
+
except Skip as exc:
|
|
561
|
+
result = entry("cargo", "skipped", reason=str(exc))
|
|
562
|
+
except Unsafe as exc:
|
|
563
|
+
result = entry("cargo", "failed", reason=str(exc))
|
|
564
|
+
except FileNotFoundError as exc:
|
|
565
|
+
result = entry("cargo", "skipped", reason=f"{exc.filename} disappeared while warming")
|
|
566
|
+
except Exception as exc: # Reported, never raised: allocation goes on cold.
|
|
567
|
+
result = entry("cargo", "failed", reason=f"{type(exc).__name__}: {exc}")
|
|
568
|
+
if staging is not None and os.path.lexists(staging):
|
|
569
|
+
try:
|
|
570
|
+
remove(staging, deadline)
|
|
571
|
+
except TimeUp:
|
|
572
|
+
# Removing a large partial clone would outlast the limit; scratch cleanup owns it.
|
|
573
|
+
result["residue"] = str(staging)
|
|
574
|
+
return result
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def warm(work, anchor, staging_root, *, budget=BUDGET_SECONDS):
|
|
578
|
+
"""Warm every eligible build directory of a new worktree; never raises for a tool's failure."""
|
|
579
|
+
work, anchor, staging_root = Path(work).resolve(), Path(anchor).resolve(), Path(staging_root).resolve()
|
|
580
|
+
results = []
|
|
581
|
+
for tool, warmer in (("cargo", warm_cargo),):
|
|
582
|
+
try:
|
|
583
|
+
result = warmer(work, anchor, staging_root, Deadline(budget))
|
|
584
|
+
except Exception as exc: # Optional warming reports any failure; allocation goes on cold.
|
|
585
|
+
result = entry(tool, "failed", reason=f"{type(exc).__name__}: {exc}")
|
|
586
|
+
if result is not None:
|
|
587
|
+
results.append(result)
|
|
588
|
+
print(f"warm-build-dirs: {tool} {result['status']}: {result.get('path') or result.get('reason')}", file=sys.stderr)
|
|
589
|
+
return results
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def main():
|
|
593
|
+
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
594
|
+
parser.add_argument("worktree", nargs="?", type=Path)
|
|
595
|
+
parser.add_argument("--from", dest="anchor", type=Path, help="the checkout to warm from first (default: the main worktree)")
|
|
596
|
+
parser.add_argument("--self-test", action="store_true")
|
|
597
|
+
args = parser.parse_args()
|
|
598
|
+
if args.self_test:
|
|
599
|
+
return self_test()
|
|
600
|
+
if args.worktree is None:
|
|
601
|
+
parser.error("a worktree is required")
|
|
602
|
+
work = args.worktree.resolve()
|
|
603
|
+
anchor = args.anchor.resolve() if args.anchor else None
|
|
604
|
+
if anchor is None:
|
|
605
|
+
listing = subprocess.run(["git", "-C", str(work), "worktree", "list", "--porcelain"],
|
|
606
|
+
capture_output=True, text=True, check=True).stdout
|
|
607
|
+
anchor = Path(listing.splitlines()[0].removeprefix("worktree ")).resolve()
|
|
608
|
+
common = subprocess.run(["git", "-C", str(work), "rev-parse", "--path-format=absolute", "--git-common-dir"],
|
|
609
|
+
capture_output=True, text=True, check=True).stdout.strip()
|
|
610
|
+
staging_root = Path(tempfile.mkdtemp(prefix=".warm-build-dirs-", dir=common))
|
|
611
|
+
results = warm(work, anchor, staging_root)
|
|
612
|
+
if not any(result.get("residue") for result in results):
|
|
613
|
+
staging_root.rmdir()
|
|
614
|
+
print(json.dumps(results, sort_keys=True))
|
|
615
|
+
return 0
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def self_test():
|
|
619
|
+
result = unittest.TextTestRunner(verbosity=1).run(unittest.defaultTestLoader.loadTestsFromTestCase(WarmTests))
|
|
620
|
+
if result.wasSuccessful():
|
|
621
|
+
print(f"warm-build-dirs self-test: PASS ({result.testsRun} isolated tests)")
|
|
622
|
+
return 0 if result.wasSuccessful() else 1
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
# Fixtures use real Git and a fake `cargo` on PATH that answers `metadata` from a JSON file;
|
|
626
|
+
# a filesystem without cloning copies instead, in tests only, so every rule still runs there.
|
|
627
|
+
FAKE_CARGO = r"""#!/usr/bin/env python3
|
|
628
|
+
import json, os, pathlib, sys, time
|
|
629
|
+
argv = sys.argv[1:]
|
|
630
|
+
fixture = json.loads(pathlib.Path(os.environ["FAKE_CARGO"]).read_text())
|
|
631
|
+
with open(os.environ["FAKE_CARGO_LOG"], "a") as log:
|
|
632
|
+
log.write(json.dumps(argv) + "\n")
|
|
633
|
+
time.sleep(fixture.get("sleep", 0))
|
|
634
|
+
target = pathlib.Path(os.environ.get("CARGO_TARGET_DIR") or os.getcwd() + "/" + fixture.get("target", "target"))
|
|
635
|
+
assert argv[0] == "metadata", argv # warming never runs another cargo command
|
|
636
|
+
found = {"target_directory": str(target), "packages": fixture["packages"] if "--no-deps" not in argv else [],
|
|
637
|
+
"metadata": fixture.get("workspace_metadata", {"devlyn": {"warm-build-dirs": True}})}
|
|
638
|
+
if not fixture.get("old_cargo"):
|
|
639
|
+
found["build_directory"] = fixture.get("build_directory", str(target))
|
|
640
|
+
print(json.dumps(found))
|
|
641
|
+
# The full resolve runs while warming, after the clone: the moment to race the destination.
|
|
642
|
+
if "--no-deps" not in argv:
|
|
643
|
+
if fixture.get("swap") and not os.path.islink("build"):
|
|
644
|
+
os.rename("build", "build-moved")
|
|
645
|
+
os.symlink(fixture["swap"], "build")
|
|
646
|
+
if fixture.get("race"):
|
|
647
|
+
(pathlib.Path(os.getcwd()) / "target").mkdir(exist_ok=True)
|
|
648
|
+
"""
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
class WarmTests(unittest.TestCase):
|
|
652
|
+
def setUp(self):
|
|
653
|
+
# Imported here: unittest.mock imports asyncio, which a caller simulating Windows cannot load.
|
|
654
|
+
from unittest import mock
|
|
655
|
+
temporary = tempfile.TemporaryDirectory(prefix="warm-build-dirs-")
|
|
656
|
+
self.addCleanup(temporary.cleanup)
|
|
657
|
+
self.root = Path(temporary.name).resolve()
|
|
658
|
+
self.anchor, self.work = self.root / "anchor", self.root / "work"
|
|
659
|
+
self.staging = self.root / "scratch"
|
|
660
|
+
self.staging.mkdir()
|
|
661
|
+
bin_dir = self.root / "bin"
|
|
662
|
+
bin_dir.mkdir()
|
|
663
|
+
(bin_dir / "cargo").write_text(FAKE_CARGO)
|
|
664
|
+
(bin_dir / "cargo").chmod(0o755)
|
|
665
|
+
self.fixture, self.log = self.root / "fixture.json", self.root / "cargo.log"
|
|
666
|
+
self.environment = mock.patch.dict(os.environ, {
|
|
667
|
+
"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", "FAKE_CARGO": str(self.fixture),
|
|
668
|
+
"FAKE_CARGO_LOG": str(self.log), "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1",
|
|
669
|
+
"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"})
|
|
670
|
+
self.environment.start()
|
|
671
|
+
self.addCleanup(self.environment.stop)
|
|
672
|
+
for key in REDIRECTS:
|
|
673
|
+
os.environ.pop(key, None)
|
|
674
|
+
self.anchor.mkdir()
|
|
675
|
+
self.git(self.anchor, "init", "-q", "-b", "main")
|
|
676
|
+
(self.anchor / "Cargo.toml").write_text("[workspace]\n")
|
|
677
|
+
(self.anchor / ".gitignore").write_text("target/\n/build/target/\n")
|
|
678
|
+
self.git(self.anchor, "add", "-A")
|
|
679
|
+
self.git(self.anchor, "commit", "-q", "-m", "init")
|
|
680
|
+
self.git(self.anchor, "worktree", "add", "-q", "-b", "task", str(self.work))
|
|
681
|
+
self.write_fixture()
|
|
682
|
+
self.donor = self.anchor / "target"
|
|
683
|
+
self.build_donor()
|
|
684
|
+
probe = self.root / "probe"
|
|
685
|
+
probe.mkdir()
|
|
686
|
+
(probe / "file").write_bytes(b"x" * 65536)
|
|
687
|
+
self.native_clone = True
|
|
688
|
+
try:
|
|
689
|
+
if os.name == "nt":
|
|
690
|
+
raise Skip("no clone primitive")
|
|
691
|
+
clone(probe, self.root / "probe-clone", Deadline(30))
|
|
692
|
+
self.native_clone = (self.root / "probe-clone" / "file").read_bytes() == b"x" * 65536
|
|
693
|
+
except (Skip, OSError):
|
|
694
|
+
self.native_clone = False
|
|
695
|
+
if not self.native_clone:
|
|
696
|
+
# A plain copy that keeps links and special files, as a clone would; tests only.
|
|
697
|
+
def copy(source, target, deadline):
|
|
698
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
699
|
+
subprocess.run(["cp", "-a", "--", str(source), str(target)], check=True)
|
|
700
|
+
patch = mock.patch(f"{__name__}.clone", copy)
|
|
701
|
+
patch.start()
|
|
702
|
+
self.addCleanup(patch.stop)
|
|
703
|
+
|
|
704
|
+
def git(self, work, *args):
|
|
705
|
+
return subprocess.run(["git", "-C", str(work), *args], check=True, capture_output=True, text=True).stdout
|
|
706
|
+
|
|
707
|
+
def write_fixture(self, **extra):
|
|
708
|
+
packages = [
|
|
709
|
+
{"name": "app", "source": None, "manifest_path": str(self.work / "Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}]},
|
|
710
|
+
{"name": "serde", "source": "registry", "manifest_path": str(self.root / "registry/serde/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}]},
|
|
711
|
+
{"name": "ring", "source": "registry", "manifest_path": str(self.root / "registry/ring/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
712
|
+
{"name": "tauri", "source": "registry", "manifest_path": str(self.root / "registry/tauri/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
713
|
+
{"name": "aws-lc-rs", "source": "registry", "manifest_path": str(self.root / "registry/aws-lc-rs/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
714
|
+
{"name": "baked", "source": "registry", "manifest_path": str(self.root / "registry/baked/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
715
|
+
{"name": "exporter", "source": "registry", "manifest_path": str(self.root / "registry/exporter/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
716
|
+
{"name": "generator", "source": "registry", "manifest_path": str(self.root / "registry/generator/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
717
|
+
{"name": "windowsdll", "source": "registry", "manifest_path": str(self.root / "registry/windowsdll/Cargo.toml"), "targets": [{"kind": ["dylib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
718
|
+
# rustc left no dep-info for its library, so nothing proves it never read OUT_DIR.
|
|
719
|
+
{"name": "undocumented", "source": "registry", "manifest_path": str(self.root / "registry/undocumented/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
720
|
+
# Two versions of one package: only 1.0 compiles OUT_DIR in.
|
|
721
|
+
{"name": "dual", "source": "registry", "manifest_path": str(self.root / "registry/dual-1/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
722
|
+
{"name": "dual", "source": "registry", "manifest_path": str(self.root / "registry/dual-2/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
|
|
723
|
+
]
|
|
724
|
+
for package in packages:
|
|
725
|
+
package["targets"] = [{**t, "name": package["name"] if t["name"] == "LIB" else t["name"]} for t in package["targets"]]
|
|
726
|
+
for package in packages[1:]:
|
|
727
|
+
source = Path(package["manifest_path"]).parent / "src"
|
|
728
|
+
source.mkdir(parents=True, exist_ok=True)
|
|
729
|
+
compiled = Path(package["manifest_path"]).parent.name in {"baked", "undocumented", "dual-1", "windowsdll"}
|
|
730
|
+
(source / "lib.rs").write_text('pub const DIR: &str = env!(/* why */ "OUT_DIR");\n' if compiled
|
|
731
|
+
else 'include!(concat!(\n env!("OUT_DIR"), "/gen.rs"));\n')
|
|
732
|
+
self.fixture.write_text(json.dumps({"packages": packages, **extra}))
|
|
733
|
+
|
|
734
|
+
def unit(self, layout, sub, name, digest, files=None):
|
|
735
|
+
directory = self.donor / layout / sub / f"{name}-{digest}"
|
|
736
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
737
|
+
for relative, data in (files or {"stamp": b"x"}).items():
|
|
738
|
+
(directory / relative).parent.mkdir(parents=True, exist_ok=True)
|
|
739
|
+
(directory / relative).write_bytes(data)
|
|
740
|
+
|
|
741
|
+
def run_unit(self, layout, name, digest, output, out_files):
|
|
742
|
+
old_out = f"{self.donor}/{layout}/build/{name}-{digest}/out"
|
|
743
|
+
self.unit(layout, ".fingerprint", name, digest)
|
|
744
|
+
self.unit(layout, "build", name, digest, {"root-output": old_out.encode(), "output": output(old_out).encode(),
|
|
745
|
+
**{f"out/{k}": v(old_out) for k, v in out_files.items()}})
|
|
746
|
+
|
|
747
|
+
def build_donor(self):
|
|
748
|
+
for layout in ("debug", "x86_64-pc-windows-msvc/debug"):
|
|
749
|
+
deps = self.donor / layout / "deps"
|
|
750
|
+
deps.mkdir(parents=True)
|
|
751
|
+
for name, digest in (("serde", "1" * 16), ("app", "2" * 16), ("gone", "7" * 16), ("baked", "8" * 16)):
|
|
752
|
+
self.unit(layout, ".fingerprint", name, digest)
|
|
753
|
+
(deps / f"lib{name}-{digest}.rlib").write_bytes(b"rlib")
|
|
754
|
+
(self.donor / layout / ".cargo-lock").write_bytes(b"")
|
|
755
|
+
# rustc's dep-info: which crates read OUT_DIR while compiling.
|
|
756
|
+
for crate, digest, reads in (("serde", "1" * 16, False), ("baked", "8" * 16, True), ("generator", "b" * 16, True),
|
|
757
|
+
("ring", "c" * 16, False), ("dual", "1a" * 8, True), ("dual", "2b" * 8, True)):
|
|
758
|
+
record = f"# env-dep:OUT_DIR={self.donor}/{layout}/build/{crate}-x/out\n" if reads else ""
|
|
759
|
+
(deps / f"{crate}-{digest}.d").write_text(f"/src/{crate}/lib.rs:\n\n# env-dep:CARGO_PKG_NAME={crate}\n{record}")
|
|
760
|
+
self.unit(layout, ".fingerprint", crate, digest)
|
|
761
|
+
(deps / "libstray-dddddddddddddddd.rlib").write_bytes(b"rlib")
|
|
762
|
+
self.unit(layout, ".fingerprint", "undocumented", "e" * 16)
|
|
763
|
+
(deps / f"libundocumented-{'e' * 16}.rlib").write_bytes(b"rlib")
|
|
764
|
+
# A Windows-style DLL (no `lib` prefix) without dep-info, of a crate that compiles OUT_DIR in.
|
|
765
|
+
self.unit(layout, ".fingerprint", "windowsdll", "9a" * 8)
|
|
766
|
+
(deps / f"windowsdll-{'9a' * 8}.dll").write_bytes(b"dll")
|
|
767
|
+
for digest in ("1a" * 8, "2b" * 8):
|
|
768
|
+
(deps / f"libdual-{digest}.rlib").write_bytes(b"rlib")
|
|
769
|
+
# A run that points into the donor and has no fingerprint left.
|
|
770
|
+
self.unit(layout, "build", "orphanrun", "f" * 16, {
|
|
771
|
+
"root-output": f"{self.donor}/{layout}/build/orphanrun-{'f' * 16}/out".encode(),
|
|
772
|
+
"output": f"cargo:include={self.donor}/{layout}/build/aws-lc-sys-{'6' * 16}/out\n".encode()})
|
|
773
|
+
(self.donor / layout / "incremental" / "app-x").mkdir(parents=True)
|
|
774
|
+
# Final artifacts Cargo puts beside the unit directories, here of the unused `gone`.
|
|
775
|
+
(self.donor / layout / "old-tool").write_bytes(b"bin")
|
|
776
|
+
(self.donor / layout / "examples").mkdir()
|
|
777
|
+
(self.donor / layout / "examples" / "old-example").write_bytes(b"bin")
|
|
778
|
+
self.run_unit(layout, "ring", "4" * 16, lambda out: f"cargo:rustc-link-search=native={out}\n",
|
|
779
|
+
{"libring.a": lambda out: b"\0\x01" + out.encode()})
|
|
780
|
+
self.run_unit(layout, "tauri", "3" * 16, lambda out: f"cargo:rustc-link-search=native={out}\n",
|
|
781
|
+
{"permission-files": lambda out: f'["{out}/app_hide.toml"]'.encode()})
|
|
782
|
+
self.run_unit(layout, "aws-lc-rs", "5" * 16,
|
|
783
|
+
lambda out: f"cargo:include={self.donor}/{layout}/build/aws-lc-sys-{'6' * 16}/out/include\n", {})
|
|
784
|
+
# A variable handed to rustc that carries the script's own OUT_DIR is compiled in as is.
|
|
785
|
+
self.run_unit(layout, "exporter", "9" * 16, lambda out: f"cargo:rustc-env=DEBUG_OUTPUT_DIR={out}\n", {})
|
|
786
|
+
# Generated Rust that compiles its OUT_DIR in names no donor path in its text.
|
|
787
|
+
self.run_unit(layout, "generator", "a" * 16, lambda out: f"cargo:rustc-link-search=native={out}\n",
|
|
788
|
+
{"gen.rs": lambda out: b'pub const DIR: &str = env!("OUT\\x5fDIR");\n'})
|
|
789
|
+
(self.donor / "CACHEDIR.TAG").write_bytes(b"Signature: 8a477f597d28d172789f06886806bc55\n")
|
|
790
|
+
|
|
791
|
+
def snapshot(self, root):
|
|
792
|
+
return {str(p.relative_to(root)): (p.lstat().st_size, p.lstat().st_mtime_ns, p.lstat().st_mode)
|
|
793
|
+
for p in sorted(root.rglob("*"))}
|
|
794
|
+
|
|
795
|
+
def warm(self, **kwargs):
|
|
796
|
+
return warm(self.work, self.anchor, self.staging, **kwargs)
|
|
797
|
+
|
|
798
|
+
def assert_nothing_left(self):
|
|
799
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
800
|
+
self.assertEqual("", self.git(self.work, "status", "--porcelain", "--untracked-files=all"))
|
|
801
|
+
|
|
802
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
803
|
+
def test_seeds_only_what_carries_over_and_never_writes_the_donor(self):
|
|
804
|
+
before = self.snapshot(self.donor)
|
|
805
|
+
[result] = self.warm()
|
|
806
|
+
self.assertEqual("seeded", result["status"], result)
|
|
807
|
+
target = self.work / "target"
|
|
808
|
+
for layout in ("debug", "x86_64-pc-windows-msvc/debug"):
|
|
809
|
+
kept = {p.name for p in (target / layout / ".fingerprint").iterdir()}
|
|
810
|
+
# generator keeps its library unit: Cargo rebuilds it after rerunning the removed run.
|
|
811
|
+
self.assertEqual({f"serde-{'1' * 16}", f"ring-{'4' * 16}", f"ring-{'c' * 16}", f"generator-{'b' * 16}"}, kept, layout)
|
|
812
|
+
self.assertFalse((target / layout / "deps" / "libstray-dddddddddddddddd.rlib").exists())
|
|
813
|
+
self.assertEqual({".fingerprint", "build", "deps", "examples", ".cargo-lock"}, {p.name for p in (target / layout).iterdir()})
|
|
814
|
+
self.assertEqual([], list((target / layout / "examples").iterdir()))
|
|
815
|
+
self.assertTrue((target / layout / "build" / f"ring-{'4' * 16}" / "out" / "libring.a").exists())
|
|
816
|
+
self.assertEqual(["app", "aws-lc-rs", "baked", "dual", "exporter", "generator", "gone", "tauri", "undocumented",
|
|
817
|
+
"windowsdll"],
|
|
818
|
+
result["removed"])
|
|
819
|
+
for layout in ("debug", "x86_64-pc-windows-msvc/debug"):
|
|
820
|
+
self.assertFalse((target / layout / "build" / f"orphanrun-{'f' * 16}").exists())
|
|
821
|
+
self.assertFalse(any((target / layout / "deps").glob("libdual-*")))
|
|
822
|
+
self.assertEqual(str(self.donor), result["donor"])
|
|
823
|
+
self.assertEqual(before, self.snapshot(self.donor))
|
|
824
|
+
self.assert_nothing_left()
|
|
825
|
+
|
|
826
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
827
|
+
def test_a_busy_donor_is_skipped_without_waiting(self):
|
|
828
|
+
import fcntl
|
|
829
|
+
with open(self.donor / "debug" / ".cargo-lock", "rb") as lock:
|
|
830
|
+
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
831
|
+
[result] = self.warm()
|
|
832
|
+
self.assertEqual("skipped", result["status"])
|
|
833
|
+
self.assertIn("a build is using", result["reason"])
|
|
834
|
+
self.assertFalse((self.work / "target").exists())
|
|
835
|
+
self.assert_nothing_left()
|
|
836
|
+
|
|
837
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
838
|
+
def test_ineligible_destinations_are_skipped_with_their_reason(self):
|
|
839
|
+
cases = {
|
|
840
|
+
"already exists": lambda: (self.work / "target").mkdir(),
|
|
841
|
+
"not ignored": lambda: (self.work / ".gitignore").write_text(""),
|
|
842
|
+
"not enabled": lambda: self.write_fixture(workspace_metadata=None),
|
|
843
|
+
"separate Cargo build-dir": lambda: self.write_fixture(build_directory=str(self.root / "elsewhere")),
|
|
844
|
+
"build directory (Cargo 1.91": lambda: self.write_fixture(old_cargo=True),
|
|
845
|
+
"comes from this environment": lambda: os.environ.__setitem__("CARGO_TARGET_DIR", str(self.root / "t")),
|
|
846
|
+
"no built target": lambda: shutil.rmtree(self.donor),
|
|
847
|
+
}
|
|
848
|
+
for reason, arrange in cases.items():
|
|
849
|
+
with self.subTest(reason):
|
|
850
|
+
self.setUp()
|
|
851
|
+
arrange()
|
|
852
|
+
[result] = self.warm()
|
|
853
|
+
self.assertEqual("skipped", result["status"], result)
|
|
854
|
+
self.assertIn(reason, result["reason"])
|
|
855
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
856
|
+
|
|
857
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
858
|
+
def test_a_clone_with_git_data_or_a_link_out_is_refused(self):
|
|
859
|
+
for name, arrange in {
|
|
860
|
+
"links out": lambda: (self.donor / "debug" / "build" / "link").symlink_to(self.root),
|
|
861
|
+
"Git data": lambda: (self.donor / "debug" / "deps" / ".git").mkdir(),
|
|
862
|
+
}.items():
|
|
863
|
+
with self.subTest(name):
|
|
864
|
+
self.setUp()
|
|
865
|
+
arrange()
|
|
866
|
+
before = self.snapshot(self.donor)
|
|
867
|
+
[result] = self.warm()
|
|
868
|
+
self.assertEqual("failed", result["status"], result)
|
|
869
|
+
self.assertIn(name, result["reason"])
|
|
870
|
+
self.assertFalse(os.path.lexists(self.work / "target"))
|
|
871
|
+
self.assertEqual(before, self.snapshot(self.donor))
|
|
872
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
873
|
+
|
|
874
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
875
|
+
def test_running_out_of_time_publishes_nothing(self):
|
|
876
|
+
self.write_fixture(sleep=2)
|
|
877
|
+
[result] = self.warm(budget=1)
|
|
878
|
+
self.assertEqual("skipped", result["status"])
|
|
879
|
+
self.assertIn("timed out", result["reason"])
|
|
880
|
+
self.assertFalse((self.work / "target").exists())
|
|
881
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
882
|
+
|
|
883
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
884
|
+
def test_a_target_that_appears_meanwhile_is_never_replaced(self):
|
|
885
|
+
self.write_fixture(race=True)
|
|
886
|
+
[result] = self.warm()
|
|
887
|
+
self.assertEqual("skipped", result["status"])
|
|
888
|
+
self.assertIn("cannot publish", result["reason"])
|
|
889
|
+
self.assertEqual([], list((self.work / "target").iterdir()))
|
|
890
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
891
|
+
|
|
892
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
893
|
+
def test_a_parent_replaced_while_warming_receives_nothing(self):
|
|
894
|
+
(self.anchor / "build").mkdir()
|
|
895
|
+
self.donor = self.anchor / "build" / "target"
|
|
896
|
+
(self.anchor / "target").rename(self.donor)
|
|
897
|
+
(self.work / "build").mkdir()
|
|
898
|
+
elsewhere = self.root / "elsewhere"
|
|
899
|
+
elsewhere.mkdir()
|
|
900
|
+
self.write_fixture(target="build/target", swap=str(elsewhere))
|
|
901
|
+
[result] = self.warm()
|
|
902
|
+
self.assertEqual("failed", result["status"], result)
|
|
903
|
+
self.assertIn("changed while warming", result["reason"])
|
|
904
|
+
self.assertEqual([], list(elsewhere.iterdir()))
|
|
905
|
+
self.assertFalse((self.work / "build-moved" / "target").exists())
|
|
906
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
907
|
+
|
|
908
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
909
|
+
def test_a_special_file_is_never_read(self):
|
|
910
|
+
os.mkfifo(self.donor / "debug" / "build" / f"tauri-{'3' * 16}" / "out" / "pipe")
|
|
911
|
+
[result] = self.warm(budget=20)
|
|
912
|
+
self.assertEqual("failed", result["status"], result)
|
|
913
|
+
self.assertIn("not a regular file or directory", result["reason"])
|
|
914
|
+
self.assertFalse((self.work / "target").exists())
|
|
915
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
916
|
+
|
|
917
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
918
|
+
def test_a_malformed_build_record_fails_explicitly(self):
|
|
919
|
+
(self.donor / "debug" / "build" / f"tauri-{'3' * 16}" / "root-output").write_bytes(b"")
|
|
920
|
+
[result] = self.warm()
|
|
921
|
+
self.assertEqual("failed", result["status"], result)
|
|
922
|
+
self.assertIn("does not name this run's OUT_DIR", result["reason"])
|
|
923
|
+
self.assertFalse((self.work / "target").exists())
|
|
924
|
+
self.assertEqual([], list(self.staging.iterdir()))
|
|
925
|
+
|
|
926
|
+
@unittest.skipIf(os.name == "nt", "warming is POSIX-only")
|
|
927
|
+
def test_allocation_warms_its_worktree_and_records_the_outcome(self):
|
|
928
|
+
tree = self.root / "allocated"
|
|
929
|
+
head = self.git(self.anchor, "rev-parse", "HEAD").strip()
|
|
930
|
+
completed = subprocess.run([sys.executable, str(Path(__file__).with_name("task-complete.py")), "allocate",
|
|
931
|
+
"--repo", str(self.anchor), "--task", "warm", "--branch", "warm-allocation",
|
|
932
|
+
"--worktree", str(tree), "--local-base", head], capture_output=True, text=True)
|
|
933
|
+
self.assertEqual(0, completed.returncode, completed.stdout + completed.stderr)
|
|
934
|
+
result = json.loads(completed.stdout)
|
|
935
|
+
[warmed] = result["build_dirs"]
|
|
936
|
+
if self.native_clone:
|
|
937
|
+
self.assertEqual(("seeded", str(tree / "target")), (warmed["status"], warmed.get("path")), warmed)
|
|
938
|
+
else: # The child cannot use the fixture's copy; a filesystem without cloning must skip.
|
|
939
|
+
self.assertEqual("skipped", warmed["status"], warmed)
|
|
940
|
+
self.assertIn("cannot clone", warmed["reason"])
|
|
941
|
+
self.assertEqual(result["build_dirs"], json.loads(Path(result["receipt"]).read_text())["build_dirs"])
|
|
942
|
+
self.assertEqual("", self.git(tree, "status", "--porcelain", "--untracked-files=all"))
|
|
943
|
+
self.assertEqual([], list(Path(result["scratch"]).iterdir()))
|
|
944
|
+
|
|
945
|
+
def test_compiled_in_out_dir_is_recognized_in_every_spelling(self):
|
|
946
|
+
compiled = ['env!("OUT_DIR")', 'env!("OUT_DIR", "missing")', 'env ! ( "OUT_DIR" )', 'std::env!(r"OUT_DIR")',
|
|
947
|
+
'option_env!("OUT_DIR")', 'myinclude!(concat!(env!("OUT_DIR"), "/a.rs"))', 'env![ "OUT_DIR" ]',
|
|
948
|
+
'env/**/!("OUT_DIR")', 'env!(/* why */ "OUT_DIR")', 'env!("OUT\\x5fDIR")', 'env!("\\u{4f}UT_DIR")',
|
|
949
|
+
'use std::env as e; e!("OUT_DIR")', 'env!(concat!("OUT", "_DIR"))', 'env!(// note\n"OUT_DIR")',
|
|
950
|
+
'env!("\\u{4_f}UT_DIR")', 'env!("\\u{0_0_0_0_4_f}UT_DIR")', 'env!("OUT_\\\n DIR")', 'let a = \'"\'; env!("OUT_DIR")',
|
|
951
|
+
'include!(concat!(env!(/* c */ "OUT_DIR"), "/a.rs"))']
|
|
952
|
+
for text in compiled:
|
|
953
|
+
self.assertTrue(compiles_path(text), text)
|
|
954
|
+
for text in ['include!(concat!(env!("OUT_DIR"), "/a.rs"));', 'include_bytes!(concat!(\n env!("OUT_DIR"),\n "/b"))',
|
|
955
|
+
'env!("OUT_DIRS")', 'env!("CARGO_PKG_NAME")', 'let s = "OUT_DIR_NAME";']:
|
|
956
|
+
self.assertFalse(compiles_path(text), text)
|
|
957
|
+
|
|
958
|
+
def test_no_cargo_workspace_or_an_unsupported_platform_does_nothing(self):
|
|
959
|
+
(self.work / "Cargo.toml").unlink()
|
|
960
|
+
self.assertEqual([], self.warm())
|
|
961
|
+
(self.work / "Cargo.toml").write_text("[workspace]\n")
|
|
962
|
+
from unittest import mock
|
|
963
|
+
with mock.patch.object(sys, "platform", "win32"):
|
|
964
|
+
[result] = self.warm()
|
|
965
|
+
self.assertEqual(("skipped", "cloning is unsupported on win32"), (result["status"], result["reason"]))
|
|
966
|
+
self.assertFalse(self.log.exists())
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
if __name__ == "__main__":
|
|
970
|
+
sys.exit(main())
|