devlyn-cli 4.2.26 → 4.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/bin/devlyn.js +510 -0
  2. package/bin/engines.js +20 -15
  3. package/bin/harness-activation.js +1 -2
  4. package/bin/release.js +16 -3
  5. package/bin/skill-history.json +5 -0
  6. package/config/skills/_shared/harness-store.py +202 -0
  7. package/config/skills/_shared/task-completion.md +6 -3
  8. package/config/skills/_shared/warm-build-dirs.py +97 -2
  9. package/config/skills/devlyn-ideate/SKILL.md +5 -1
  10. package/config/skills/devlyn-ideate/references/loop.md +12 -2
  11. package/config/skills/devlyn-ideate/references/plan-add.md +12 -0
  12. package/config/skills/devlyn-ideate/scripts/acceptance.py +5 -0
  13. package/config/skills/devlyn-ideate/scripts/queue.py +165 -15
  14. package/harness/AGENTS.md +37 -0
  15. package/harness/CLAUDE.md +37 -0
  16. package/native/claude/.claude-plugin/plugin.json +6 -0
  17. package/native/claude/skills/_shared/.ruff.toml +2 -0
  18. package/native/claude/skills/_shared/adapters/claude.md +15 -0
  19. package/native/claude/skills/_shared/adapters/codex.md +14 -0
  20. package/native/claude/skills/_shared/adapters/omp.md +6 -0
  21. package/native/claude/skills/_shared/codex-monitored.sh +263 -0
  22. package/native/claude/skills/_shared/engine-doctor.sh +70 -0
  23. package/native/claude/skills/_shared/engine-preflight.md +29 -0
  24. package/native/claude/skills/_shared/expected-contract.py +242 -0
  25. package/native/claude/skills/_shared/expected.schema.json +120 -0
  26. package/native/claude/skills/_shared/harness-store.py +202 -0
  27. package/native/claude/skills/_shared/invocation-receipt.py +499 -0
  28. package/native/claude/skills/_shared/platform-support.py +395 -0
  29. package/native/claude/skills/_shared/role-config.py +247 -0
  30. package/native/claude/skills/_shared/runtime-principles.md +35 -0
  31. package/native/claude/skills/_shared/task-complete.py +2865 -0
  32. package/native/claude/skills/_shared/task-completion.md +249 -0
  33. package/native/claude/skills/_shared/warm-build-dirs.py +1065 -0
  34. package/native/claude/skills/devlyn-engines/SKILL.md +63 -0
  35. package/native/claude/skills/devlyn-ideate/SKILL.md +132 -0
  36. package/native/claude/skills/devlyn-ideate/references/elicitation.md +29 -0
  37. package/native/claude/skills/devlyn-ideate/references/loop.md +123 -0
  38. package/native/claude/skills/devlyn-ideate/references/package-format.md +128 -0
  39. package/native/claude/skills/devlyn-ideate/references/plan-add.md +12 -0
  40. package/native/claude/skills/devlyn-ideate/scripts/acceptance.py +882 -0
  41. package/native/claude/skills/devlyn-ideate/scripts/queue.py +2191 -0
  42. package/native/codex/.agents/plugins/marketplace.json +12 -0
  43. package/native/codex/plugins/devlyn/.codex-plugin/plugin.json +6 -0
  44. package/native/codex/plugins/devlyn/skills/_shared/.ruff.toml +2 -0
  45. package/native/codex/plugins/devlyn/skills/_shared/adapters/claude.md +15 -0
  46. package/native/codex/plugins/devlyn/skills/_shared/adapters/codex.md +14 -0
  47. package/native/codex/plugins/devlyn/skills/_shared/adapters/omp.md +6 -0
  48. package/native/codex/plugins/devlyn/skills/_shared/codex-monitored.sh +263 -0
  49. package/native/codex/plugins/devlyn/skills/_shared/engine-doctor.sh +70 -0
  50. package/native/codex/plugins/devlyn/skills/_shared/engine-preflight.md +29 -0
  51. package/native/codex/plugins/devlyn/skills/_shared/expected-contract.py +242 -0
  52. package/native/codex/plugins/devlyn/skills/_shared/expected.schema.json +120 -0
  53. package/native/codex/plugins/devlyn/skills/_shared/harness-store.py +202 -0
  54. package/native/codex/plugins/devlyn/skills/_shared/invocation-receipt.py +499 -0
  55. package/native/codex/plugins/devlyn/skills/_shared/platform-support.py +395 -0
  56. package/native/codex/plugins/devlyn/skills/_shared/role-config.py +247 -0
  57. package/native/codex/plugins/devlyn/skills/_shared/runtime-principles.md +35 -0
  58. package/native/codex/plugins/devlyn/skills/_shared/task-complete.py +2865 -0
  59. package/native/codex/plugins/devlyn/skills/_shared/task-completion.md +249 -0
  60. package/native/codex/plugins/devlyn/skills/_shared/warm-build-dirs.py +1065 -0
  61. package/native/codex/plugins/devlyn/skills/devlyn-engines/SKILL.md +63 -0
  62. package/native/codex/plugins/devlyn/skills/devlyn-ideate/SKILL.md +132 -0
  63. package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/elicitation.md +29 -0
  64. package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/loop.md +123 -0
  65. package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/package-format.md +128 -0
  66. package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/plan-add.md +12 -0
  67. package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/acceptance.py +882 -0
  68. package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/queue.py +2191 -0
  69. package/package.json +3 -2
  70. package/release.json +1630 -0
@@ -0,0 +1,1065 @@
1
+ #!/usr/bin/env python3
2
+ """Start a new worktree's build directory warm, from a built checkout (task-completion.md).
3
+
4
+ Of the common build tools only Cargo keeps a large cache inside each checkout that a new
5
+ worktree starts without: pnpm, bun, npm, Go and uv share content-addressed caches machine-wide,
6
+ Turborepo shares the main worktree's cache, and the path-keyed caches of Vite, Gradle, Xcode and
7
+ Flutter would miss after a copy. For a Cargo workspace whose default target directory is absent
8
+ and ignored, this clones the dev-profile layouts of the anchor's (else another registered
9
+ worktree's) built target directory into private staging, sharing its blocks (APFS clonefile,
10
+ Linux reflink; never a copy), removes what must not carry over, and moves the result into place:
11
+
12
+ - every unit of the recipient's path packages: Cargo judges their freshness by mtime against the
13
+ other tree's sources;
14
+ - every unit of a package the recipient does not use: nothing classified it;
15
+ - each build-script run whose kept results point into the donor where Cargo does not rewrite
16
+ them: on reuse Cargo rewrites a script's own OUT_DIR in its directives, not paths elsewhere
17
+ (aws-lc-rs's `cargo:include` into aws-lc-sys) nor inside the text it wrote (tauri's permission
18
+ lists);
19
+ - every unit of a package whose crates read OUT_DIR while compiling, by rustc's dep-info record
20
+ of each env!/option_env! read in any spelling, and whose sources or generated Rust compile it
21
+ in other than through include!/include_str!/include_bytes!: Cargo never recompiles that path
22
+ (fingerprint/dep_info.rs). Telling an include from a compiled-in path is a best-effort
23
+ reading of the source, not a proof;
24
+ - incremental caches, final artifacts beside the unit directories, and outputs no fingerprint
25
+ claims.
26
+
27
+ Removal goes by unit hash across the layouts, so no Cargo command touches the clone. Cargo
28
+ rebuilds what it then finds missing, and everything that depends on it. A path that
29
+ survives inside compiled output (a proc-macro's install name, debug maps, a native library or
30
+ a crate that compiled a path in where the reading above does not see it) still names the
31
+ donor's files: when the donor rebuilds or removes them, the recipient can change behavior
32
+ without a warning. That is why a workspace opts in, with `[workspace.metadata.devlyn]
33
+ warm-build-dirs = true` in its Cargo.toml, once its dependencies do not use such paths.
34
+ The repository's shared local Git config `devlyn.warmBuildDirs` overrides that opt-in on
35
+ this machine: true enables, false disables, unset uses Cargo.toml, and invalid values skip.
36
+ Global, system, include and per-worktree config values are not consulted.
37
+
38
+ The donor is only read: its dev build lock is taken from an existing lock file opened read-only,
39
+ without waiting, while the layouts are cloned. The whole operation has a time limit. Every skip
40
+ is reported with its reason; the target directory then stays absent and the first build is an
41
+ ordinary cold one.
42
+
43
+ warm-build-dirs.py <worktree> [--from <checkout>]
44
+ warm-build-dirs.py --self-test
45
+ """
46
+ from __future__ import annotations
47
+
48
+ import argparse
49
+ import contextlib
50
+ import ctypes
51
+ import json
52
+ import os
53
+ from pathlib import Path
54
+ import re
55
+ import shutil
56
+ import stat
57
+ import subprocess
58
+ import sys
59
+ import tempfile
60
+ import time
61
+ import unittest
62
+ import uuid
63
+
64
+ PROFILE_DIR = "debug"
65
+ BUDGET_SECONDS = 300
66
+ CARGO_ENTRIES = {".fingerprint", "build", "deps", "examples", ".cargo-lock"}
67
+ UNIT = re.compile(r"^(?P<name>.+)-(?P<hash>[0-9a-f]{16})(?:\.[^/]*)?$")
68
+ SNIFF_BYTES = 8192 # a NUL among the first bytes marks a binary file
69
+ # Conservative on purpose. Every string literal that spells OUT_DIR, escapes included, counts as
70
+ # a compiled-in path, and only the plain include!(concat!(env!("OUT_DIR"), ...)) forms, which
71
+ # embed contents, offset one each; an env!/option_env! whose argument is not a literal counts too.
72
+ _SPACE = r"(?:\s|//[^\n]*|/\*[\s\S]*?\*/)*"
73
+ # A literal that can decode to OUT_DIR holds only its letters, \x and \u{} escapes and string
74
+ # continuations; every quote may start one (a lookahead, so candidates overlap).
75
+ STRING_CANDIDATE = re.compile(r'"(?=((?:[OUTDIR_]|\\x[0-9a-fA-F]{2}|\\u\{[0-9a-fA-F_]+\}|\\\r?\n\s*)+)")')
76
+ ESCAPE = re.compile(r'\\(?:x([0-9a-fA-F]{2})|u\{([0-9a-fA-F_]+)\}|\n\s*|(.))')
77
+ NONLITERAL_ENV = re.compile(rf'env{_SPACE}!{_SPACE}[(\[{{]{_SPACE}(?!r?#*")')
78
+ # Both start with a literal word, which re scans for quickly; the word's left boundary is checked
79
+ # on each match (a lookbehind first would make re try every position).
80
+ INCLUDED_OUT_DIR = re.compile(
81
+ r'include(?:_str|_bytes)?\s*!\s*\(\s*concat\s*!\s*\(\s*env\s*!\s*\(\s*"OUT_DIR"\s*\)')
82
+ REDIRECTS = ("CARGO_TARGET_DIR", "CARGO_BUILD_TARGET_DIR", "CARGO_BUILD_BUILD_DIR")
83
+ RUSTC_ENV = re.compile(r"^cargo::?rustc-env=[^=]+=")
84
+ # Runs in a child so the time limit can stop a long clone; argv: source, target.
85
+ CLONE_CHILD = """import ctypes, os, sys
86
+ if ctypes.CDLL(None, use_errno=True).clonefile(os.fsencode(sys.argv[1]), os.fsencode(sys.argv[2]), 1) != 0:
87
+ sys.exit(os.strerror(ctypes.get_errno()))
88
+ """
89
+
90
+
91
+ class Skip(Exception):
92
+ """An expected reason not to warm; the build starts cold."""
93
+
94
+
95
+ class CommandFailed(Skip):
96
+ """A command failed; retain its exit code and diagnostics for callers."""
97
+
98
+ def __init__(self, argv, result):
99
+ self.result = result
100
+ super().__init__(f"{' '.join(map(str, argv[:3]))} failed: {(result.stderr or result.stdout).strip()[-400:]}")
101
+
102
+
103
+ class Unsafe(Exception):
104
+ """A safety check failed; nothing was published and the donor was not written."""
105
+
106
+
107
+ def entry(tool, status, **fields):
108
+ return {"tool": tool, "status": status, **fields}
109
+
110
+
111
+ class Deadline:
112
+ def __init__(self, seconds):
113
+ self.seconds = seconds
114
+ self.end = time.monotonic() + seconds
115
+
116
+ def left(self):
117
+ remaining = self.end - time.monotonic()
118
+ if remaining <= 0:
119
+ raise TimeUp(f"warming took longer than {self.seconds} s")
120
+ return remaining
121
+
122
+
123
+ class TimeUp(Skip):
124
+ """The time limit ran out; staging is left in scratch for its cleanup."""
125
+
126
+
127
+ def run(argv, *, cwd, deadline, env=None):
128
+ try:
129
+ result = subprocess.run(argv, cwd=cwd, env=env, capture_output=True, text=True,
130
+ encoding="utf-8", timeout=deadline.left())
131
+ except subprocess.TimeoutExpired as exc:
132
+ raise TimeUp(f"{' '.join(map(str, argv[:3]))} timed out") from exc
133
+ if result.returncode != 0:
134
+ raise CommandFailed(argv, result)
135
+ return result.stdout
136
+
137
+
138
+ def git(work, *args, deadline):
139
+ return run(["git", "-C", str(work), *args], cwd=work, deadline=deadline).removesuffix("\n")
140
+
141
+
142
+ def without_redirects():
143
+ return {key: value for key, value in os.environ.items() if key not in REDIRECTS}
144
+
145
+
146
+ def metadata(work, deadline, *, env=None, deps=False):
147
+ argv = ["cargo", "metadata", "--format-version", "1", "--frozen", *([] if deps else ["--no-deps"])]
148
+ found = json.loads(run(argv, cwd=work, deadline=deadline, env=env))
149
+ if "build_directory" not in found:
150
+ raise Skip("this Cargo does not report its build directory (Cargo 1.91 or later needed)")
151
+ return found
152
+
153
+
154
+ def open_beneath(root, parts):
155
+ """A descriptor for root/parts, reached without following any link below root."""
156
+ fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
157
+ try:
158
+ for part in parts:
159
+ child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
160
+ os.close(fd)
161
+ fd = child
162
+ except OSError:
163
+ os.close(fd)
164
+ raise
165
+ return fd
166
+
167
+
168
+ def no_symlink_between(root, path):
169
+ current = root
170
+ for part in path.relative_to(root).parts:
171
+ current = current / part
172
+ if current.is_symlink():
173
+ return False
174
+ return True
175
+
176
+
177
+ def destination(work, deadline, stack):
178
+ """The worktree's own Cargo target directory and a descriptor for its parent, when it can be warmed."""
179
+ found = metadata(work, deadline)
180
+ setting = "git config --local devlyn.warmBuildDirs"
181
+ try:
182
+ enabled = git(work, "config", "--local", "--type=bool", "--get", "devlyn.warmBuildDirs", deadline=deadline)
183
+ except CommandFailed as exc:
184
+ if exc.result.returncode != 1:
185
+ error = (exc.result.stderr or exc.result.stdout or str(exc)).splitlines()[0]
186
+ raise Skip(f"cannot read {setting}: {error}") from exc
187
+ enabled = None
188
+ except OSError as exc:
189
+ raise Skip(f"cannot read {setting}: {exc}") from exc
190
+ if enabled == "false":
191
+ raise Skip(f"turned off: {setting} is false")
192
+ if enabled != "true" and ((found.get("metadata") or {}).get("devlyn") or {}).get("warm-build-dirs") is not True:
193
+ raise Skip("not enabled: the workspace's Cargo.toml has no [workspace.metadata.devlyn] warm-build-dirs = true")
194
+ if any(key in os.environ for key in REDIRECTS) and metadata(work, deadline, env=without_redirects()) != found:
195
+ raise Skip("the target directory comes from this environment, which later commands may not share")
196
+ target = Path(found["target_directory"])
197
+ if Path(found["build_directory"]) != target:
198
+ raise Skip("a separate Cargo build-dir is configured")
199
+ if not target.is_absolute() or not target.is_relative_to(work) or target == work:
200
+ raise Skip(f"{target} is not inside the worktree")
201
+ relative = target.relative_to(work)
202
+ if relative.parts[0] in {".git", ".devlyn"} or ".." in relative.parts:
203
+ raise Skip(f"{relative} is reserved")
204
+ try:
205
+ parent = stack.enter_context(contextlib.closing(_Fd(open_beneath(work, relative.parts[:-1]))))
206
+ except OSError as exc:
207
+ raise Skip(f"{relative.parent} cannot be reached without a link: {exc.strerror}") from exc
208
+ if os.path.lexists(target):
209
+ raise Skip(f"{relative} already exists")
210
+ if git(work, "ls-files", "--", str(relative), deadline=deadline):
211
+ raise Skip(f"{relative} holds tracked files")
212
+ try:
213
+ # The trailing slash names a directory, which directory-only rules such as `target/` need.
214
+ git(work, "check-ignore", "-q", "--", f"{relative}/", deadline=deadline)
215
+ except TimeUp:
216
+ raise
217
+ except Skip as exc:
218
+ raise Skip(f"{relative} is not ignored by Git") from exc
219
+ return target, relative, parent
220
+
221
+
222
+ class _Fd:
223
+ def __init__(self, fd):
224
+ self.fd = fd
225
+
226
+ def close(self):
227
+ os.close(self.fd)
228
+
229
+
230
+ def donors(work, anchor, relative, deadline):
231
+ """Built targets at the same place in the anchor, then the other worktrees, newest first."""
232
+ others = []
233
+ for line in git(work, "worktree", "list", "--porcelain", deadline=deadline).splitlines():
234
+ if line.startswith("worktree "):
235
+ checkout = Path(line[len("worktree "):]).resolve()
236
+ if checkout not in {work, anchor}:
237
+ others.append(checkout)
238
+ found = []
239
+ for checkout in [anchor, *others]:
240
+ donor = checkout / relative
241
+ layout = donor / PROFILE_DIR
242
+ if not (no_symlink_between(checkout, donor) and (layout / ".fingerprint").is_dir() and not layout.is_symlink()):
243
+ continue
244
+ if donor.is_relative_to(work) or work.is_relative_to(donor):
245
+ continue
246
+ found.append((checkout, donor, (layout / ".fingerprint").stat().st_mtime))
247
+ first = [item for item in found if item[0] == anchor]
248
+ return first + sorted((item for item in found if item[0] != anchor), key=lambda item: -item[2])
249
+
250
+
251
+ @contextlib.contextmanager
252
+ def read_lock(donor):
253
+ """The donor's dev build lock, held only if free now, through a read-only descriptor."""
254
+ import fcntl
255
+ path = donor / PROFILE_DIR / ".cargo-lock"
256
+ try:
257
+ # O_NONBLOCK: a FIFO in its place must not stall the open.
258
+ fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
259
+ except OSError as exc:
260
+ raise Skip(f"{path} cannot be opened: {exc.strerror}") from exc
261
+ try:
262
+ if not stat.S_ISREG(os.fstat(fd).st_mode):
263
+ raise Skip(f"{path} is not a regular file")
264
+ try:
265
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
266
+ except BlockingIOError as exc:
267
+ raise Skip(f"a build is using {donor}") from exc
268
+ yield fd, path
269
+ finally:
270
+ os.close(fd)
271
+
272
+
273
+ def same_file(fd, path):
274
+ held, current = os.fstat(fd), os.lstat(path)
275
+ return (held.st_dev, held.st_ino) == (current.st_dev, current.st_ino)
276
+
277
+
278
+ def layouts(target):
279
+ """The dev-profile layouts under a target directory, host first, relative."""
280
+ found = [Path(PROFILE_DIR)] if (target / PROFILE_DIR / ".fingerprint").is_dir() else []
281
+ for kind in sorted(p for p in target.iterdir() if p.is_dir() and not p.is_symlink() and p.name != PROFILE_DIR):
282
+ if (kind / PROFILE_DIR / ".fingerprint").is_dir() and not (kind / PROFILE_DIR).is_symlink():
283
+ found.append(Path(kind.name, PROFILE_DIR))
284
+ return found
285
+
286
+
287
+ def clone(source, target, deadline):
288
+ """Share source's blocks at target (never copy them), within the time limit."""
289
+ target.parent.mkdir(parents=True, exist_ok=True)
290
+ argv = ([sys.executable, "-c", CLONE_CHILD, str(source), str(target)] if sys.platform == "darwin"
291
+ else ["cp", "-a", "--reflink=always", "--", str(source), str(target)])
292
+ try:
293
+ result = subprocess.run(argv, capture_output=True, text=True, timeout=deadline.left())
294
+ except subprocess.TimeoutExpired as exc:
295
+ raise TimeUp(f"cloning {source} timed out") from exc
296
+ if result.returncode != 0:
297
+ raise Skip(f"this filesystem cannot clone {source}: {result.stderr.strip()[-200:]}")
298
+
299
+
300
+ def publish(staging_parent, staging_name, target_parent, target_name):
301
+ """Move staging into the checked parent only if nothing exists there."""
302
+ names = os.fsencode(staging_name), os.fsencode(target_name)
303
+ if sys.platform == "darwin":
304
+ rename = ctypes.CDLL(None, use_errno=True).renameatx_np
305
+ rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
306
+ failed = rename(staging_parent, names[0], target_parent, names[1], 0x4) != 0 # RENAME_EXCL
307
+ else:
308
+ rename = ctypes.CDLL(None, use_errno=True).renameat2
309
+ rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
310
+ failed = rename(staging_parent, names[0], target_parent, names[1], 1) != 0 # RENAME_NOREPLACE
311
+ if failed:
312
+ raise Skip(f"cannot publish {target_name}: {os.strerror(ctypes.get_errno())}")
313
+
314
+
315
+ def check_clone(staging, deadline):
316
+ """No Git data, no special file, and no link that leads out of the clone."""
317
+ pending = [staging]
318
+ while pending:
319
+ deadline.left()
320
+ with os.scandir(pending.pop()) as entries:
321
+ for item in entries:
322
+ if item.name.casefold() == ".git":
323
+ raise Unsafe(f"{item.path} is Git data")
324
+ if item.is_symlink():
325
+ if not Path(os.path.realpath(item.path)).is_relative_to(staging):
326
+ raise Unsafe(f"{item.path} links out of the clone")
327
+ elif item.is_dir(follow_symlinks=False):
328
+ pending.append(item.path)
329
+ elif not item.is_file(follow_symlinks=False):
330
+ raise Unsafe(f"{item.path} is not a regular file or directory")
331
+
332
+
333
+ def mentions(data, roots):
334
+ """Whether bytes name a path at or below any root (not a sibling such as `<root>.other`)."""
335
+ return any(re.search(re.escape(os.fsencode(root)) + rb"(?![A-Za-z0-9._-])", data) for root in roots)
336
+
337
+
338
+ def is_text(path):
339
+ """Whether a regular file (never a link or special file) holds no NUL in its first bytes."""
340
+ if not stat.S_ISREG(path.lstat().st_mode):
341
+ return False
342
+ with open(path, "rb") as stream:
343
+ return b"\0" not in stream.read(SNIFF_BYTES)
344
+
345
+
346
+ def foreign_runs(staging, layout, roots, readers, deadline):
347
+ """Hashes of build-script runs whose kept results name a donor root Cargo does not rewrite."""
348
+ found = set()
349
+ for run_dir in (staging / layout / "build").iterdir():
350
+ deadline.left()
351
+ root_output = run_dir / "root-output"
352
+ if not root_output.is_file():
353
+ continue # the compiled build script, not a run of it
354
+ old_out = os.fsdecode(root_output.read_bytes())
355
+ # <old target>/<layout>/build/<unit>/out
356
+ tail = (*layout.parts, "build", run_dir.name, "out")
357
+ if not Path(old_out).is_absolute() or Path(old_out).parts[-len(tail):] != tail:
358
+ raise Unsafe(f"{root_output} does not name this run's OUT_DIR")
359
+ needles = {*roots, str(Path(old_out).parents[len(tail) - 1])}
360
+ output = run_dir / "output"
361
+ lines = output.read_text(errors="replace").splitlines() if output.is_file() else []
362
+ # A variable the script hands to rustc is compiled in as it is, own OUT_DIR included.
363
+ exported = [line.split("=", 2)[2] for line in lines if RUSTC_ENV.match(line)]
364
+ foreign = mentions(os.fsencode("\n".join(exported)), {*needles, old_out}) or mentions(
365
+ os.fsencode("\n".join(lines).replace(old_out, "")), needles)
366
+ for directory, _, files in os.walk(run_dir / "out") if not foreign else ():
367
+ deadline.left()
368
+ for name in files:
369
+ path = Path(directory, name)
370
+ deadline.left()
371
+ if is_text(path):
372
+ data = path.read_bytes()
373
+ # Generated Rust is compiled only through an include, which reads OUT_DIR.
374
+ foreign = mentions(data, needles) or (
375
+ name.endswith(".rs") and UNIT.match(run_dir.name)["name"] in readers
376
+ and compiles_path(data.decode(errors="replace")))
377
+ if foreign:
378
+ break
379
+ if foreign:
380
+ break
381
+ if foreign:
382
+ found.add(UNIT.match(run_dir.name)["hash"])
383
+ return found
384
+
385
+
386
+ def decoded(literal):
387
+ def one(match):
388
+ byte, code, other = match.groups()
389
+ if byte or code:
390
+ return chr(int((byte or code).replace("_", ""), 16))
391
+ return "" if other is None else other # a continuation, or an escaped character
392
+ return ESCAPE.sub(one, literal)
393
+
394
+
395
+ def at_word(text, start, prefix=""):
396
+ """Whether text[start:] begins a word (or one led by `prefix`, as option_ leads env)."""
397
+ if prefix and text.endswith(prefix, 0, start):
398
+ start -= len(prefix)
399
+ return start == 0 or not (text[start - 1].isalnum() or text[start - 1] == "_")
400
+
401
+
402
+ def compiles_path(text):
403
+ """Whether Rust source may compile its OUT_DIR in other than through an include macro."""
404
+ # Without OUT_DIR spelled out or a backslash, no literal can decode to it.
405
+ literals = (sum(decoded(match) == "OUT_DIR" for match in STRING_CANDIDATE.findall(text))
406
+ if "OUT_DIR" in text or "\\" in text else 0)
407
+ included = sum(at_word(text, m.start()) for m in INCLUDED_OUT_DIR.finditer(text)) if "OUT_DIR" in text else 0
408
+ return literals > included or any(at_word(text, m.start(), "option_") for m in NONLITERAL_ENV.finditer(text))
409
+
410
+
411
+ def compiles_out_dir(package, deadline):
412
+ """Whether the package's library sources compile its OUT_DIR in."""
413
+ root = Path(package["manifest_path"]).parent
414
+ for directory, dirs, files in os.walk(root):
415
+ deadline.left()
416
+ dirs[:] = [d for d in dirs if d not in {"target", "tests", "benches", "examples", ".git"}]
417
+ for name in files:
418
+ if name.endswith(".rs") and not (Path(directory) == root and name == "build.rs"):
419
+ if compiles_path(Path(directory, name).read_text(encoding="utf-8", errors="replace")):
420
+ return True
421
+ return False
422
+
423
+
424
+ def remove(item, deadline):
425
+ """Delete a file or a whole directory without following links, within the time limit."""
426
+ deadline.left()
427
+ if not item.is_dir() or item.is_symlink():
428
+ item.unlink()
429
+ return
430
+ for directory, dirs, files in os.walk(item, topdown=False):
431
+ for name in files:
432
+ deadline.left()
433
+ os.unlink(os.path.join(directory, name))
434
+ for name in dirs:
435
+ deadline.left()
436
+ path = os.path.join(directory, name)
437
+ os.unlink(path) if os.path.islink(path) else os.rmdir(path)
438
+ deadline.left()
439
+ os.rmdir(item)
440
+
441
+
442
+ def remove_units(staging, layout, hashes, deadline):
443
+ for sub in (".fingerprint", "build", "deps", "examples"):
444
+ directory = staging / layout / sub
445
+ for item in directory.iterdir() if directory.is_dir() else ():
446
+ match = UNIT.match(item.name)
447
+ if match and match["hash"] in hashes:
448
+ remove(item, deadline)
449
+
450
+
451
+ def remove_unclassified(staging, layout, deadline):
452
+ """Final artifacts Cargo places beside the unit directories carry no unit to classify them by."""
453
+ for item in (staging / layout).iterdir():
454
+ if item.name not in CARGO_ENTRIES:
455
+ remove(item, deadline)
456
+ for sub in ("deps", "examples"):
457
+ directory = staging / layout / sub
458
+ for item in directory.iterdir() if directory.is_dir() else ():
459
+ if not UNIT.match(item.name):
460
+ remove(item, deadline)
461
+
462
+
463
+ def crates_reading_out_dir(staging, found, crates, deadline):
464
+ """The crates among these that may have read OUT_DIR while compiling, by rustc's dep-info.
465
+
466
+ rustc writes `# env-dep:OUT_DIR=...` into a unit's deps/<crate>-<hash>.d for every
467
+ env!/option_env! read of it, whatever the spelling or macro that made it. A crate counts as
468
+ not reading it only when every kept output of it has that file and none says so. Any
469
+ hashed output counts, with or without the `lib` prefix: rlibs, rmeta, dylibs, Windows DLLs
470
+ and static libraries, test executables.
471
+ """
472
+ reading, recorded, outputs = set(), {}, {}
473
+ for layout in found:
474
+ for item in (staging / layout / "deps").iterdir():
475
+ deadline.left()
476
+ if not (match := UNIT.match(item.name)):
477
+ continue
478
+ if item.name.endswith(".d"):
479
+ if match["name"] in crates:
480
+ recorded.setdefault(match["name"], set()).add(match["hash"])
481
+ if b"\n# env-dep:OUT_DIR=" in b"\n" + item.read_bytes():
482
+ reading.add(match["name"])
483
+ continue
484
+ stem = match["name"]
485
+ for crate in {stem, stem.removeprefix("lib")} & crates:
486
+ outputs.setdefault(crate, set()).add(match["hash"])
487
+ return reading | {crate for crate, hashes in outputs.items() if hashes - recorded.get(crate, set())}
488
+
489
+
490
+ def sanitize(work, staging, found, roots, deadline):
491
+ for layout in found:
492
+ remove_unclassified(staging, layout, deadline)
493
+ resolved = metadata(work, deadline, env=without_redirects(), deps=True)
494
+ names = {package["name"] for package in resolved["packages"]}
495
+ path_packages = {package["name"] for package in resolved["packages"] if package["source"] is None}
496
+ # Every version of a package counts on its own; a name is removed if any version must be.
497
+ scripted = [package for package in resolved["packages"]
498
+ if package["source"] is not None and any("custom-build" in t["kind"] for t in package["targets"])]
499
+ crates = [{t["name"].replace("-", "_") for t in package["targets"] if "custom-build" not in t["kind"]}
500
+ for package in scripted]
501
+ reading = crates_reading_out_dir(staging, found, set().union(*crates), deadline)
502
+ reader_packages = [package for package, own in zip(scripted, crates) if own & reading]
503
+ readers = {package["name"] for package in reader_packages}
504
+ out_dir_packages = {package["name"] for package in reader_packages if compiles_out_dir(package, deadline)}
505
+ removed = set()
506
+ for layout in found:
507
+ units = {m["hash"]: m["name"] for item in (staging / layout / ".fingerprint").iterdir()
508
+ if (m := UNIT.match(item.name))}
509
+ drop = {digest for digest, name in units.items()
510
+ if name not in names or name in path_packages or name in out_dir_packages}
511
+ drop |= foreign_runs(staging, layout, roots, readers, deadline)
512
+ removed |= {units[digest] for digest in drop if digest in units}
513
+ # Outputs without a fingerprint belong to no unit Cargo would reuse.
514
+ drop |= {m["hash"] for sub in ("build", "deps", "examples") if (staging / layout / sub).is_dir()
515
+ for item in (staging / layout / sub).iterdir() if (m := UNIT.match(item.name)) and m["hash"] not in units}
516
+ remove_units(staging, layout, drop, deadline)
517
+ return sorted(removed)
518
+
519
+
520
+ def warm_cargo(work, anchor, staging_root, deadline):
521
+ if not (work / "Cargo.toml").is_file():
522
+ return None
523
+ if sys.platform not in {"darwin", "linux"}:
524
+ return entry("cargo", "skipped", reason=f"cloning is unsupported on {sys.platform}")
525
+ started = time.monotonic()
526
+ with contextlib.ExitStack() as stack:
527
+ staging = None
528
+ try:
529
+ if shutil.which("cargo") is None:
530
+ raise Skip("cargo is not on PATH")
531
+ target, relative, parent = destination(work, deadline, stack)
532
+ candidates = donors(work, anchor, relative, deadline)
533
+ if not candidates:
534
+ raise Skip(f"no built {relative} in this repository's checkouts")
535
+ if os.stat(staging_root).st_dev != os.fstat(parent.fd).st_dev:
536
+ raise Skip("the staging and target directories are on different volumes")
537
+ root = stack.enter_context(contextlib.closing(_Fd(os.open(staging_root, os.O_RDONLY | os.O_DIRECTORY))))
538
+ busy = []
539
+ for checkout, donor, _ in candidates:
540
+ if os.stat(donor).st_dev != os.stat(staging_root).st_dev:
541
+ busy.append(f"{donor} is on another volume")
542
+ continue
543
+ name = f".warm-cargo-{uuid.uuid4().hex}"
544
+ try:
545
+ with read_lock(donor) as (fd, lock):
546
+ identity = os.stat(donor / PROFILE_DIR)
547
+ found = layouts(donor)
548
+ os.mkdir(name, dir_fd=root.fd)
549
+ staging = staging_root / name
550
+ for layout in found:
551
+ clone(donor / layout, staging / layout, deadline)
552
+ after = os.stat(donor / PROFILE_DIR)
553
+ if (identity.st_dev, identity.st_ino) != (after.st_dev, after.st_ino) or not same_file(fd, lock):
554
+ raise Unsafe(f"{donor} was replaced while it was cloned")
555
+ except TimeUp:
556
+ raise
557
+ except Skip as exc:
558
+ busy.append(str(exc))
559
+ if staging is not None:
560
+ remove(staging, deadline)
561
+ staging = None
562
+ continue
563
+ check_clone(staging, deadline)
564
+ if (donor / "CACHEDIR.TAG").is_file():
565
+ (staging / "CACHEDIR.TAG").write_bytes((donor / "CACHEDIR.TAG").read_bytes())
566
+ removed = sanitize(work, staging, found, [str(donor), str(checkout)], deadline)
567
+ deadline.left()
568
+ try:
569
+ reached = open_beneath(work, relative.parts[:-1])
570
+ except OSError as exc:
571
+ raise Unsafe(f"{relative.parent} changed while warming: {exc.strerror}") from exc
572
+ try:
573
+ held, now = os.fstat(parent.fd), os.fstat(reached)
574
+ if (held.st_dev, held.st_ino) != (now.st_dev, now.st_ino):
575
+ raise Unsafe(f"{relative.parent} changed while warming")
576
+ finally:
577
+ os.close(reached)
578
+ publish(root.fd, name, parent.fd, relative.name)
579
+ staging = None
580
+ return entry("cargo", "seeded", path=str(target), donor=str(donor),
581
+ removed=removed, seconds=round(time.monotonic() - started, 1))
582
+ raise Skip("; ".join(busy))
583
+ except Skip as exc:
584
+ result = entry("cargo", "skipped", reason=str(exc))
585
+ except Unsafe as exc:
586
+ result = entry("cargo", "failed", reason=str(exc))
587
+ except FileNotFoundError as exc:
588
+ result = entry("cargo", "skipped", reason=f"{exc.filename} disappeared while warming")
589
+ except Exception as exc: # Reported, never raised: allocation goes on cold.
590
+ result = entry("cargo", "failed", reason=f"{type(exc).__name__}: {exc}")
591
+ if staging is not None and os.path.lexists(staging):
592
+ try:
593
+ remove(staging, deadline)
594
+ except TimeUp:
595
+ # Removing a large partial clone would outlast the limit; scratch cleanup owns it.
596
+ result["residue"] = str(staging)
597
+ return result
598
+
599
+
600
+ def warm(work, anchor, staging_root, *, budget=BUDGET_SECONDS):
601
+ """Warm every eligible build directory of a new worktree; never raises for a tool's failure."""
602
+ work, anchor, staging_root = Path(work).resolve(), Path(anchor).resolve(), Path(staging_root).resolve()
603
+ results = []
604
+ for tool, warmer in (("cargo", warm_cargo),):
605
+ try:
606
+ result = warmer(work, anchor, staging_root, Deadline(budget))
607
+ except Exception as exc: # Optional warming reports any failure; allocation goes on cold.
608
+ result = entry(tool, "failed", reason=f"{type(exc).__name__}: {exc}")
609
+ if result is not None:
610
+ results.append(result)
611
+ print(f"warm-build-dirs: {tool} {result['status']}: {result.get('path') or result.get('reason')}", file=sys.stderr)
612
+ return results
613
+
614
+
615
+ def main():
616
+ parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
617
+ parser.add_argument("worktree", nargs="?", type=Path)
618
+ parser.add_argument("--from", dest="anchor", type=Path, help="the checkout to warm from first (default: the main worktree)")
619
+ parser.add_argument("--self-test", action="store_true")
620
+ args = parser.parse_args()
621
+ if args.self_test:
622
+ return self_test()
623
+ if args.worktree is None:
624
+ parser.error("a worktree is required")
625
+ work = args.worktree.resolve()
626
+ anchor = args.anchor.resolve() if args.anchor else None
627
+ if anchor is None:
628
+ listing = subprocess.run(["git", "-C", str(work), "worktree", "list", "--porcelain"],
629
+ capture_output=True, text=True, check=True).stdout
630
+ anchor = Path(listing.splitlines()[0].removeprefix("worktree ")).resolve()
631
+ common = subprocess.run(["git", "-C", str(work), "rev-parse", "--path-format=absolute", "--git-common-dir"],
632
+ capture_output=True, text=True, check=True).stdout.strip()
633
+ staging_root = Path(tempfile.mkdtemp(prefix=".warm-build-dirs-", dir=common))
634
+ results = warm(work, anchor, staging_root)
635
+ if not any(result.get("residue") for result in results):
636
+ staging_root.rmdir()
637
+ print(json.dumps(results, sort_keys=True))
638
+ return 0
639
+
640
+
641
+ def self_test():
642
+ result = unittest.TextTestRunner(verbosity=1).run(unittest.defaultTestLoader.loadTestsFromTestCase(WarmTests))
643
+ if result.wasSuccessful():
644
+ print(f"warm-build-dirs self-test: PASS ({result.testsRun} isolated tests)")
645
+ return 0 if result.wasSuccessful() else 1
646
+
647
+
648
+ # Fixtures use real Git and a fake `cargo` on PATH that answers `metadata` from a JSON file;
649
+ # a filesystem without cloning copies instead, in tests only, so every rule still runs there.
650
+ FAKE_CARGO = r"""#!/usr/bin/env python3
651
+ import json, os, pathlib, sys, time
652
+ argv = sys.argv[1:]
653
+ fixture = json.loads(pathlib.Path(os.environ["FAKE_CARGO"]).read_text())
654
+ with open(os.environ["FAKE_CARGO_LOG"], "a") as log:
655
+ log.write(json.dumps(argv) + "\n")
656
+ time.sleep(fixture.get("sleep", 0))
657
+ target = pathlib.Path(os.environ.get("CARGO_TARGET_DIR") or os.getcwd() + "/" + fixture.get("target", "target"))
658
+ assert argv[0] == "metadata", argv # warming never runs another cargo command
659
+ found = {"target_directory": str(target), "packages": fixture["packages"] if "--no-deps" not in argv else [],
660
+ "metadata": fixture.get("workspace_metadata", {"devlyn": {"warm-build-dirs": True}})}
661
+ if not fixture.get("old_cargo"):
662
+ found["build_directory"] = fixture.get("build_directory", str(target))
663
+ print(json.dumps(found))
664
+ # The full resolve runs while warming, after the clone: the moment to race the destination.
665
+ if "--no-deps" not in argv:
666
+ if fixture.get("swap") and not os.path.islink("build"):
667
+ os.rename("build", "build-moved")
668
+ os.symlink(fixture["swap"], "build")
669
+ if fixture.get("race"):
670
+ (pathlib.Path(os.getcwd()) / "target").mkdir(exist_ok=True)
671
+ """
672
+
673
+
674
+ class WarmTests(unittest.TestCase):
675
+ def setUp(self):
676
+ # Imported here: unittest.mock imports asyncio, which a caller simulating Windows cannot load.
677
+ from unittest import mock
678
+ temporary = tempfile.TemporaryDirectory(prefix="warm-build-dirs-")
679
+ self.addCleanup(temporary.cleanup)
680
+ self.root = Path(temporary.name).resolve()
681
+ self.anchor, self.work = self.root / "anchor", self.root / "work"
682
+ self.staging = self.root / "scratch"
683
+ self.staging.mkdir()
684
+ bin_dir = self.root / "bin"
685
+ bin_dir.mkdir()
686
+ (bin_dir / "cargo").write_text(FAKE_CARGO)
687
+ (bin_dir / "cargo").chmod(0o755)
688
+ self.fixture, self.log = self.root / "fixture.json", self.root / "cargo.log"
689
+ self.environment = mock.patch.dict(os.environ, {
690
+ "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", "FAKE_CARGO": str(self.fixture),
691
+ "FAKE_CARGO_LOG": str(self.log), "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1",
692
+ "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"})
693
+ self.environment.start()
694
+ self.addCleanup(self.environment.stop)
695
+ for key in REDIRECTS:
696
+ os.environ.pop(key, None)
697
+ self.anchor.mkdir()
698
+ self.git(self.anchor, "init", "-q", "-b", "main")
699
+ (self.anchor / "Cargo.toml").write_text("[workspace]\n")
700
+ (self.anchor / ".gitignore").write_text("target/\n/build/target/\n")
701
+ self.git(self.anchor, "add", "-A")
702
+ self.git(self.anchor, "commit", "-q", "-m", "init")
703
+ self.git(self.anchor, "worktree", "add", "-q", "-b", "task", str(self.work))
704
+ self.write_fixture()
705
+ self.donor = self.anchor / "target"
706
+ self.build_donor()
707
+ probe = self.root / "probe"
708
+ probe.mkdir()
709
+ (probe / "file").write_bytes(b"x" * 65536)
710
+ self.native_clone = True
711
+ try:
712
+ if os.name == "nt":
713
+ raise Skip("no clone primitive")
714
+ clone(probe, self.root / "probe-clone", Deadline(30))
715
+ self.native_clone = (self.root / "probe-clone" / "file").read_bytes() == b"x" * 65536
716
+ except (Skip, OSError):
717
+ self.native_clone = False
718
+ if not self.native_clone:
719
+ # A plain copy that keeps links and special files, as a clone would; tests only.
720
+ def copy(source, target, deadline):
721
+ target.parent.mkdir(parents=True, exist_ok=True)
722
+ subprocess.run(["cp", "-a", "--", str(source), str(target)], check=True)
723
+ patch = mock.patch(f"{__name__}.clone", copy)
724
+ patch.start()
725
+ self.addCleanup(patch.stop)
726
+
727
+ def git(self, work, *args):
728
+ return subprocess.run(["git", "-C", str(work), *args], check=True, capture_output=True, text=True).stdout
729
+
730
+ def write_fixture(self, **extra):
731
+ packages = [
732
+ {"name": "app", "source": None, "manifest_path": str(self.work / "Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}]},
733
+ {"name": "serde", "source": "registry", "manifest_path": str(self.root / "registry/serde/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}]},
734
+ {"name": "ring", "source": "registry", "manifest_path": str(self.root / "registry/ring/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
735
+ {"name": "tauri", "source": "registry", "manifest_path": str(self.root / "registry/tauri/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
736
+ {"name": "aws-lc-rs", "source": "registry", "manifest_path": str(self.root / "registry/aws-lc-rs/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
737
+ {"name": "baked", "source": "registry", "manifest_path": str(self.root / "registry/baked/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
738
+ {"name": "exporter", "source": "registry", "manifest_path": str(self.root / "registry/exporter/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
739
+ {"name": "generator", "source": "registry", "manifest_path": str(self.root / "registry/generator/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
740
+ {"name": "windowsdll", "source": "registry", "manifest_path": str(self.root / "registry/windowsdll/Cargo.toml"), "targets": [{"kind": ["dylib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
741
+ # rustc left no dep-info for its library, so nothing proves it never read OUT_DIR.
742
+ {"name": "undocumented", "source": "registry", "manifest_path": str(self.root / "registry/undocumented/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
743
+ # Two versions of one package: only 1.0 compiles OUT_DIR in.
744
+ {"name": "dual", "source": "registry", "manifest_path": str(self.root / "registry/dual-1/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
745
+ {"name": "dual", "source": "registry", "manifest_path": str(self.root / "registry/dual-2/Cargo.toml"), "targets": [{"kind": ["lib"], "name": "LIB"}, {"kind": ["custom-build"], "name": "build-script-build"}]},
746
+ ]
747
+ for package in packages:
748
+ package["targets"] = [{**t, "name": package["name"] if t["name"] == "LIB" else t["name"]} for t in package["targets"]]
749
+ for package in packages[1:]:
750
+ source = Path(package["manifest_path"]).parent / "src"
751
+ source.mkdir(parents=True, exist_ok=True)
752
+ compiled = Path(package["manifest_path"]).parent.name in {"baked", "undocumented", "dual-1", "windowsdll"}
753
+ (source / "lib.rs").write_text('pub const DIR: &str = env!(/* why */ "OUT_DIR");\n' if compiled
754
+ else 'include!(concat!(\n env!("OUT_DIR"), "/gen.rs"));\n')
755
+ self.fixture.write_text(json.dumps({"packages": packages, **extra}))
756
+
757
+ def unit(self, layout, sub, name, digest, files=None):
758
+ directory = self.donor / layout / sub / f"{name}-{digest}"
759
+ directory.mkdir(parents=True, exist_ok=True)
760
+ for relative, data in (files or {"stamp": b"x"}).items():
761
+ (directory / relative).parent.mkdir(parents=True, exist_ok=True)
762
+ (directory / relative).write_bytes(data)
763
+
764
+ def run_unit(self, layout, name, digest, output, out_files):
765
+ old_out = f"{self.donor}/{layout}/build/{name}-{digest}/out"
766
+ self.unit(layout, ".fingerprint", name, digest)
767
+ self.unit(layout, "build", name, digest, {"root-output": old_out.encode(), "output": output(old_out).encode(),
768
+ **{f"out/{k}": v(old_out) for k, v in out_files.items()}})
769
+
770
+ def build_donor(self):
771
+ for layout in ("debug", "x86_64-pc-windows-msvc/debug"):
772
+ deps = self.donor / layout / "deps"
773
+ deps.mkdir(parents=True)
774
+ for name, digest in (("serde", "1" * 16), ("app", "2" * 16), ("gone", "7" * 16), ("baked", "8" * 16)):
775
+ self.unit(layout, ".fingerprint", name, digest)
776
+ (deps / f"lib{name}-{digest}.rlib").write_bytes(b"rlib")
777
+ (self.donor / layout / ".cargo-lock").write_bytes(b"")
778
+ # rustc's dep-info: which crates read OUT_DIR while compiling.
779
+ for crate, digest, reads in (("serde", "1" * 16, False), ("baked", "8" * 16, True), ("generator", "b" * 16, True),
780
+ ("ring", "c" * 16, False), ("dual", "1a" * 8, True), ("dual", "2b" * 8, True)):
781
+ record = f"# env-dep:OUT_DIR={self.donor}/{layout}/build/{crate}-x/out\n" if reads else ""
782
+ (deps / f"{crate}-{digest}.d").write_text(f"/src/{crate}/lib.rs:\n\n# env-dep:CARGO_PKG_NAME={crate}\n{record}")
783
+ self.unit(layout, ".fingerprint", crate, digest)
784
+ (deps / "libstray-dddddddddddddddd.rlib").write_bytes(b"rlib")
785
+ self.unit(layout, ".fingerprint", "undocumented", "e" * 16)
786
+ (deps / f"libundocumented-{'e' * 16}.rlib").write_bytes(b"rlib")
787
+ # A Windows-style DLL (no `lib` prefix) without dep-info, of a crate that compiles OUT_DIR in.
788
+ self.unit(layout, ".fingerprint", "windowsdll", "9a" * 8)
789
+ (deps / f"windowsdll-{'9a' * 8}.dll").write_bytes(b"dll")
790
+ for digest in ("1a" * 8, "2b" * 8):
791
+ (deps / f"libdual-{digest}.rlib").write_bytes(b"rlib")
792
+ # A run that points into the donor and has no fingerprint left.
793
+ self.unit(layout, "build", "orphanrun", "f" * 16, {
794
+ "root-output": f"{self.donor}/{layout}/build/orphanrun-{'f' * 16}/out".encode(),
795
+ "output": f"cargo:include={self.donor}/{layout}/build/aws-lc-sys-{'6' * 16}/out\n".encode()})
796
+ (self.donor / layout / "incremental" / "app-x").mkdir(parents=True)
797
+ # Final artifacts Cargo puts beside the unit directories, here of the unused `gone`.
798
+ (self.donor / layout / "old-tool").write_bytes(b"bin")
799
+ (self.donor / layout / "examples").mkdir()
800
+ (self.donor / layout / "examples" / "old-example").write_bytes(b"bin")
801
+ self.run_unit(layout, "ring", "4" * 16, lambda out: f"cargo:rustc-link-search=native={out}\n",
802
+ {"libring.a": lambda out: b"\0\x01" + out.encode()})
803
+ self.run_unit(layout, "tauri", "3" * 16, lambda out: f"cargo:rustc-link-search=native={out}\n",
804
+ {"permission-files": lambda out: f'["{out}/app_hide.toml"]'.encode()})
805
+ self.run_unit(layout, "aws-lc-rs", "5" * 16,
806
+ lambda out: f"cargo:include={self.donor}/{layout}/build/aws-lc-sys-{'6' * 16}/out/include\n", {})
807
+ # A variable handed to rustc that carries the script's own OUT_DIR is compiled in as is.
808
+ self.run_unit(layout, "exporter", "9" * 16, lambda out: f"cargo:rustc-env=DEBUG_OUTPUT_DIR={out}\n", {})
809
+ # Generated Rust that compiles its OUT_DIR in names no donor path in its text.
810
+ self.run_unit(layout, "generator", "a" * 16, lambda out: f"cargo:rustc-link-search=native={out}\n",
811
+ {"gen.rs": lambda out: b'pub const DIR: &str = env!("OUT\\x5fDIR");\n'})
812
+ (self.donor / "CACHEDIR.TAG").write_bytes(b"Signature: 8a477f597d28d172789f06886806bc55\n")
813
+
814
+ def snapshot(self, root):
815
+ return {str(p.relative_to(root)): (p.lstat().st_size, p.lstat().st_mtime_ns, p.lstat().st_mode)
816
+ for p in sorted(root.rglob("*"))}
817
+
818
+ def warm(self, **kwargs):
819
+ return warm(self.work, self.anchor, self.staging, **kwargs)
820
+
821
+ def assert_nothing_left(self):
822
+ self.assertEqual([], list(self.staging.iterdir()))
823
+ self.assertEqual("", self.git(self.work, "status", "--porcelain", "--untracked-files=all"))
824
+
825
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
826
+ def test_local_git_true_seeds_without_cargo_opt_in(self):
827
+ self.write_fixture(workspace_metadata=None)
828
+ self.git(self.anchor, "config", "--local", "devlyn.warmBuildDirs", "true")
829
+ [result] = self.warm()
830
+ self.assertEqual("seeded", result["status"], result)
831
+ self.assertTrue((self.work / "target" / "debug" / "deps" / f"libserde-{'1' * 16}.rlib").is_file())
832
+ self.assert_nothing_left()
833
+
834
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
835
+ def test_local_git_false_overrides_cargo_opt_in(self):
836
+ self.git(self.anchor, "config", "--local", "devlyn.warmBuildDirs", "false")
837
+ [result] = self.warm()
838
+ self.assertEqual("skipped", result["status"], result)
839
+ self.assertEqual("turned off: git config --local devlyn.warmBuildDirs is false", result["reason"])
840
+ self.assertFalse((self.work / "target").exists())
841
+ self.assert_nothing_left()
842
+
843
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
844
+ def test_invalid_local_git_value_skips_with_git_error(self):
845
+ self.git(self.anchor, "config", "--local", "devlyn.warmBuildDirs", "invalid")
846
+ [result] = self.warm()
847
+ self.assertEqual("skipped", result["status"], result)
848
+ self.assertIn("git config --local devlyn.warmBuildDirs", result["reason"])
849
+ self.assertIn("fatal: bad boolean config value 'invalid' for 'devlyn.warmbuilddirs'", result["reason"])
850
+ self.assertFalse((self.work / "target").exists())
851
+ self.assert_nothing_left()
852
+
853
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
854
+ def test_unset_local_git_value_uses_cargo_opt_in(self):
855
+ for opted_in in (False, True):
856
+ with self.subTest(opted_in=opted_in):
857
+ self.write_fixture(workspace_metadata={"devlyn": {"warm-build-dirs": opted_in}})
858
+ [result] = self.warm()
859
+ self.assertEqual("seeded" if opted_in else "skipped", result["status"], result)
860
+ if not opted_in:
861
+ self.assertIn("not enabled", result["reason"])
862
+ self.assert_nothing_left()
863
+
864
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
865
+ def test_other_git_config_failures_skip_with_first_error_line(self):
866
+ config = self.anchor / ".git" / "config"
867
+ config.write_text(config.read_text() + "\n[broken\nsecond bad line\n")
868
+ error = subprocess.run(["git", "-C", str(self.work), "config", "--local", "--type=bool",
869
+ "--get", "devlyn.warmBuildDirs"], capture_output=True, text=True)
870
+ self.assertNotIn(error.returncode, (0, 1))
871
+ [result] = self.warm()
872
+ self.assertEqual("skipped", result["status"], result)
873
+ self.assertEqual(f"cannot read git config --local devlyn.warmBuildDirs: {error.stderr.splitlines()[0]}",
874
+ result["reason"])
875
+ self.assertFalse((self.work / "target").exists())
876
+ self.assertEqual([], list(self.staging.iterdir()))
877
+
878
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
879
+ def test_global_include_and_per_worktree_config_do_not_override_cargo(self):
880
+ global_config, included = self.root / "global-config", self.root / "included-config"
881
+ os.environ["GIT_CONFIG_GLOBAL"] = str(global_config)
882
+ self.git(self.anchor, "config", "--local", "include.path", str(included))
883
+ self.git(self.anchor, "config", "--local", "extensions.worktreeConfig", "true")
884
+ for opted_in in (False, True):
885
+ with self.subTest(opted_in=opted_in):
886
+ value = "false" if opted_in else "true"
887
+ for config in (global_config, included):
888
+ config.write_text(f"[devlyn]\n\twarmBuildDirs = {value}\n")
889
+ self.git(self.work, "config", "--worktree", "devlyn.warmBuildDirs", value)
890
+ self.write_fixture(workspace_metadata={"devlyn": {"warm-build-dirs": opted_in}})
891
+ [result] = self.warm()
892
+ self.assertEqual("seeded" if opted_in else "skipped", result["status"], result)
893
+ if not opted_in:
894
+ self.assertIn("not enabled", result["reason"])
895
+ self.assert_nothing_left()
896
+
897
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
898
+ def test_seeds_only_what_carries_over_and_never_writes_the_donor(self):
899
+ before = self.snapshot(self.donor)
900
+ [result] = self.warm()
901
+ self.assertEqual("seeded", result["status"], result)
902
+ target = self.work / "target"
903
+ for layout in ("debug", "x86_64-pc-windows-msvc/debug"):
904
+ kept = {p.name for p in (target / layout / ".fingerprint").iterdir()}
905
+ # generator keeps its library unit: Cargo rebuilds it after rerunning the removed run.
906
+ self.assertEqual({f"serde-{'1' * 16}", f"ring-{'4' * 16}", f"ring-{'c' * 16}", f"generator-{'b' * 16}"}, kept, layout)
907
+ self.assertFalse((target / layout / "deps" / "libstray-dddddddddddddddd.rlib").exists())
908
+ self.assertEqual({".fingerprint", "build", "deps", "examples", ".cargo-lock"}, {p.name for p in (target / layout).iterdir()})
909
+ self.assertEqual([], list((target / layout / "examples").iterdir()))
910
+ self.assertTrue((target / layout / "build" / f"ring-{'4' * 16}" / "out" / "libring.a").exists())
911
+ self.assertEqual(["app", "aws-lc-rs", "baked", "dual", "exporter", "generator", "gone", "tauri", "undocumented",
912
+ "windowsdll"],
913
+ result["removed"])
914
+ for layout in ("debug", "x86_64-pc-windows-msvc/debug"):
915
+ self.assertFalse((target / layout / "build" / f"orphanrun-{'f' * 16}").exists())
916
+ self.assertFalse(any((target / layout / "deps").glob("libdual-*")))
917
+ self.assertEqual(str(self.donor), result["donor"])
918
+ self.assertEqual(before, self.snapshot(self.donor))
919
+ self.assert_nothing_left()
920
+
921
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
922
+ def test_a_busy_donor_is_skipped_without_waiting(self):
923
+ import fcntl
924
+ with open(self.donor / "debug" / ".cargo-lock", "rb") as lock:
925
+ fcntl.flock(lock, fcntl.LOCK_EX)
926
+ [result] = self.warm()
927
+ self.assertEqual("skipped", result["status"])
928
+ self.assertIn("a build is using", result["reason"])
929
+ self.assertFalse((self.work / "target").exists())
930
+ self.assert_nothing_left()
931
+
932
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
933
+ def test_ineligible_destinations_are_skipped_with_their_reason(self):
934
+ cases = {
935
+ "already exists": lambda: (self.work / "target").mkdir(),
936
+ "not ignored": lambda: (self.work / ".gitignore").write_text(""),
937
+ "not enabled": lambda: self.write_fixture(workspace_metadata=None),
938
+ "separate Cargo build-dir": lambda: self.write_fixture(build_directory=str(self.root / "elsewhere")),
939
+ "build directory (Cargo 1.91": lambda: self.write_fixture(old_cargo=True),
940
+ "comes from this environment": lambda: os.environ.__setitem__("CARGO_TARGET_DIR", str(self.root / "t")),
941
+ "no built target": lambda: shutil.rmtree(self.donor),
942
+ }
943
+ for reason, arrange in cases.items():
944
+ with self.subTest(reason):
945
+ self.setUp()
946
+ arrange()
947
+ [result] = self.warm()
948
+ self.assertEqual("skipped", result["status"], result)
949
+ self.assertIn(reason, result["reason"])
950
+ self.assertEqual([], list(self.staging.iterdir()))
951
+
952
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
953
+ def test_a_clone_with_git_data_or_a_link_out_is_refused(self):
954
+ for name, arrange in {
955
+ "links out": lambda: (self.donor / "debug" / "build" / "link").symlink_to(self.root),
956
+ "Git data": lambda: (self.donor / "debug" / "deps" / ".git").mkdir(),
957
+ }.items():
958
+ with self.subTest(name):
959
+ self.setUp()
960
+ arrange()
961
+ before = self.snapshot(self.donor)
962
+ [result] = self.warm()
963
+ self.assertEqual("failed", result["status"], result)
964
+ self.assertIn(name, result["reason"])
965
+ self.assertFalse(os.path.lexists(self.work / "target"))
966
+ self.assertEqual(before, self.snapshot(self.donor))
967
+ self.assertEqual([], list(self.staging.iterdir()))
968
+
969
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
970
+ def test_running_out_of_time_publishes_nothing(self):
971
+ self.write_fixture(sleep=2)
972
+ [result] = self.warm(budget=1)
973
+ self.assertEqual("skipped", result["status"])
974
+ self.assertIn("timed out", result["reason"])
975
+ self.assertFalse((self.work / "target").exists())
976
+ self.assertEqual([], list(self.staging.iterdir()))
977
+
978
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
979
+ def test_a_target_that_appears_meanwhile_is_never_replaced(self):
980
+ self.write_fixture(race=True)
981
+ [result] = self.warm()
982
+ self.assertEqual("skipped", result["status"])
983
+ self.assertIn("cannot publish", result["reason"])
984
+ self.assertEqual([], list((self.work / "target").iterdir()))
985
+ self.assertEqual([], list(self.staging.iterdir()))
986
+
987
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
988
+ def test_a_parent_replaced_while_warming_receives_nothing(self):
989
+ (self.anchor / "build").mkdir()
990
+ self.donor = self.anchor / "build" / "target"
991
+ (self.anchor / "target").rename(self.donor)
992
+ (self.work / "build").mkdir()
993
+ elsewhere = self.root / "elsewhere"
994
+ elsewhere.mkdir()
995
+ self.write_fixture(target="build/target", swap=str(elsewhere))
996
+ [result] = self.warm()
997
+ self.assertEqual("failed", result["status"], result)
998
+ self.assertIn("changed while warming", result["reason"])
999
+ self.assertEqual([], list(elsewhere.iterdir()))
1000
+ self.assertFalse((self.work / "build-moved" / "target").exists())
1001
+ self.assertEqual([], list(self.staging.iterdir()))
1002
+
1003
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
1004
+ def test_a_special_file_is_never_read(self):
1005
+ os.mkfifo(self.donor / "debug" / "build" / f"tauri-{'3' * 16}" / "out" / "pipe")
1006
+ [result] = self.warm(budget=20)
1007
+ self.assertEqual("failed", result["status"], result)
1008
+ self.assertIn("not a regular file or directory", result["reason"])
1009
+ self.assertFalse((self.work / "target").exists())
1010
+ self.assertEqual([], list(self.staging.iterdir()))
1011
+
1012
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
1013
+ def test_a_malformed_build_record_fails_explicitly(self):
1014
+ (self.donor / "debug" / "build" / f"tauri-{'3' * 16}" / "root-output").write_bytes(b"")
1015
+ [result] = self.warm()
1016
+ self.assertEqual("failed", result["status"], result)
1017
+ self.assertIn("does not name this run's OUT_DIR", result["reason"])
1018
+ self.assertFalse((self.work / "target").exists())
1019
+ self.assertEqual([], list(self.staging.iterdir()))
1020
+
1021
+ @unittest.skipIf(os.name == "nt", "warming is POSIX-only")
1022
+ def test_allocation_warms_its_worktree_and_records_the_outcome(self):
1023
+ tree = self.root / "allocated"
1024
+ head = self.git(self.anchor, "rev-parse", "HEAD").strip()
1025
+ completed = subprocess.run([sys.executable, str(Path(__file__).with_name("task-complete.py")), "allocate",
1026
+ "--repo", str(self.anchor), "--task", "warm", "--branch", "warm-allocation",
1027
+ "--worktree", str(tree), "--local-base", head], capture_output=True, text=True)
1028
+ self.assertEqual(0, completed.returncode, completed.stdout + completed.stderr)
1029
+ result = json.loads(completed.stdout)
1030
+ [warmed] = result["build_dirs"]
1031
+ if self.native_clone:
1032
+ self.assertEqual(("seeded", str(tree / "target")), (warmed["status"], warmed.get("path")), warmed)
1033
+ else: # The child cannot use the fixture's copy; a filesystem without cloning must skip.
1034
+ self.assertEqual("skipped", warmed["status"], warmed)
1035
+ self.assertIn("cannot clone", warmed["reason"])
1036
+ self.assertEqual(result["build_dirs"], json.loads(Path(result["receipt"]).read_text())["build_dirs"])
1037
+ self.assertEqual("", self.git(tree, "status", "--porcelain", "--untracked-files=all"))
1038
+ self.assertEqual([], list(Path(result["scratch"]).iterdir()))
1039
+
1040
+ def test_compiled_in_out_dir_is_recognized_in_every_spelling(self):
1041
+ compiled = ['env!("OUT_DIR")', 'env!("OUT_DIR", "missing")', 'env ! ( "OUT_DIR" )', 'std::env!(r"OUT_DIR")',
1042
+ 'option_env!("OUT_DIR")', 'myinclude!(concat!(env!("OUT_DIR"), "/a.rs"))', 'env![ "OUT_DIR" ]',
1043
+ 'env/**/!("OUT_DIR")', 'env!(/* why */ "OUT_DIR")', 'env!("OUT\\x5fDIR")', 'env!("\\u{4f}UT_DIR")',
1044
+ 'use std::env as e; e!("OUT_DIR")', 'env!(concat!("OUT", "_DIR"))', 'env!(// note\n"OUT_DIR")',
1045
+ 'env!("\\u{4_f}UT_DIR")', 'env!("\\u{0_0_0_0_4_f}UT_DIR")', 'env!("OUT_\\\n DIR")', 'let a = \'"\'; env!("OUT_DIR")',
1046
+ 'include!(concat!(env!(/* c */ "OUT_DIR"), "/a.rs"))']
1047
+ for text in compiled:
1048
+ self.assertTrue(compiles_path(text), text)
1049
+ for text in ['include!(concat!(env!("OUT_DIR"), "/a.rs"));', 'include_bytes!(concat!(\n env!("OUT_DIR"),\n "/b"))',
1050
+ 'env!("OUT_DIRS")', 'env!("CARGO_PKG_NAME")', 'let s = "OUT_DIR_NAME";']:
1051
+ self.assertFalse(compiles_path(text), text)
1052
+
1053
+ def test_no_cargo_workspace_or_an_unsupported_platform_does_nothing(self):
1054
+ (self.work / "Cargo.toml").unlink()
1055
+ self.assertEqual([], self.warm())
1056
+ (self.work / "Cargo.toml").write_text("[workspace]\n")
1057
+ from unittest import mock
1058
+ with mock.patch.object(sys, "platform", "win32"):
1059
+ [result] = self.warm()
1060
+ self.assertEqual(("skipped", "cloning is unsupported on win32"), (result["status"], result["reason"]))
1061
+ self.assertFalse(self.log.exists())
1062
+
1063
+
1064
+ if __name__ == "__main__":
1065
+ sys.exit(main())