@shardflux/cli 0.20.0 → 0.22.0

package/dist/prepared-repository.py

@@ -0,0 +1,1306 @@
+"""Prepared projects, the VM half: one exec per call, a JSON request on stdin, one JSON line on stdout.
+
+The project cache (/shardflux/project/prepared/<remote hash>) is shared storage (EFS behind an NFS bridge): large
+sequential I/O is fast, but every file lookup is a round trip, and one Git command there reads dozens of files (a
+`git for-each-ref` on a project hub took 9.7 s cold). So nothing on the way to a first prompt runs Git against the
+cache: the cache keeps small marker files (state.json, published/, ready/, media-index) that the probe reads, and a
+start indexes what it uploads into the conversation's own repository on the VM's disk. Filling the cache (indexing
+packs into its repository, publishing, bookkeeping) happens after the reply, in a detached child.
+
+A revision names a snapshot S (the user's working state as a commit on their HEAD, worktree-twin.ts), the large
+untracked media files beside it (Git blobs kept by a content ref, never in S), and a tar of what Git must not hold
+(.env files, filter output). A conversation worktree is S checked out on the conversation branch, as the twin seed does,
+so landing compares S..T; the media land as files its Git ignores, as the user's Git does, possibly after the prompt.
+
+A worktree that holds another revision is never rewritten, unless it is a project template's checkout.
+"""
+import errno
+import fcntl
+import hashlib
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+import uuid
+
+HASH = re.compile(r"^[a-f0-9]{64}$")
+OID = re.compile(r"^[a-f0-9]{40}(?:[a-f0-9]{24})?$")
+BRANCH = re.compile(r"^shard/[a-z0-9][a-z0-9-]{0,47}$")
+PARTS = re.compile(r"^/(?:[^/\0]+/)*prepared-parts/[a-f0-9]{16}$")
+REVISION_TTL = 86400
+KEEP_REVISIONS = 8
+DEPENDENCY_DIRS = {"node_modules", ".venv", "venv", "vendor", "target", ".bundle", ".gradle"}
+EXCLUDE_BEGIN, EXCLUDE_END = "# shardflux: large files outside the snapshot (begin)", "# shardflux: (end)"
+STARTED = time.monotonic()
+TIMINGS = {}
+AFTER = []  # work that follows the reply: run in a detached child once the CLI has its answer
+PRUNE_EVERY = 6 * 3600  # seconds between the passes a start runs after its reply
+FULL_REPACK_EVERY = 7 * 86400  # a full repack (unreachable objects into a cruft pack) at most this often
+PREFETCH_WAIT = 30  # seconds a start waits for the VM's own fetch of its branch (the copy seed's FETCH_ORIGIN waits as long)
+
+
+def mark(name):
+    TIMINGS[name] = int((time.monotonic() - STARTED) * 1000)
+
+
+class Refused(Exception):
+    """An expected refusal: reported as {"error": reason}, exit 0, so the CLI acts on the reason."""
+
+    def __init__(self, reason, message, **extra):
+        super().__init__(message)
+        self.reason, self.extra = reason, extra
+
+
+def git(*argv, data=None, env=None, check=True):
+    p = subprocess.run(["git", "-c", "core.hooksPath=/dev/null", "-c", "gc.auto=0", *argv], input=data, stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE, env=dict(os.environ, GIT_TERMINAL_PROMPT="0", **(env or {})))
+    if check and p.returncode != 0:
+        verb = next((a for a in argv if not a.startswith("-") and "=" not in a and "/" not in a), "")
+        raise RuntimeError("git %s: %s" % (verb, p.stderr.decode("utf-8", "replace").strip()[-1500:]))
+    return p
+
+
+def out(*argv, **kw):
+    return git(*argv, **kw).stdout.decode().strip()
+
+
+def atomic(path, data, mode=0o600):
+    os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
+    tmp = "%s.tmp-%s" % (path, uuid.uuid4().hex)
+    fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
+    try:
+        with os.fdopen(fd, "wb") as f:
+            f.write(data)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp, path)
+        # An exclusive create over NFSv4 can leave the create verifier in the file's times (years ahead): set them.
+        now = time.time()
+        os.utime(path, (now, now))
+    except BaseException:
+        try:
+            os.unlink(tmp)
+        except FileNotFoundError:
+            pass
+        raise
+
+
+def read_json(path, default=None):
+    try:
+        with open(path, "rb") as f:
+            return json.loads(f.read())
+    except (FileNotFoundError, NotADirectoryError, ValueError):
+        return default
+
+
+def safe_path(p):
+    parts = p.split("/")
+    if not p or "\\" in p or "\0" in p or any(x in ("", ".", "..") for x in parts) or parts[0] in (".git", ".shard") or ".git" in parts:
+        raise ValueError("unsafe project path %r" % p)
+    return p
+
+
+def valid_manifest(m):
+    if m.get("version") != 2 or m.get("format") not in ("sha1", "sha256") or not OID.fullmatch(m["head"]) or not OID.fullmatch(m["seed"]):
+        raise ValueError("invalid prepared manifest")
+    for e in m.get("extras", []):
+        safe_path(e["path"])
+        if not HASH.fullmatch(e["sha256"]) or not 0 <= e["mode"] <= 0o777:
+            raise ValueError("invalid prepared file")
+    if m.get("extras") and not HASH.fullmatch(m.get("extras_key") or ""):
+        raise ValueError("invalid prepared files archive")
+    for f in m.get("files", []):
+        safe_path(f["path"])
+        if not OID.fullmatch(f["oid"]) or not 0 <= f["mode"] <= 0o777 or "\n" in f["path"] or "\r" in f["path"]:
+            raise ValueError("invalid prepared media file")
+    for p in m.get("remove", []):
+        safe_path(p)
+    if not all(OID.fullmatch(o) for o in m.get("shallow", [])):
+        raise ValueError("invalid shallow boundary")
+    return m
+
+
+def add_shallow(gitdir, oids):
+    """A shallow clone's boundary, merged into a repository's own (its oldest commits have no parents here either)."""
+    if not oids:
+        return
+    path = os.path.join(gitdir, "shallow")
+    try:
+        have = set(open(path).read().split())
+    except FileNotFoundError:
+        have = set()
+    if not set(oids) <= have:
+        atomic(path, ("\n".join(sorted(have | set(oids))) + "\n").encode(), 0o644)
+
+
+def canonical(m):
+    """The manifest's identity bytes, the same as the CLI's canonical JSON (sorted keys, no spaces, UTF-8)."""
+    return json.dumps(m, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
+
+
+def revision_of(m):
+    return hashlib.sha256(canonical(m)).hexdigest()
+
+
+class Parts:
+    """Where uploaded 8 MiB parts are: the cache's own directory, or a conversation VM's local disk."""
+
+    def __init__(self, directory):
+        self.dir = directory
+
+    def path(self, key):
+        if not HASH.fullmatch(key):
+            raise ValueError("invalid content block")
+        return os.path.join(self.dir, key)
+
+    def listing(self):
+        try:
+            return sorted(n for n in os.listdir(self.dir) if HASH.fullmatch(n))
+        except FileNotFoundError:
+            return []
+
+    def require(self, keys):
+        missing = [k for k in dict.fromkeys(keys) if not os.path.isfile(self.path(k))]
+        if missing:
+            raise Refused("missing", "%d upload parts are missing" % len(missing), chunks=missing)
+
+    def verify(self, keys):
+        """The corrupt or missing parts, removed so the sender uploads them again."""
+        bad = []
+        for key in dict.fromkeys(keys):
+            p = self.path(key)
+            try:
+                with open(p, "rb") as f:
+                    ok = hashlib.file_digest(f, "sha256").hexdigest() == key
+            except FileNotFoundError:
+                ok = False
+            if not ok:
+                bad.append(key)
+                try:
+                    os.unlink(p)
+                except FileNotFoundError:
+                    pass
+        return bad
+
+    def remove(self, keys):
+        for key in dict.fromkeys(keys):
+            try:
+                os.unlink(self.path(key))
+            except FileNotFoundError:
+                pass
+
+
+class Cache:
+    def __init__(self, root):
+        if os.path.islink(root):
+            raise ValueError("project cache is a symlink")
+        self.root = root
+        self.repo = os.path.join(root, "history.git")
+
+    def path(self, *p):
+        return os.path.join(self.root, *p)
+
+    def parts(self, directory=None):
+        if directory and directory != self.path("chunks"):
+            if not PARTS.fullmatch(directory) or ".." in directory.split("/"):
+                raise ValueError("invalid parts directory")
+            return Parts(directory)
+        return Parts(self.path("chunks"))
+
+    def lock(self, shared=False):
+        # On the project space (NFSv4, local_lock=none) flock is a POSIX lock: a shared one needs a readable file.
+        os.makedirs(self.root, mode=0o700, exist_ok=True)
+        f = open(self.path("lock"), "a+")
+        fcntl.flock(f, fcntl.LOCK_SH if shared else fcntl.LOCK_EX)
+        return f
+
+    def ensure_repo(self, fmt):
+        if not os.path.isfile(os.path.join(self.repo, "HEAD")):
+            os.makedirs(self.root, mode=0o700, exist_ok=True)
+            tmp = self.path("history.git.tmp-" + uuid.uuid4().hex)
+            git("init", "-q", "--bare", "--object-format=" + fmt, tmp)
+            git("-C", tmp, "config", "core.logAllRefUpdates", "false")
+            try:
+                os.rename(tmp, self.repo)
+            except OSError as error:
+                if error.errno not in (errno.EEXIST, errno.ENOTEMPTY):
+                    raise
+                shutil.rmtree(tmp, ignore_errors=True)
+        got = out("--git-dir=" + self.repo, "rev-parse", "--show-object-format")
+        if got != fmt:
+            raise Refused("format", "the project cache stores %s objects, this repository uses %s" % (got, fmt))
+
+    def git(self, *argv, **kw):
+        return git("--git-dir=" + self.repo, *argv, **kw)
+
+    def manifest(self, revision):
+        if not HASH.fullmatch(revision):
+            raise ValueError("invalid prepared revision")
+        m = valid_manifest(json.loads(open(self.path("manifests", revision + ".json"), "rb").read()))
+        if revision_of(m) != revision:
+            raise ValueError("prepared manifest identity changed")
+        return m
+
+    # The markers the probe reads instead of Git (written by import, under the cache lock).
+    def state(self):
+        return read_json(self.path("state.json"), {}) or {}
+
+    def published(self, revision):
+        return os.path.isfile(self.path("published", revision + ".json"))
+
+    def ready(self, revision):
+        return os.path.isfile(self.path("ready", revision + ".json"))
+
+    def media_index(self):
+        try:
+            return set(open(self.path("media-index")).read().split())
+        except FileNotFoundError:
+            return set()
+
+    def present_objects(self, oids):
+        """Git's own answer (after the reply only: Git on the cache is slow)."""
+        if not oids or not os.path.isfile(os.path.join(self.repo, "HEAD")):
+            return set()
+        listing = self.git("cat-file", "--batch-check=%(objectname) %(objecttype)", data=("\n".join(oids) + "\n").encode()).stdout.decode().split("\n")
+        return {line.split(" ")[0] for line in listing if line.endswith(" blob")}
+
+
+def imported(cache, revision):
+    return read_json(cache.path("imports", revision + ".json"), []) or []
+
+
+def mark_used(cache, revision):
+    """When a revision was last wanted, in the file itself: the project space's file times are not reliable."""
+    atomic(cache.path("used", revision + ".json"), json.dumps({"at": int(time.time())}).encode())
+
+
+def refresh_state(cache):
+    """state.json: the commits the cache holds whole (what a sender may leave out of a pack) and whether its history is
+    blobless. Written under the cache lock after the cache changes."""
+    tips = cache.git("for-each-ref", "--sort=-committerdate", "--count=64", "--format=%(objectname)",
+                     "refs/shard/prepared/", "refs/shard/origin/", "refs/shard/upstream/").stdout.decode().split()
+    partial = cache.git("config", "--bool", "remote.origin.promisor", check=False).stdout.strip() == b"true"
+    atomic(cache.path("state.json"), json.dumps({"tips": list(dict.fromkeys(tips)), "partial": partial,
+                                                 "format": out("--git-dir=" + cache.repo, "rev-parse", "--show-object-format")}).encode())
+
+
+def prefetch_dir(root):
+    """Where the VM fetches a new conversation's branch from origin at boot (guest coding 53+, prefetch.go), keyed by
+    its worktree as the copy seed's FETCH_ORIGIN reads it: history/ and tip/ bare repositories, remote, commit."""
+    home = os.environ.get("HOME") or "/home/user"
+    return os.path.join(home, ".cache", "shardflux", "prefetch", hashlib.sha256(root.encode("utf-8")).hexdigest()[:16])
+
+
+def prefetch_status(root, url):
+    """"ready" or "running" when the VM holds, or is still fetching, this worktree's branch from `url`. With neither,
+    a marker keeps a late one from starting: this start brings its history another way."""
+    if not url:
+        return None
+    pre = prefetch_dir(root)
+    try:
+        os.makedirs(os.path.dirname(pre), exist_ok=True)
+        fd = os.open(pre + ".lock", os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o600)
+    except OSError:
+        return None
+    try:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            return "running"
+        if os.path.isdir(os.path.join(pre, "history")):
+            with open(os.path.join(pre, "remote"), encoding="utf-8", errors="replace") as f:
+                return "ready" if f.read().strip() == url else None
+        os.makedirs(pre, exist_ok=True)
+        return None
+    except OSError:
+        return None
+    finally:
+        os.close(fd)
+
+
+def wait_prefetch(root):
+    """The lock of the VM's own fetch of this worktree's branch, once that fetch is over (bounded wait), or None when
+    there is none. Waited for outside the cache lock: other VMs' imports go on meanwhile."""
+    pre = prefetch_dir(root)
+    if not (os.path.exists(pre + ".lock") or os.path.isdir(pre)):
+        return None
+    fd = os.open(pre + ".lock", os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o600)
+    deadline = time.monotonic() + PREFETCH_WAIT
+    while True:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            return fd
+        except BlockingIOError:
+            if time.monotonic() > deadline:
+                os.close(fd)
+                return None
+            time.sleep(0.05)
+
+
+def adopt_prefetch(cache, root, url, fmt):
+    """Moves the VM's own fetch of this branch (its lock held: wait_prefetch) into the cache: the history a fetch from
+    origin would bring, already here. Returns its commit, or None (another remote, failed). Leaves a marker so no second
+    prefetch starts. Called under the cache lock."""
+    pre = prefetch_dir(root)
+    try:
+        with open(os.path.join(pre, "remote"), encoding="utf-8", errors="replace") as f:
+            remote = f.read().strip()
+        with open(os.path.join(pre, "commit"), encoding="utf-8", errors="replace") as f:
+            commit = f.read().strip()
+    except OSError:
+        return None
+    if fmt != "sha1" or remote != url or not re.fullmatch(r"[a-f0-9]{40}", commit):
+        return None
+    dest = os.path.join(cache.repo, "objects", "pack")
+    packs = [os.path.join(pre, repo, "objects", "pack", n) for repo in ("history", "tip")
+             if os.path.isdir(os.path.join(pre, repo, "objects", "pack")) for n in os.listdir(os.path.join(pre, repo, "objects", "pack"))
+             if n.startswith("pack-")]
+    # Each pack before its index: a reader never sees an index without its pack. Another file system: copy, then
+    # rename into place.
+    for suffix in (".pack", ".rev", ".promisor", ".idx"):
+        for src in packs:
+            if src.endswith(suffix):
+                tmp = os.path.join(dest, ".adopt-" + uuid.uuid4().hex)
+                shutil.copyfile(src, tmp)
+                os.replace(tmp, os.path.join(dest, os.path.basename(src)))
+    shutil.rmtree(pre, ignore_errors=True)
+    os.makedirs(pre, exist_ok=True)
+    return commit if cache.git("cat-file", "-e", commit + "^{commit}", check=False).returncode == 0 else None
+
+
+def probe(cache, req):
+    """What the cache holds for this revision, from its markers alone. With `apply`, a revision whose snapshot and
+    archive are here is checked out in the same exec (a warm start is one exec); media on their way land later."""
+    m = valid_manifest(req["manifest"])
+    revision = revision_of(m)
+    ready = cache.ready(revision)
+    published = ready or cache.published(revision)
+    extras = not m.get("extras") or os.path.isfile(cache.path("extras", m["extras_key"]))
+    media = {f["oid"] for f in m.get("files", [])}
+    missing = [] if ready else sorted(media - cache.media_index())
+    state = cache.state()
+    mark("probe")
+    applied = None
+    if published and extras and req.get("apply"):
+        applied = apply(cache, dict(req["apply"], revision=revision), present=media - set(missing))
+    if req.get("pin") and (not applied or missing):
+        AFTER.append(lambda: pin(cache, {"id": req["pin"], "revision": revision}))
+    if published:
+        AFTER.append(lambda: mark_used(cache, revision))
+        AFTER.append(lambda: maybe_prune(cache))
+    parts = cache.parts(req.get("parts_dir"))
+    # A start whose history the cache may lack: the VM's own fetch of its branch can stand in for sending it.
+    fetched = prefetch_status(req["apply"]["root"], m.get("origin")) if req.get("apply") and not published else None
+    return {"revision": revision, "ready": published and extras and not missing, "published": published, "tips": state.get("tips", []),
+            "partial": state.get("partial", False), "chunks": parts.listing(), "imported": imported(cache, revision), "extras_present": extras,
+            "missing_files": missing, **({"applied": applied} if applied else {}), **({"prefetch": fetched} if fetched else {})}
+
+
+def origin(cache, req):
+    """Fetch the branch's history from its origin into the cache, inside the cloud: commits and trees for all history,
+    plus every file of the branch tip in one pack. Older blobs stay with origin (a partial cache)."""
+    url, ref, name, expect = req["url"], req["ref"], req["name"], req.get("expect")
+    if not re.fullmatch(r"refs/heads/[^\s\0~^:?*\[\\]+", ref) or not re.fullmatch(r"[a-f0-9]{16}", name) or url.startswith("-") \
+            or (expect is not None and not OID.match(expect)) or (req.get("prefetch") is not None and not os.path.isabs(req["prefetch"])):
+        raise ValueError("invalid origin request")
+    has = lambda oid: oid is not None and cache.git("cat-file", "-e", oid + "^{commit}", check=False).returncode == 0
+    prefetched = wait_prefetch(req["prefetch"]) if req.get("prefetch") else None
+    try:
+        return fetch_origin(cache, req, url, ref, name, expect, has, prefetched is not None)
+    finally:
+        if prefetched is not None:
+            os.close(prefetched)
+
+
+def fetch_origin(cache, req, url, ref, name, expect, has, prefetched):
+    with cache.lock():
+        cache.ensure_repo(req["format"])
+        cache.git("config", "remote.origin.url", url)
+        dst = "refs/shard/origin/" + name
+        # The VM fetched this conversation's branch at boot: its history is the cache's, with no second fetch when it
+        # holds the commit this computer's upstream is at (or a newer one).
+        adopted = adopt_prefetch(cache, req["prefetch"], url, req["format"]) if prefetched else None
+        if adopted:
+            cache.git("config", "remote.origin.promisor", "true")
+            cache.git("config", "remote.origin.partialclonefilter", "blob:none")
+            cache.git("update-ref", dst, adopted)
+        had = cache.git("rev-parse", "-q", "--verify", dst, check=False).returncode == 0
+        partial = cache.git("config", "--bool", "remote.origin.promisor", check=False).stdout.strip() == b"true"
+        args = ["-c", "credential.interactive=false", "fetch", "-q", "--no-tags", "--no-write-fetch-head"]
+        if adopted and (expect is None or has(expect)):
+            pass
+        elif had and not partial:
+            cache.git(*args, "origin", "+%s:%s" % (ref, dst))
+        else:
+            # The tip's files in one depth-1 pack beside the blobless history: a checkout of the tip asks origin for
+            # nothing one blob at a time.
+            tip_repo = cache.path("tip-" + uuid.uuid4().hex)
+            git("init", "-q", "--bare", "--object-format=" + req["format"], tip_repo)
+            try:
+                tip = subprocess.Popen(["git", "-C", tip_repo, "-c", "credential.interactive=false", "-c", "fetch.unpackLimit=1",
+                                        "fetch", "-q", "--no-tags", "--depth=1", url, "+%s:refs/tip" % ref],
+                                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=dict(os.environ, GIT_TERMINAL_PROMPT="0"))
+                cache.git("config", "remote.origin.promisor", "true")
+                cache.git("config", "remote.origin.partialclonefilter", "blob:none")
+                fetched = cache.git(*args, "--filter=blob:none", "origin", "+%s:%s" % (ref, dst), check=False)
+                if tip.wait() == 0:
+                    take_packs(tip_repo, cache.repo)
+                if fetched.returncode != 0:
+                    if not had:
+                        cache.git("config", "--unset", "remote.origin.promisor", check=False)
+                        cache.git("config", "--unset", "remote.origin.partialclonefilter", check=False)
+                    raise Refused("origin", fetched.stderr.decode("utf-8", "replace").strip()[-600:])
+            finally:
+                shutil.rmtree(tip_repo, ignore_errors=True)
+        tip = out("--git-dir=" + cache.repo, "rev-parse", dst)
+        whole = has(expect)
+        if whole and expect != tip and cache.git("config", "--bool", "remote.origin.promisor", check=False).stdout.strip() == b"true":
+            # This computer's upstream is behind origin's tip: the cache holds that commit's history but only the tip's
+            # files. The sender leaves out what its upstream reaches, so that commit's files come too, in one pack.
+            whole = fetch_files(cache, url, expect, req["format"])
+        cache.git("pack-refs", "--all")
+        refresh_state(cache)
+        state = cache.state()
+        return {"tip": tip, "tips": state["tips"], "partial": state["partial"], "adopted": bool(adopted),
+                **({"has_expect": whole} if expect else {})}
+
+
+def take_packs(source, repo):
+    """Moves a fetch's packs into the cache, each pack before its index: a reader never sees an index without its pack."""
+    pack_dir = os.path.join(source, "objects", "pack")
+    packs = sorted(n for n in os.listdir(pack_dir) if n.startswith("pack-"))
+    for suffix in (".pack", ".rev", ".promisor", ".idx"):
+        for n in packs:
+            if n.endswith(suffix):
+                os.replace(os.path.join(pack_dir, n), os.path.join(repo, "objects", "pack", n))
+
+
+def fetch_files(cache, url, commit, fmt):
+    """Every file of one commit, from origin in one depth-1 pack. False when origin will not send it."""
+    tip_repo = cache.path("tip-" + uuid.uuid4().hex)
+    git("init", "-q", "--bare", "--object-format=" + fmt, tip_repo)
+    try:
+        fetched = git("-C", tip_repo, "-c", "credential.interactive=false", "-c", "fetch.unpackLimit=1", "fetch", "-q", "--no-tags",
+                      "--depth=1", url, "+%s:refs/tip" % commit, check=False)
+        if fetched.returncode != 0:
+            return False
+        take_packs(tip_repo, cache.repo)
+        return True
+    finally:
+        shutil.rmtree(tip_repo, ignore_errors=True)
+
+
+def index_pack(gitdir, parts, keys, thin):
+    """Streams the parts of one pack into `index-pack` for this repository (its alternates resolve a thin pack's bases)."""
+    parts.require(keys)
+    args = ["--git-dir=" + gitdir, "-c", "core.hooksPath=/dev/null", "index-pack", "--stdin"] + (["--fix-thin"] if thin else [])
+    p = subprocess.Popen(["git", *args], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+    try:
+        try:
+            for key in keys:
+                with open(parts.path(key), "rb") as f:
+                    shutil.copyfileobj(f, p.stdin, 1 << 20)
+        except BrokenPipeError:
+            pass
+        p.stdin.close()
+        err = p.stderr.read()
+        if p.wait() != 0:
+            bad = parts.verify(keys)
+            if bad:
+                raise Refused("corrupt", "%d upload parts failed verification" % len(bad), chunks=bad)
+            raise RuntimeError("Git rejected the pack: " + err.decode("utf-8", "replace").strip()[-800:])
+    finally:
+        if p.poll() is None:
+            p.kill()
+            p.wait()
+
+
+def assemble(parts, keys, target, digest):
+    """Concatenates parts into one file whose SHA-256 must be `digest`."""
+    if os.path.isfile(target):
+        return
+    parts.require(keys)
+    os.makedirs(os.path.dirname(target), mode=0o700, exist_ok=True)
+    tmp = "%s.tmp-%s" % (target, uuid.uuid4().hex)
+    h = hashlib.sha256()
+    with open(tmp, "xb") as f:
+        os.fchmod(f.fileno(), 0o600)
+        for key in keys:
+            with open(parts.path(key), "rb") as part:
+                while b := part.read(1 << 20):
+                    h.update(b)
+                    f.write(b)
+        f.flush()
+        os.fsync(f.fileno())
+    if h.hexdigest() != digest:
+        os.unlink(tmp)
+        raise Refused("corrupt", "the project files archive failed verification", chunks=parts.verify(keys))
+    os.replace(tmp, target)
+    now = time.time()
+    os.utime(target, (now, now))
+
+
+def keep_files(cache, m, revision):
+    """A tree naming the revision's media blobs that have arrived keeps them in the cache while the revision is; the
+    media index lets the probe answer without Git."""
+    have = sorted(cache.present_objects([f["oid"] for f in m.get("files", [])]))
+    if have:
+        tree = out("--git-dir=" + cache.repo, "mktree", "-z", data="".join("100644 blob %s\t%s\0" % (o, o) for o in have).encode())
+        cache.git("update-ref", "refs/shard/content/" + revision, tree)
+        index = cache.media_index()
+        if not set(have) <= index:
+            atomic(cache.path("media-index"), ("\n".join(sorted(index | set(have))) + "\n").encode())
+    return set(have)
+
+
+def import_(cache, req):
+    """Index the uploaded packs and the files archive into the cache; publish S once every object it needs is here. A
+    failure refreshes the cache's record of what it holds, so the sender's next attempt negotiates from the truth."""
+    with cache.lock():
+        try:
+            return import_locked(cache, req)
+        except Exception as error:
+            if not (isinstance(error, Refused) and error.reason in ("missing", "corrupt")):
+                try:
+                    cache.ensure_repo(req["manifest"]["format"])
+                    refresh_state(cache)
+                except Exception:
+                    pass
+            raise
+
+
+def import_locked(cache, req):
+    m = valid_manifest(req["manifest"])
+    revision = revision_of(m)
+    parts = cache.parts(req.get("parts_dir"))
+    cache.ensure_repo(m["format"])
+    consumed, done = [], imported(cache, revision)
+    for pack in req.get("packs", []):
+        if not HASH.fullmatch(pack["key"]):
+            raise ValueError("invalid pack identity")
+        if pack["key"] not in done:
+            index_pack(cache.repo, parts, pack["chunks"], pack.get("thin", False))
+            done.append(pack["key"])
+            atomic(cache.path("imports", revision + ".json"), json.dumps(done).encode())
+        consumed += pack["chunks"]
+    mark("index_pack")
+    if m.get("extras") and req.get("extras_chunks") is not None:
+        assemble(parts, req["extras_chunks"], cache.path("extras", m["extras_key"]), m["extras_key"])
+        consumed += req["extras_chunks"]
+    atomic(cache.path("manifests", revision + ".json"), canonical(m))
+    add_shallow(cache.repo, m.get("shallow", []))
+    if not cache.published(revision):
+        tops = [m["seed"], m["head"]] + ([m["upstream"]["oid"]] if m.get("upstream") else [])
+        partial = cache.git("config", "--bool", "remote.origin.promisor", check=False).stdout.strip() == b"true"
+        # Only the new objects are walked (what the cache's refs reach is whole already); a blobless cache accepts
+        # the blobs its origin promises.
+        check = cache.git("rev-list", "--objects", "--quiet", *(["--missing=allow-promisor"] if partial else []),
+                          *tops, "--not", "--all", check=False)
+        if check.returncode != 0:
+            raise Refused("incomplete", "the snapshot is missing objects: " + check.stderr.decode("utf-8", "replace").strip()[-400:])
+        updates = "update refs/shard/prepared/%s %s\n" % (revision, m["seed"])
+        if m.get("upstream"):
+            updates += "update refs/shard/upstream/%s %s\n" % (revision, m["upstream"]["oid"])
+        cache.git("update-ref", "--stdin", data=updates.encode())
+        cache.git("pack-refs", "--all")
+        atomic(cache.path("published", revision + ".json"), json.dumps({"at": int(time.time())}).encode())
+        refresh_state(cache)
+    mark_used(cache, revision)
+    have = keep_files(cache, m, revision)
+    missing_files = sorted({f["oid"] for f in m.get("files", [])} - have)
+    extras = not m.get("extras") or os.path.isfile(cache.path("extras", m["extras_key"]))
+    if extras and not missing_files:
+        atomic(cache.path("ready", revision + ".json"), json.dumps({"at": int(time.time())}).encode())
+    mark("publish")
+    parts.remove(consumed)
+    return {"revision": revision, "complete": extras and not missing_files, "missing_files": missing_files, "tips": cache.state().get("tips", [])}
+
+
+def stage(cache, req):
+    """A conversation start's upload, indexed into the conversation's own repository on the VM's disk and checked out
+    at once (the cache answers no Git question on the way). The cache receives the same packs after the reply."""
+    m = valid_manifest(req["manifest"])
+    revision = revision_of(m)
+    parts = cache.parts(req["parts_dir"])
+    target = dict(req["apply"], revision=revision)
+    root = target["root"]
+    if not root.startswith("/") or ".." in root.split("/"):
+        raise ValueError("invalid prepared binding")
+    current = marker(root)
+    applied = None
+    if current and current.get("revision") == revision and current.get("state") == "ready":
+        applied = {"revision": revision, "mode": "existing"}
+    inplace = os.path.isdir(os.path.join(root, ".git")) or (os.path.isdir(root) and bool(os.listdir(root)))
+    lock = cache.lock(shared=True)
+    work = None
+    try:
+        if not applied:
+            work = prepare_root(cache, m, root, target, inplace)
+        gitdir = os.path.join(work or root, ".git")
+        for pack in req.get("packs", []):
+            if not HASH.fullmatch(pack["key"]):
+                raise ValueError("invalid pack identity")
+            index_pack(gitdir, parts, pack["chunks"], pack.get("thin", False))
+        mark("index_local")
+        extras_tar = None
+        if m.get("extras"):
+            cached = cache.path("extras", m["extras_key"])
+            extras_tar = cached if os.path.isfile(cached) else os.path.join(gitdir, "shard-extras-%s.tar" % m["extras_key"])
+            if extras_tar != cached:
+                assemble(parts, req.get("extras_chunks") or [], extras_tar, m["extras_key"])
+        media = {f["oid"] for f in m.get("files", [])}
+        # What this start sent, from the request itself: asking Git would read the media pack over the network.
+        local = set(req.get("media_staged", [])) & media
+        if not applied:
+            applied = apply_locked(cache, target, revision, root, target["branch"], m=m, extras_tar=extras_tar,
+                                   present=local | (media & cache.media_index()), inplace=inplace, work=work)
+        else:
+            # Media that followed the prompt: they land before this call returns.
+            land(cache, root, [])
+    except BaseException as error:
+        discard(work, root)
+        # Parts the sender can resend (missing, damaged) stay for its retry; after any other failure it negotiates again.
+        if not (isinstance(error, Refused) and error.reason in ("missing", "corrupt")):
+            shutil.rmtree(parts.dir, ignore_errors=True)
+        raise
+    finally:
+        lock.close()
+    packs, extras_chunks = req.get("packs", []), req.get("extras_chunks")
+    if packs or extras_chunks:
+        # The cache gets what this start sent, for the next start, after the reply.
+        AFTER.insert(0, lambda: import_(cache, {"manifest": m, "packs": packs, "extras_chunks": extras_chunks, "parts_dir": parts.dir}))
+        AFTER.append(lambda: maybe_prune(cache))
+    if req.get("pin") and not (media - local - cache.media_index()):
+        AFTER.append(lambda: pin(cache, {"id": req["pin"], "release": True}))
+    return {"revision": revision, "applied": applied, "timings": TIMINGS}
+
+
+def present_in(root, oids):
+    """Which of these blobs this repository (with its alternates) holds."""
+    if not oids:
+        return set()
+    listing = git("-C", root, "cat-file", "--batch-check=%(objectname) %(objecttype)", data=("\n".join(oids) + "\n").encode()).stdout.decode().split("\n")
+    return {line.split(" ")[0] for line in listing if line.endswith(" blob")}
+
+
+def parent_fd(rootfd, path):
+    """The parent directory of path, opened without following symbolic links (created when missing)."""
+    parts = path.split("/")
+    fd = os.dup(rootfd)
+    try:
+        for part in parts[:-1]:
+            try:
+                os.mkdir(part, 0o755, dir_fd=fd)
+            except FileExistsError:
+                pass
+            child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
+            os.close(fd)
+            fd = child
+        return fd, parts[-1]
+    except BaseException:
+        os.close(fd)
+        raise
+
+
+def place(root, path, mode, write, symlink=None, only_if_absent=False):
+    """Writes one file (or link) through its real parent directories, replacing atomically; False when skipped."""
+    rootfd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+    try:
+        fd, name = parent_fd(rootfd, safe_path(path))
+    finally:
+        os.close(rootfd)
+    tmp = ".shard-tmp-" + uuid.uuid4().hex
+    try:
+        if only_if_absent:
+            try:
+                os.lstat(name, dir_fd=fd)
+                return False
+            except FileNotFoundError:
+                pass
+        if symlink is not None:
+            os.symlink(symlink, tmp, dir_fd=fd)
+        else:
+            dest = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode, dir_fd=fd)
+            with os.fdopen(dest, "wb") as w:
+                write(w)
+                os.fchmod(w.fileno(), mode)
+        try:
+            if only_if_absent:
+                os.link(tmp, name, src_dir_fd=fd, dst_dir_fd=fd, follow_symlinks=False)
+            else:
+                os.replace(tmp, name, src_dir_fd=fd, dst_dir_fd=fd)
+        except FileExistsError:
+            return False  # the agent wrote it meanwhile: its version stands
+        except IsADirectoryError:
+            # A directory where the user's computer has a file (a template's leftover): the file wins.
+            shutil.rmtree(os.path.join(root, path))
+            os.replace(tmp, name, src_dir_fd=fd, dst_dir_fd=fd)
+        return True
+    finally:
+        try:
+            os.unlink(tmp, dir_fd=fd)
+        except FileNotFoundError:
+            pass
+        os.close(fd)
+
+
+def extract_extras(m, root, archive):
+    """The .env files and filter output, written exactly as they are on the user's computer."""
+    if not m.get("extras"):
+        return 0
+    import tarfile
+    wanted = {e["path"]: e for e in m["extras"]}
+    with open(archive, "rb") as f:
+        if hashlib.file_digest(f, "sha256").hexdigest() != m["extras_key"]:
+            raise RuntimeError("the project files archive changed")
+    written = 0
+    with tarfile.open(archive, "r:") as tar:
+        for info in tar:
+            e = wanted.get(info.name)
+            if e is None:
+                continue
+            if e["symlink"]:
+                if not info.issym() or info.linkname != e["link_target"]:
+                    raise RuntimeError("archive entry differs: " + info.name)
+                place(root, info.name, 0o777, None, symlink=e["link_target"])
+            else:
+                src, h = tar.extractfile(info), hashlib.sha256()
+
+                def copy(w, src=src, h=h):
+                    while b := src.read(1 << 20):
+                        h.update(b)
+                        w.write(b)
+                place(root, info.name, e["mode"], copy)
+                if h.hexdigest() != e["sha256"]:
+                    raise RuntimeError("archive entry differs: " + info.name)
+            written += 1
+    if written != len(wanted):
+        raise RuntimeError("the project files archive is incomplete")
+    return written
+
+
+def write_blobs(root, files, only_if_absent):
+    """Streams media blobs from the worktree's repository (its alternates include the cache) into their paths."""
+    if not files:
+        return []
+    p = subprocess.Popen(["git", "-C", root, "cat-file", "--batch"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    placed = []
+    try:
+        p.stdin.write(("".join(f["oid"] + "\n" for f in files)).encode())
+        p.stdin.close()
+        for f in files:
+            header = p.stdout.readline().decode().split()
+            if len(header) != 3 or header[0] != f["oid"] or header[1] != "blob":
+                raise RuntimeError("missing media blob " + f["oid"])
+            size = int(header[2])
+
+            def copy(w, size=size):
+                left = size
+                while left:
+                    b = p.stdout.read(min(left, 1 << 20))
+                    if not b:
+                        raise RuntimeError("short media blob")
+                    w.write(b)
+                    left -= len(b)
+            if place(root, f["path"], f["mode"], copy, only_if_absent=only_if_absent):
+                placed.append(f["path"])
+            else:
+                for at in range(0, size, 1 << 20):
+                    p.stdout.read(min(1 << 20, size - at))
+            p.stdout.read(1)
+    finally:
+        p.stdout.close()
+        p.wait()
+    return placed
+
+
+def exclude_pattern(path):
+    """A gitignore line matching exactly this path from the worktree root."""
+    s = "".join("\\" + c if c in "\\*?[" else c for c in path)
+    if s[:1] in ("!", "#"):
+        s = "\\" + s
+    stripped = s.rstrip(" ")
+    return "/" + stripped + "\\ " * (len(s) - len(stripped))
+
+
+def write_excludes(root, files):
+    """The media are untracked on the user's computer; here too: Git ignores them, so an agent never commits them."""
+    path = os.path.join(root, ".git", "info", "exclude")
+    try:
+        text = open(path, encoding="utf-8").read()
+    except FileNotFoundError:
+        text = ""
+    if EXCLUDE_BEGIN in text:
+        text = text[:text.index(EXCLUDE_BEGIN)] + text[text.index(EXCLUDE_END) + len(EXCLUDE_END):].lstrip("\n")
+    if files:
+        text = text.rstrip("\n") + ("\n" if text.strip() else "") + "\n".join([EXCLUDE_BEGIN] + [exclude_pattern(f["path"]) for f in files] + [EXCLUDE_END]) + "\n"
+    atomic(path, text.encode(), 0o644)
+
+
+def configure(work, req, partial, root=None):
+    """The repository's settings in one write: origin, identity, the branch's upstream, exclude rules. A key already set
+    to the same value is left; one set to another is replaced (a template's checkout taken over keeps one origin)."""
+    root = root or work
+    cfg = [("gc.auto", "0")]
+    o = req.get("origin_url")
+    if o:
+        cfg.append(("remote.origin.url", o))
+        if partial:
+            # The cache is a blobless clone of origin: this repository asks origin for an old blob it needs.
+            cfg += [("remote.origin.promisor", "true"), ("remote.origin.partialclonefilter", "blob:none"),
+                    ("core.repositoryformatversion", "1"), ("extensions.partialClone", "origin")]
+    for key in ("name", "email"):
+        if req.get("user", {}).get(key):
+            cfg.append(("user." + key, req["user"][key]))
+    up = req.get("tracking")
+    if up and o:
+        cfg += [("branch.%s.remote" % up["branch"], "origin"), ("branch.%s.merge" % up["branch"], up["merge"])]
+    if req.get("excludes") is not None:
+        atomic(os.path.join(work, ".git", "shard-sync-excludes"), req["excludes"].encode())
+        cfg.append(("core.excludesFile", os.path.join(root, ".git", "shard-sync-excludes")))
+    path = os.path.join(work, ".git", "config")
+    have = {}
+    for line in git("config", "--file", path, "--null", "--list", check=False).stdout.decode("utf-8", "replace").split("\0"):
+        key, _, value = line.partition("\n")
+        if key:
+            have.setdefault(key.lower(), []).append(value)
+    append = []
+    for key, value in cfg:
+        current = have.get(key.lower())
+        if current == [str(value)]:
+            continue
+        if current:
+            git("config", "--file", path, "--replace-all", key, str(value))
+        else:
+            append.append((key, value))
+    # One process for a new repository's keys: `git config --file` per key spawns Git each time.
+    with open(path, "a") as f:
+        sections = {}
+        for key, value in append:
+            section, _, name = key.rpartition(".")
+            sections.setdefault(section, []).append((name, value))
+        for section, items in sections.items():
+            head, _, sub = section.partition(".")
+            f.write('[%s%s]\n' % (head, ' "%s"' % sub.replace("\\", "\\\\").replace('"', '\\"') if sub else ""))
+            for name, value in items:
+                f.write('\t%s = "%s"\n' % (name, str(value).replace("\\", "\\\\").replace('"', '\\"')))
+
+
+def marker(root):
+    return read_json(os.path.join(root, ".git", "shard-prepared.json"))
+
+
+def pending_path(root):
+    return os.path.join(root, ".git", "shard-pending-files.json")
+
+
+def apply(cache, req, present=None):
+    revision, root, branch = req["revision"], req["root"], req["branch"]
+    if not HASH.fullmatch(revision) or not BRANCH.fullmatch(branch) or not root.startswith("/") or ".." in root.split("/"):
+        raise ValueError("invalid prepared binding")
+    lock = cache.lock(shared=True)
+    try:
+        result = apply_locked(cache, req, revision, root, branch, present=present)
+    finally:
+        lock.close()
+    if req.get("release_pin"):
+        AFTER.append(lambda: pin(cache, {"id": req["release_pin"], "release": True}))
+    return result
+
+
+def check_owner(root, revision, req):
+    """A conversation's worktree is never rewritten. Refused when this VM's conversation made it (our marker, from this
+    workspace) or another seed did (the conversation's branch is there: the copy seed, a restore). A project template's
+    checkout, and one a VM's image brought from the VM that saved it, are taken over."""
+    if req.get("workspace") is not None and not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", str(req["workspace"])):
+        raise ValueError("invalid workspace")
+    current = marker(root)
+    if current:
+        if current.get("revision") == revision or current.get("role") == "template" or req.get("replace_preparation"):
+            return
+        if current.get("workspace") and req.get("workspace") and current["workspace"] != req["workspace"]:
+            return
+        raise Refused("owned", "this worktree already holds another prepared revision")
+    if os.path.isdir(os.path.join(root, ".git")) and \
+            git("-C", root, "rev-parse", "-q", "--verify", "refs/heads/" + req["branch"], check=False).returncode == 0:
+        raise Refused("owned", "this worktree already holds the conversation")
+
+
+def prepare_root(cache, m, root, req, inplace):
+    """The worktree's repository, borrowing the cache's objects; returns where it is built. A new one is built beside
+    the worktree path and moved into place whole (apply_locked): the guest starts the agent as soon as <worktree>/.git
+    exists, so it never sees one half made, and a failed start leaves nothing at the path. A checkout already there (a
+    project template's, files a new VM's image left) is taken over in place."""
+    check_owner(root, revision_of(m), req)
+    if os.path.lexists(root) and (os.path.islink(root) or not os.path.isdir(root)):
+        raise Refused("exists", "the worktree path is not a directory")
+    if inplace:
+        if not os.path.isdir(os.path.join(root, ".git")):
+            if not req.get("adopt"):
+                raise Refused("exists", "refusing to replace a nonempty directory")
+            git("init", "-q", "--object-format=" + m["format"], root)
+        work = root
+    else:
+        parent, name = os.path.split(root.rstrip("/"))
+        os.makedirs(parent, exist_ok=True)
+        work = os.path.join(parent, ".shard-preparing-%s-%s" % (name, uuid.uuid4().hex[:12]))
+        git("init", "-q", "--object-format=" + m["format"], work)
+    info = os.path.join(work, ".git", "objects", "info")
+    os.makedirs(info, exist_ok=True)
+    with open(os.path.join(info, "alternates"), "w") as f:
+        f.write(os.path.join(cache.repo, "objects") + "\n")
+    add_shallow(os.path.join(work, ".git"), m.get("shallow", []))
+    return work
+
+
+def move_into_place(work, root):
+    """Moves a repository built aside into the worktree path (an empty directory there is replaced)."""
+    if work != root:
+        os.rename(work, root)
+
+
+def discard(work, root):
+    if work and work != root:
+        shutil.rmtree(work, ignore_errors=True)
+
+
+def write_setup_intent(intent):
+    """The hash a new conversation's worktree is set up with (project-template-setup.ts recordSetupIntent), written
+    before its repository appears: its guest then starts the agent with the setup's context and toolchains."""
+    conversation, deps_hash = intent.get("conversation", ""), intent.get("deps_hash", "")
+    if not re.fullmatch(r"conv_[a-z0-9]{26}", conversation) or not HASH.match(deps_hash):
+        raise ValueError("invalid setup intent")
+    home = os.environ.get("HOME") or "/home/user"
+    atomic(os.path.join(home, ".local", "share", "shardflux", "conversation-setup", conversation + ".json"),
+           ('{"deps_hash":"%s"}' % deps_hash).encode())
+
+
+def apply_locked(cache, req, revision, root, branch, m=None, extras_tar=None, present=None, inplace=None, work=None):
+    """Check out S as the conversation branch. A fresh root borrows the cache's objects (alternates) and only writes
+    files; a root that already holds a checkout (the project template's, or an interrupted apply) moves to S in place,
+    keeping its ignored dependency folders. Media land after the reply; those not here yet stay pending."""
+    m = m or cache.manifest(revision)
+    current = marker(root)
+    if current and current.get("revision") == revision and current.get("state") == "ready":
+        return {"revision": revision, "mode": "existing", "pending": len(read_json(pending_path(root), []) or []), "landing": 0, "dependencies": 0,
+                "timings": TIMINGS}
+    if req.get("setup_intent"):
+        write_setup_intent(req["setup_intent"])
+    if inplace is None:
+        # A checkout already there (the template's, files a new VM's image left, an interrupted apply) moves to S in place.
+        inplace = os.path.isdir(os.path.join(root, ".git")) or (os.path.isdir(root) and bool(os.listdir(root)))
+    try:
+        work = work or prepare_root(cache, m, root, req, inplace)
+        result = checkout_locked(cache, req, revision, root, work, branch, m, extras_tar, present, inplace)
+    except BaseException:
+        discard(work, root)
+        raise
+    # Media are files Git ignores: the agent can start before they are written. They land right after the reply.
+    land_now(cache, root, [], sync=not req.get("background", True))
+    if req.get("after"):
+        AFTER.append(lambda: run_after(root, req["after"]))
+    if req.get("background", True):
+        AFTER.append(lambda: background(root, req.get("dissociate", True) is True))
+    return result
+
+
+def checkout_locked(cache, req, revision, root, work, branch, m, extras_tar, present, inplace):
+    mode = "inplace" if inplace else "fresh"
+    role = "template" if req.get("preparation") else "conversation"
+    owner = {"revision": revision, "seed": m["seed"], "role": role, **({"workspace": req["workspace"]} if req.get("workspace") else {})}
+    atomic(os.path.join(work, ".git", "shard-prepared.json"), json.dumps(dict(owner, state="applying")).encode())
+    # Settings first: in a repository borrowing from a blobless cache, a file whose content only origin holds is fetched
+    # by the checkout itself (in one batch), not missed.
+    configure(work, req, cache.state().get("partial", False), root)
+    mark("init")
+    refs = "update refs/heads/%s %s\nupdate refs/worktree/shard/seed %s\nupdate refs/shard/seed %s\n" % (branch, m["seed"], m["seed"], m["seed"])
+    if m.get("branch") and m["branch"] != "refs/heads/" + branch:
+        refs += "update %s %s\n" % (m["branch"], m["head"])
+    if m.get("upstream"):
+        refs += "update %s %s\n" % (m["upstream"]["ref"], m["upstream"]["oid"])
+    lfs = ["-c", "filter.lfs.smudge=cat", "-c", "filter.lfs.process=", "-c", "filter.lfs.required=false"]
+    if mode == "inplace":
+        # Changed files are rewritten, files S lacks removed, ignored dependency folders kept. The index's stat data
+        # survives the image except inode and ctime, so only those are not compared.
+        git("-C", work, "-c", "core.checkStat=minimal", "-c", "core.trustctime=false", *lfs, "checkout", "-q", "-f", "-B", branch, m["seed"])
+        git("-C", work, "update-ref", "--stdin", data=refs.encode())
+    else:
+        git("-C", work, "update-ref", "--stdin", data=refs.encode())
+        git("-C", work, "symbolic-ref", "HEAD", "refs/heads/" + branch)
+        git("-C", work, *lfs, "read-tree", "--reset", "-u", m["seed"])
+    mark("checkout")
+    for path in m.get("remove", []):
+        target = os.path.join(work, path)
+        if os.path.lexists(target) and not os.path.isdir(target):
+            os.unlink(target)
+    extract_extras(m, work, extras_tar or cache.path("extras", m["extras_key"])) if m.get("extras") else None
+    mark("extras")
+    files = m.get("files", [])
+    write_excludes(work, files)
+    if mode == "inplace":
+        # A template's older copy goes; the captured one lands (now or when it arrives), never over the agent's file.
+        for f in files:
+            target = os.path.join(work, f["path"])
+            if os.path.lexists(target) and not os.path.isdir(target):
+                os.unlink(target)
+    if present is None:
+        present = {f["oid"] for f in files} & cache.media_index()
+    atomic(pending_path(work), json.dumps(files).encode())
+    dependencies = copy_dependencies(work, req.get("dependency_source"), m) if mode == "fresh" else 0
+    mark("dependencies")
+    if req.get("dissociate") == "now":
+        # A project template must not borrow from the cache: its own repository holds every object it names.
+        land_now(cache, work, [], sync=True)
+        git("-C", work, "repack", "-a", "-d", "-q")
+        os.unlink(os.path.join(work, ".git", "objects", "info", "alternates"))
+        mark("dissociate")
+    atomic(os.path.join(work, ".git", "shard-prepared.json"), json.dumps(dict(owner, state="ready", deps_hash=m.get("deps_hash"), mode=mode)).encode())
+    move_into_place(work, root)
+    landing = {f["oid"] for f in files} & present
+    return {"revision": revision, "mode": mode, "pending": len({f["oid"] for f in files} - present), "landing": len(landing),
+            "dependencies": dependencies, "timings": TIMINGS}
+
+
+def run_after(root, after):
+    """The caller's follow-up script (the seed's copy to the project hub), in the worktree, detached and quiet."""
+    with open(os.devnull, "rb") as null_in, open(os.devnull, "wb") as null_out:
+        subprocess.run(["sh", "-c", after["script"], "shard-seed", *after.get("args", [])], cwd=root, stdin=null_in, stdout=null_out, stderr=null_out)
+
+
+def land_now(cache, root, drop, sync):
+    if sync:
+        land(cache, root, drop)
+    else:
+        AFTER.append(lambda: land(cache, root, drop))
+
+
+def copy_dependencies(root, source, m):
+    """A conversation outside the template's checkout (a Codex Cloud worktree) copies its ignored dependency folders
+    when they were installed for the same dependency hash."""
+    if not source or source == root or not m.get("deps_hash") or not os.path.isdir(os.path.join(source, ".git")):
+        return 0
+    recorded = (marker(source) or {}).get("deps_hash")
+    if recorded and recorded != m["deps_hash"]:
+        return 0
+    copied = 0
+    listed = git("-C", source, "ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z", check=False).stdout.decode("utf-8", "surrogateescape").split("\0")
+    for rel in listed:
+        rel = rel.rstrip("/")
+        if not rel or os.path.basename(rel) not in DEPENDENCY_DIRS or ".git" in rel.split("/") or ".." in rel.split("/"):
+            continue
+        src, dst = os.path.join(source, rel), os.path.join(root, rel)
+        if os.path.lexists(dst) or not os.path.isdir(src) or os.path.islink(src):
+            continue
+        os.makedirs(os.path.dirname(dst), exist_ok=True)
+        subprocess.run(["cp", "-a", src, dst], check=True)
+        copied += 1
+    return copied
+
+
+def background(root, dissociate):
+    """At the lowest priority: refresh the index, and copy a borrowed object store into the worktree's own repository,
+    so pruning the cache can never reach a running conversation."""
+    script = 'cd "$1"; git -c core.hooksPath=/dev/null update-index -q --refresh >/dev/null 2>&1 || true\n'
+    if dissociate:
+        script += ('if [ -f .git/objects/info/alternates ]; then git -c core.hooksPath=/dev/null -c gc.auto=0 repack -a -d -q '
+                   '&& rm -f .git/objects/info/alternates; fi\n')
+    with open(os.devnull, "rb") as null_in, open(os.devnull, "wb") as null_out:
+        subprocess.Popen(["nice", "-n", "19", "sh", "-c", script, "shard-prepared", root], stdin=null_in, stdout=null_out, stderr=null_out,
+                         start_new_session=True)
+
+
+def land(cache, root, drop):
+    """Writes the media its repository holds where nothing is yet, so the agent's own file always stands. Files that
+    changed on the user's computer before they were sent never arrive as captured: `drop` gives them up."""
+    pending = read_json(pending_path(root), []) or []
+    pending = [f for f in pending if f["oid"] not in set(drop)]
+    have = present_in(root, sorted({f["oid"] for f in pending}))
+    arrived = [f for f in pending if f["oid"] in have]
+    landed = write_blobs(root, arrived, True)
+    rest = [f for f in pending if f["oid"] not in have]
+    atomic(pending_path(root), json.dumps(rest).encode())
+    return {"landed": len(landed), "kept": len(arrived) - len(landed), "pending": len(rest)}
+
+
+def materialize(cache, req):
+    """Media that arrived after the start (in the cache, or staged into the worktree's repository)."""
+    current = marker(req["root"])
+    if not current or current.get("revision") != req["revision"]:
+        raise Refused("owned", "this worktree holds another revision")
+    return land(cache, req["root"], req.get("drop", []))
+
+
+def pin(cache, req):
+    name = req["id"]
+    if not re.fullmatch(r"[a-zA-Z0-9_-]{8,128}", name):
+        raise ValueError("invalid preparation pin")
+    path = cache.path("pins", name + ".json")
+    if req.get("release"):
+        try:
+            os.unlink(path)
+        except FileNotFoundError:
+            pass
+    else:
+        if not HASH.fullmatch(req["revision"]):
+            raise ValueError("invalid prepared revision")
+        atomic(path, json.dumps({"revision": req["revision"], "at": int(time.time())}).encode())
+    return {"ok": True}
+
+
+def prune(cache, req):
+    """A bounded cache: a revision unused for a day (beyond the newest eight) leaves unless a pending start pins it, with
+    its refs, markers, archive and media; unconsumed upload parts and requests leave after a day. Ages come from the
+    files' contents and from when this pass first saw a file, never from file times (the project space can stamp them
+    years ahead). Git drops what no ref reaches after its two-week window, long after a conversation copied what it
+    borrowed."""
+    now = int(time.time())
+    listdir = lambda d: os.listdir(cache.path(d)) if os.path.isdir(cache.path(d)) else []
+    with cache.lock():
+        seen = read_json(cache.path("prune-seen.json"), {}) or {}
+        current = {}
+        for d in ("manifests", "extras", "chunks", "requests", "imports", "used", "published", "ready"):
+            for name in listdir(d):
+                if not os.path.islink(cache.path(d, name)) and ".tmp-" not in name:
+                    current[d + "/" + name] = seen.get(d + "/" + name, now)
+        seen = current
+        age = lambda key: now - seen.get(key, now)
+        pins = set()
+        for name in listdir("pins"):
+            p = read_json(cache.path("pins", name))
+            if p and now - p.get("at", 0) < 7 * 86400 and HASH.fullmatch(p.get("revision", "")):
+                pins.add(p["revision"])
+            else:
+                try:
+                    os.unlink(cache.path("pins", name))
+                except FileNotFoundError:
+                    pass
+
+        def last_used(rev):
+            used = read_json(cache.path("used", rev + ".json"))
+            return int(used["at"]) if used and "at" in used else now - age("manifests/" + rev + ".json")
+        manifests = sorted((n[:-5] for n in listdir("manifests") if n.endswith(".json") and HASH.fullmatch(n[:-5])), key=last_used, reverse=True)
+        keep, drop = set(), []
+        for at, rev in enumerate(manifests):
+            (keep.add(rev) if rev in pins or (at < KEEP_REVISIONS and now - last_used(rev) < REVISION_TTL) else drop.append(rev))
+        extras_kept, media_kept = set(), set()
+        for rev in keep:
+            try:
+                m = cache.manifest(rev)
+                if m.get("extras_key"):
+                    extras_kept.add(m["extras_key"])
+                media_kept |= {f["oid"] for f in m.get("files", [])}
+            except (ValueError, FileNotFoundError):
+                pass
+        if os.path.isfile(os.path.join(cache.repo, "HEAD")) and drop:
+            cache.git("update-ref", "--stdin", check=False, data="".join(
+                "delete refs/shard/%s/%s\n" % (kind, r) for r in drop for kind in ("prepared", "upstream", "content")).encode())
+            cache.git("pack-refs", "--all")
+            refresh_state(cache)
+        removed = 0
+        for rev in drop:
+            for d in ("ready", "published", "manifests", "imports", "used"):
+                try:
+                    os.unlink(cache.path(d, rev + ".json"))
+                    removed += 1
+                except FileNotFoundError:
+                    pass
+        index = cache.media_index()
+        if drop and index - media_kept:
+            atomic(cache.path("media-index"), ("\n".join(sorted(index & media_kept)) + "\n").encode())
+        for d, keepers in (("extras", extras_kept), ("chunks", set()), ("requests", set())):
+            for name in listdir(d):
+                if name not in keepers and age(d + "/" + name) > REVISION_TTL and not os.path.islink(cache.path(d, name)):
+                    os.unlink(cache.path(d, name))
+                    removed += 1
+        atomic(cache.path("prune-seen.json"), json.dumps({k: v for k, v in seen.items() if os.path.lexists(cache.path(k))}).encode())
+        if os.path.isfile(os.path.join(cache.repo, "HEAD")):
+            packs = [n for n in os.listdir(os.path.join(cache.repo, "objects", "pack")) if n.endswith(".pack")]
+            if len(packs) > 8:
+                # Many packs slow every lookup over NFS. Most passes fold only the small recent packs (a geometric
+                # progression keeps the large ones as they are); about weekly everything is rewritten, with what no
+                # ref reaches in a cruft pack that Git drops after its usual two weeks.
+                full = read_json(cache.path("repacked.json"), {}) or {}
+                if now - int(full.get("at", 0)) > FULL_REPACK_EVERY:
+                    cache.git("-c", "pack.threads=2", "repack", "-a", "-d", "-q", "--cruft", "--cruft-expiration=2.weeks.ago", check=False)
+                    atomic(cache.path("repacked.json"), json.dumps({"at": now}).encode())
+                else:
+                    cache.git("-c", "pack.threads=2", "repack", "-d", "-q", "--geometric=2", check=False)
+        return {"revisions": len(keep), "dropped": len(drop), "removed_files": removed}
+
+
+def maybe_prune(cache):
+    """A start's pass, after its reply, at most every PRUNE_EVERY: a project nobody selected stays bounded too. Its age
+    is in the file, never its time (the project space can stamp file times years ahead)."""
+    now = int(time.time())
+    last = read_json(cache.path("pruned.json"), {}) or {}
+    if now - int(last.get("at", 0)) < PRUNE_EVERY or not os.path.isdir(cache.root):
+        return
+    atomic(cache.path("pruned.json"), json.dumps({"at": now}).encode())
+    os.nice(10)
+    prune(cache, {})
+
+
+ACTIONS = {"probe": probe, "origin": origin, "import": import_, "stage": stage, "apply": apply, "materialize": materialize, "pin": pin, "prune": prune}
+
+
+def after_failed(action):
+    """Work after the reply has no one to report to: its failures go to a small log in the VM (the next start repeats
+    what failed: an import the cache lacks is sent again)."""
+    import traceback
+    try:
+        log = os.path.join(os.environ.get("HOME") or "/home/user", ".cache", "shardflux", "prepared-after.log")
+        os.makedirs(os.path.dirname(log), exist_ok=True)
+        if os.path.isfile(log) and os.path.getsize(log) > 256 << 10:
+            os.replace(log, log + ".1")
+        with open(log, "a") as f:
+            f.write("%s %s: %s\n" % (time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), action, traceback.format_exc()[-4000:]))
+    except OSError:
+        pass
+
+
+def main():
+    action, root = sys.argv[1], sys.argv[2]
+    if action not in ACTIONS or not root.startswith("/"):
+        raise ValueError("unknown prepared repository operation")
+    req = json.loads(sys.stdin.buffer.read() or b"{}")
+    if req.get("request_file"):
+        path = req["request_file"]
+        if not path.startswith(root + "/requests/") or ".." in path.split("/"):
+            raise ValueError("invalid request file")
+        with open(path, "rb") as f:
+            req = json.loads(f.read())
+        os.unlink(path)
+    cache = Cache(root)
+    try:
+        result = ACTIONS[action](cache, req)
+    except Refused as r:
+        result = {"error": r.reason, "message": str(r), **r.extra}
+    # Tests (background false) run the follow-up work before the reply, so they observe its effects.
+    if AFTER and (req.get("background") is False or (req.get("apply") or {}).get("background") is False):
+        for job in AFTER:
+            job()
+        AFTER.clear()
+    result.setdefault("timings", TIMINGS)
+    sys.stdout.write(json.dumps(result) + "\n")
+    sys.stdout.flush()
+    if AFTER and os.fork() == 0:
+        # The reply is out; this child outlives the exec (its own session, no pipe of the exec held open).
+        os.setsid()
+        null = os.open(os.devnull, os.O_RDWR)
+        for fd in (0, 1, 2):
+            os.dup2(null, fd)
+        try:
+            for job in AFTER:
+                try:
+                    job()
+                except Exception:
+                    after_failed(action)
+        finally:
+            os._exit(0)
+
+
+if __name__ == "__main__":
+    main()