@shardflux/cli 0.20.0 → 0.22.0

package/dist/prepared-repository.js

@@ -0,0 +1,860 @@
+/**
+ * Prepared projects: a project's working state reaches a VM as Git packs, once (0.22.0+).
+ *
+ * A start snapshots the worktree as the twin commit S (worktree-twin.ts: HEAD plus the dirty and untracked files).
+ * Everything S needs that the project cache lacks travels as Git packs: a thin pack of history and small files (normal
+ * Git deltas and compression) and the large blobs apart. Large untracked media stay out of S: their bytes travel once,
+ * as stored Git blobs, and land in the worktree as files its Git ignores (as the user's Git does), possibly after the
+ * first prompt. What Git must not hold (.env files, filter output) travels as a small tar. The cache
+ * (/shardflux/project/prepared/<remote hash>, shared by the project's VMs) keeps all of it, so a later start sends only
+ * what changed since a revision it holds, and a start from a prepared project sends nothing.
+ *
+ * Packs travel as 8 MiB binary parts, several at once, each named by its SHA-256 and acknowledged with the VM's own
+ * hash of what it stored: an interrupted upload resumes with the missing parts.
+ *
+ * The VM half is prepared-repository.py: one exec per step (probe, origin, import, apply, materialize, prune).
+ */
+import { spawn } from 'node:child_process';
+import { createHash, randomUUID } from 'node:crypto';
+import { constants, readFileSync } from 'node:fs';
+import { lstat, mkdir, open, readFile, readdir, readlink, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { createDeflate, deflateRawSync } from 'node:zlib';
+import { tarEnd, tarHeader, tarPadding } from '@shardflux/sdk';
+import { localChanges } from "./agent-repository.js";
+import { localEntry, projectFilePath, projectGit } from "./project-sync-manifest.js";
+import { SYNC_CHUNK_BYTES, writeSyncBlob } from "./project-sync-transfer.js";
+import { inspectWorktree, worktreeId } from "./worktree-twin.js";
+import { dependencyHash } from "./project-template-setup.js";
+import { withTargetLock } from "./target-lock.js";
+import { blobId, largeUntrackedMedia } from "./large-media.js";
+/** One upload part. */
+export const PART_BYTES = SYNC_CHUNK_BYTES;
+/** Blobs of S from this size on leave the thin pack: stored when incompressible, compressed apart otherwise. */
+export const LARGE_BLOB_BYTES = 1 << 20;
+/** Large media above this follow the first prompt when the caller allows it. */
+export const DEFER_BYTES = 32 << 20;
+/** Parts in flight at once, and their bytes. */
+export const UPLOAD_WINDOW = 4;
+export const UPLOAD_BUDGET_BYTES = 32 << 20;
+/** The history the laptop sends before the cache fetches it from origin instead. */
+export const ORIGIN_FIRST_BYTES = 32 << 20;
+/** The history the laptop sends before the cache takes the VM's own fetch of the branch (its boot prefetch) instead. */
+export const PREFETCH_ADOPT_BYTES = 4 << 20;
+/** The local spool of generated packs (for resuming), pruned past this or a day. */
+export const SPOOL_BYTES = 4 << 30;
+const STDIN_BYTES = 768 << 10;
+const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
+let helperSource;
+const helper = () => helperSource ??= readFileSync(new URL('./prepared-repository.py', import.meta.url), 'utf8');
+export function preparedCachePath(remote) { return `/shardflux/project/prepared/${sha256(remote)}`; }
+export const preparedDirectory = (stateDir, folder) => join(stateDir, 'prepared', worktreeId(folder));
+/** The manifest's identity bytes: sorted keys, no spaces, UTF-8 (prepared-repository.py `canonical`). */
+export function canonicalJson(value) {
+    const sort = (v) => Array.isArray(v) ? v.map(sort)
+        : v && typeof v === 'object' ? Object.fromEntries(Object.keys(v).sort().filter(k => v[k] !== undefined).map(k => [k, sort(v[k])])) : v;
+    return JSON.stringify(sort(value));
+}
+export const manifestRevision = (m) => sha256(canonicalJson(m));
+/** A refusal the VM half reports by reason (missing or corrupt parts, an owned worktree, ...). */
+export class PreparedRefusal extends Error {
+    reason;
+    chunks;
+    constructor(reason, message, chunks = []) { super(message); this.reason = reason; this.chunks = chunks; }
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Capture
+async function optional(folder, args, signal) {
+    try {
+        return (await projectGit(folder, args, signal)).trim();
+    }
+    catch (error) {
+        if (signal?.aborted)
+            throw error;
+        return '';
+    }
+}
+/** Git's output of any size (projectGit buffers at most 4 MiB). */
+function gitStream(folder, args, signal, input) {
+    return new Promise((resolveOut, reject) => {
+        const child = spawn('git', ['-C', folder, '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], { stdio: ['pipe', 'pipe', 'pipe'], signal, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0', GIT_TERMINAL_PROMPT: '0', LC_ALL: 'C' } });
+        const out = [];
+        let err = '';
+        child.stdout.on('data', (b) => out.push(b));
+        child.stderr.on('data', (b) => { err = (err + String(b)).slice(-2000); });
+        child.stdin.on('error', () => undefined);
+        child.once('error', reject);
+        child.once('close', code => code === 0 ? resolveOut(Buffer.concat(out)) : reject(new Error(`git ${args[0]} failed: ${err.trim()}`)));
+        child.stdin.end(input ?? '');
+    });
+}
+async function treeEntries(folder, commit, signal) {
+    const map = new Map();
+    for (const rec of (await gitStream(folder, ['ls-tree', '-r', '-z', '--full-tree', commit], signal)).toString('utf8').split('\0')) {
+        if (!rec)
+            continue;
+        const tab = rec.indexOf('\t'), [mode, , oid] = rec.slice(0, tab).split(' ');
+        map.set(rec.slice(tab + 1), { mode: mode, oid: oid });
+    }
+    return map;
+}
+async function rawBlobId(folder, path, format) {
+    const target = await projectFilePath(folder, path), st = await lstat(target);
+    if (st.isSymbolicLink()) {
+        const link = Buffer.from(await readlink(target));
+        return createHash(format).update(`blob ${link.length}\0`).update(link).digest('hex');
+    }
+    return blobId(target, format, st.size);
+}
+/**
+ * The changed paths whose bytes S does not carry exactly: not in S (.env files, which Git ignores and we always bring;
+ * every change when S is HEAD), or in S through a filter or line-ending conversion that changes them (Git LFS, CRLF).
+ */
+async function extraPaths(folder, tree, copy, seedIsHead, format, signal) {
+    if (!copy.length)
+        return [];
+    const autocrlf = ['true', 'input'].includes((await optional(folder, ['config', '--get', 'core.autocrlf'], signal)).toLowerCase());
+    const convert = new Set(), inTree = copy.filter(path => tree.has(path));
+    for (let at = 0; at < inTree.length; at += 4096) {
+        const out = (await gitStream(folder, ['check-attr', '-z', '--stdin', 'filter', 'text', 'eol', 'ident', 'working-tree-encoding'], signal, inTree.slice(at, at + 4096).join('\0') + '\0')).toString('utf8').split('\0');
+        for (let i = 0; i + 2 < out.length; i += 3)
+            if (!['unspecified', 'unset'].includes(out[i + 2]))
+                convert.add(out[i]);
+    }
+    const extras = [];
+    for (const path of copy) {
+        const entry = tree.get(path);
+        if (!entry) {
+            extras.push(path);
+            continue;
+        }
+        if (entry.mode === '160000' || (!seedIsHead && !autocrlf && !convert.has(path)))
+            continue;
+        try {
+            if (await rawBlobId(folder, path, format) !== entry.oid)
+                extras.push(path);
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT')
+                throw error;
+        }
+    }
+    return extras;
+}
+async function writeExact(file, data) {
+    for (let at = 0; at < data.length;)
+        at += (await file.write(data, at, data.length - at)).bytesWritten;
+}
+/** A deterministic tar of the extras (sorted, zero times and owners): its SHA-256 and the entries it holds. */
+async function extrasArchive(folder, directory, paths, signal) {
+    const entries = [];
+    if (!paths.length)
+        return { entries };
+    const temp = join(directory, `extras-${randomUUID()}.tmp`), hash = createHash('sha256');
+    const file = await open(temp, 'wx', 0o600);
+    let offset = 0;
+    const put = async (data) => { hash.update(data); await writeExact(file, data); offset += data.length; };
+    try {
+        for (const path of [...paths].sort()) {
+            signal?.throwIfAborted();
+            const entry = await localEntry(folder, path, signal);
+            if (!entry || !('sha256' in entry))
+                continue;
+            if (entry.symlink) {
+                await put(tarHeader({ path, type: 'symlink', mode: 0o777, size: 0, linkname: entry.link_target }));
+            }
+            else {
+                const source = await open(await projectFilePath(folder, path), constants.O_RDONLY | constants.O_NOFOLLOW);
+                try {
+                    await put(tarHeader({ path, type: 'file', mode: entry.mode, size: entry.size }));
+                    const check = createHash('sha256'), buffer = Buffer.allocUnsafe(1 << 20);
+                    for (let read = 0; read < entry.size;) {
+                        const { bytesRead } = await source.read(buffer, 0, Math.min(buffer.length, entry.size - read), read);
+                        if (!bytesRead)
+                            throw new Error(`${path} changed while it was captured; start again.`);
+                        check.update(buffer.subarray(0, bytesRead));
+                        await put(buffer.subarray(0, bytesRead));
+                        read += bytesRead;
+                    }
+                    if (check.digest('hex') !== entry.sha256)
+                        throw new Error(`${path} changed while it was captured; start again.`);
+                    await put(tarPadding(entry.size));
+                }
+                finally {
+                    await source.close();
+                }
+            }
+            entries.push({ path, sha256: entry.sha256, size: entry.symlink ? 0 : entry.size, mode: entry.mode, symlink: entry.symlink, link_target: entry.link_target });
+        }
+        await put(tarEnd(offset));
+        await file.sync();
+    }
+    catch (error) {
+        await file.close();
+        await rm(temp, { force: true });
+        throw error;
+    }
+    await file.close();
+    const key = hash.digest('hex');
+    await rename(temp, join(directory, `extras-${key}.tar`));
+    return { key, entries };
+}
+/** Each media file's Git blob id, hashed once per version (size, times, inode). */
+async function mediaFiles(folder, directory, outside, format) {
+    const cachePath = join(directory, `blob-ids-${format}.json`);
+    let ids = {};
+    try {
+        ids = JSON.parse(await readFile(cachePath, 'utf8'));
+    }
+    catch { /* first capture */ }
+    const next = {}, files = new Array(outside.length);
+    let at = 0;
+    await Promise.all(Array.from({ length: Math.min(8, outside.length) }, async () => {
+        for (let i = at++; i < outside.length; i = at++) {
+            const f = outside[i], oid = ids[f.path]?.meta === f.meta ? ids[f.path].oid : await blobId(join(folder, f.path), format, f.size);
+            next[f.path] = { meta: f.meta, oid };
+            files[i] = { path: f.path, oid, size: f.size, mode: f.mode };
+        }
+    }));
+    const temp = `${cachePath}.${process.pid}.tmp`;
+    await writeFile(temp, JSON.stringify(next), { mode: 0o600 });
+    await rename(temp, cachePath);
+    return files;
+}
+/**
+ * The prepared revision of a worktree's state: S (the twin seed, already made), its large media and its extras. S's
+ * objects stay in the local repository under refs/shard/prepared/<revision> until the start that uses it is done, so a
+ * start accepted before a deadline resumes later with exactly the state it began from, whatever the user edits.
+ */
+export async function capturePreparedSnapshot(stateDir, folder, seed, origin, signal, options = {}) {
+    // A remote on this computer's disk is out of the VM's reach; the history then comes from here.
+    const reachable = options.reachable ?? ((url) => !url.startsWith('file:') && !url.startsWith('/'));
+    const directory = preparedDirectory(stateDir, folder);
+    await mkdir(join(directory, 'spool'), { recursive: true, mode: 0o700 });
+    const info = await inspectWorktree(folder);
+    if (!info.head)
+        throw new Error('The project has no commit to prepare.');
+    const format = (await projectGit(folder, ['rev-parse', '--show-object-format'], signal)).trim() === 'sha256' ? 'sha256' : 'sha1';
+    let upstream;
+    if (info.branch) {
+        const ref = await optional(folder, ['rev-parse', '--symbolic-full-name', `${info.branch.slice(11)}@{upstream}`], signal);
+        const oid = ref.startsWith('refs/remotes/') ? await optional(folder, ['rev-parse', '--verify', '-q', `${ref}^{commit}`], signal) : '';
+        if (oid)
+            upstream = { ref, oid };
+    }
+    const outside = seed.outside ?? await largeUntrackedMedia(info.root, stateDir, worktreeId(folder), signal);
+    let shallow = [];
+    if (await optional(folder, ['rev-parse', '--is-shallow-repository'], signal) === 'true') {
+        const file = await optional(folder, ['rev-parse', '--git-path', 'shallow'], signal);
+        shallow = file ? (await readFile(resolve(folder, file), 'utf8').catch(() => '')).split('\n').filter(line => /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(line)).sort() : [];
+    }
+    const outsidePaths = new Set(outside.map(f => f.path));
+    const [changes, tree] = await Promise.all([localChanges(folder, signal), treeEntries(folder, seed.commit, signal)]);
+    const copy = changes.copy.filter(path => !outsidePaths.has(path));
+    const [extras, files] = await Promise.all([
+        extraPaths(folder, tree, copy, seed.commit === info.head, format, signal).then(paths => extrasArchive(folder, directory, paths, signal)),
+        mediaFiles(folder, directory, outside, format),
+    ]);
+    const manifest = { version: 2, format, head: info.head, seed: seed.commit, ...(info.branch ? { branch: info.branch } : {}), ...(upstream ? { upstream } : {}),
+        origin: origin && reachable(origin) ? origin : null, repository: origin ?? `file://${folder}`, worktree: worktreeId(folder),
+        deps_hash: options.depsHash ?? dependencyHash(folder), extras: extras.entries, ...(extras.key ? { extras_key: extras.key } : {}), files,
+        remove: changes.remove.filter(path => tree.has(path)), ...(shallow.length ? { shallow } : {}) };
+    const revision = manifestRevision(manifest);
+    await projectGit(folder, ['update-ref', `refs/shard/prepared/${revision}`, seed.commit], signal);
+    await writeFile(join(directory, `manifest-${revision}.json`), canonicalJson(manifest), { mode: 0o600 });
+    // Which checkout pins these revisions (refs/shard/prepared/*): local pruning releases a revision it drops.
+    await writeFile(join(directory, 'folder'), folder, { mode: 0o600 });
+    return { revision, manifest, directory };
+}
+/** The local pin of a revision's objects, once its start no longer needs them. */
+export async function releaseLocalSnapshot(folder, revision) {
+    if (!/^[a-f0-9]{64}$/.test(revision))
+        return;
+    await projectGit(folder, ['update-ref', '-d', `refs/shard/prepared/${revision}`]).catch(() => undefined);
+}
+export async function readPreparedSnapshot(stateDir, folder, revision) {
+    const directory = preparedDirectory(stateDir, folder);
+    const manifest = JSON.parse(await readFile(join(directory, `manifest-${revision}.json`), 'utf8'));
+    if (manifestRevision(manifest) !== revision)
+        throw new Error('The saved project revision changed on disk.');
+    return { revision, manifest, directory };
+}
+const noStats = () => ({ puts: 0, execs: 0, uploaded: 0, reused: 0, inFlight: 0, maxInFlight: 0 });
+async function invoke(cell, action, cache, req, stats, signal) {
+    let body = JSON.stringify(req);
+    if (Buffer.byteLength(body) > STDIN_BYTES) {
+        const file = `${cache}/requests/${randomUUID()}.json`;
+        await cell.files.mkdir(`${cache}/requests`, { parents: true });
+        await writeSyncBlob(cell, file, Buffer.from(body));
+        stats.puts++;
+        body = JSON.stringify({ request_file: file });
+    }
+    stats.execs++;
+    const r = await cell.exec.run(['python3', '-c', helper(), action, cache], { stdin: body, timeoutMs: 1_800_000, maxOutputBytes: 8 << 20, ...(signal ? { signal } : {}) });
+    const line = r.stdout.trim().split('\n').at(-1) ?? '';
+    if (r.exitCode !== 0 || !line.startsWith('{'))
+        throw new Error(`Project ${action} failed: ${(r.stderr || r.stdout).trim().slice(-1500)}`);
+    const result = JSON.parse(line);
+    if (result.error)
+        throw new PreparedRefusal(result.error, result.message ?? result.error, result.chunks ?? []);
+    return result;
+}
+/** Bounded concurrent part uploads: a window of parts and of bytes, every part acknowledged by the VM's own SHA-256. */
+class Uploader {
+    #running = new Set();
+    #bytes = 0;
+    #failure;
+    cell;
+    parts;
+    stats;
+    signal;
+    constructor(cell, parts, stats, signal) { this.cell = cell; this.parts = parts; this.stats = stats; this.signal = signal; }
+    async put(key, data) {
+        while (this.#running.size >= UPLOAD_WINDOW || (this.#running.size && this.#bytes + data.length > UPLOAD_BUDGET_BYTES))
+            await Promise.race(this.#running);
+        if (this.#failure)
+            throw this.#failure;
+        this.signal?.throwIfAborted();
+        this.#bytes += data.length;
+        this.stats.inFlight += data.length;
+        this.stats.maxInFlight = Math.max(this.stats.maxInFlight, this.stats.inFlight);
+        const task = (async () => {
+            try {
+                const written = await this.cell.files.write(`${this.parts}/${key}`, data, { mode: '0600', createParents: true });
+                this.stats.puts++;
+                if (written && typeof written === 'object' && 'sha256' in written && written.sha256 !== key)
+                    throw new Error('An uploaded part was stored with different bytes.');
+                this.stats.uploaded += data.length;
+            }
+            catch (error) {
+                this.#failure ??= error instanceof Error ? error : new Error(String(error));
+            }
+            finally {
+                this.#bytes -= data.length;
+                this.stats.inFlight -= data.length;
+            }
+        })();
+        this.#running.add(task);
+        void task.finally(() => this.#running.delete(task));
+    }
+    async drain() {
+        await Promise.all(this.#running);
+        if (this.#failure)
+            throw this.#failure;
+    }
+}
+/** Which of these commits this repository also has: what the packs may leave out. */
+async function localCommits(folder, oids, signal) {
+    if (!oids.length)
+        return [];
+    const out = (await gitStream(folder, ['cat-file', '--batch-check=%(objectname) %(objecttype)'], signal, oids.join('\n') + '\n')).toString('utf8').split('\n');
+    return out.filter(line => line.endsWith(' commit')).map(line => line.split(' ')[0]);
+}
+/** Large blobs whose first 64 KiB barely compress are stored as they are: zlib on media costs seconds and saves nothing. */
+async function incompressible(folder, path) {
+    let file;
+    try {
+        file = await open(await projectFilePath(folder, path), constants.O_RDONLY | constants.O_NOFOLLOW);
+        const sample = Buffer.allocUnsafe(64 << 10), { bytesRead } = await file.read(sample, 0, sample.length, 0);
+        return bytesRead > 0 && deflateRawSync(sample.subarray(0, bytesRead), { level: 1 }).length >= bytesRead * 0.92;
+    }
+    catch {
+        return false;
+    }
+    finally {
+        await file?.close();
+    }
+}
+async function planTransfer(folder, snapshot, known, missingFiles, history, signal) {
+    const m = snapshot.manifest, tops = [m.seed, m.head, ...(m.upstream ? [m.upstream.oid] : [])];
+    const negatives = [...new Set(known)].sort().map(oid => `^${oid}`), revs = [...tops, ...negatives].join('\n') + '\n';
+    const identity = (kind, content) => sha256(`${kind}\0${snapshot.revision}\0${negatives.join(',')}\0${content}`);
+    const packs = [];
+    if (history) {
+        // What the cache lacks, and the same walk without large blobs: the difference is the large blobs to send. (A
+        // filtered walk's omitted list also names blobs the cache already has.) Aggressive edges: the cache's snapshots are
+        // siblings of S, not ancestors, and a plain walk would send again everything their trees hold.
+        const list = async (filter) => new Set((await gitStream(folder, ['rev-list', '--objects-edge-aggressive', '--no-object-names', ...filter, '--stdin'], signal, revs))
+            .toString('utf8').split('\n').filter(line => line && !line.startsWith('-')));
+        const needed = await list([]), small = await list([`--filter=blob:limit=${LARGE_BLOB_BYTES}`]);
+        const large = [...needed].filter(oid => !small.has(oid));
+        // A large blob of the working state (a tracked asset edited) has a path to sample; history's large blobs keep Git's
+        // compression, usually reused from its packs as they are.
+        const paths = new Map();
+        if (large.length) {
+            const wanted = new Set(large);
+            for (const [path, entry] of await treeEntries(folder, m.seed, signal))
+                if (wanted.has(entry.oid) && !paths.has(entry.oid))
+                    paths.set(entry.oid, path);
+        }
+        const stored = [], compressed = [];
+        for (const oid of large) {
+            const path = paths.get(oid);
+            (path && await incompressible(folder, path) ? stored : compressed).push(oid);
+        }
+        if (small.size)
+            packs.push({ key: identity('history', revs), kind: 'history', thin: true, stdin: revs,
+                // --shallow: aggressive edges for the same reason (a one-line edit sent a 300-file folder again without it).
+                args: ['pack-objects', '--revs', '--thin', '--shallow', '--stdout', '--delta-base-offset', '-q', `--filter=blob:limit=${LARGE_BLOB_BYTES}`] });
+        if (compressed.length) {
+            const stdin = compressed.sort().join('\n') + '\n';
+            packs.push({ key: identity('content', stdin), kind: 'content', thin: false, stdin, args: ['pack-objects', '--stdout', '--delta-base-offset', '-q'] });
+        }
+        if (stored.length) {
+            const stdin = stored.sort().join('\n') + '\n';
+            packs.push({ key: identity('large', stdin), kind: 'large', thin: false, stdin, args: ['-c', 'pack.compression=0', 'pack-objects', '--stdout', '--window=0', '-q'] });
+        }
+    }
+    const wanted = new Set(missingFiles), seen = new Set(), files = [];
+    for (const f of m.files)
+        if (wanted.has(f.oid) && !seen.has(f.oid)) {
+            seen.add(f.oid);
+            files.push(f);
+        }
+    if (files.length)
+        packs.push({ key: identity('media', files.map(f => f.oid).join('\n')), kind: 'media', thin: false, files });
+    return { revision: snapshot.revision, packs };
+}
+/** A pack's object header: type and size, 4 then 7 bits at a time. */
+function objectHeader(type, size) {
+    const bytes = [];
+    let c = (type << 4) | (size % 16), rest = Math.floor(size / 16);
+    while (rest > 0) {
+        bytes.push(c | 0x80);
+        c = rest % 128;
+        rest = Math.floor(rest / 128);
+    }
+    bytes.push(c);
+    return Buffer.from(bytes);
+}
+/**
+ * The media pack, written here from the files themselves: each blob stored (zlib level 0), no copy in the local
+ * repository. A file that changed since the capture still yields one object of its captured size, so the pack stays
+ * valid; its captured blob id never arrives and the file is reported in `changed`.
+ */
+async function* mediaPack(folder, files, format, changed) {
+    const trailer = createHash(format);
+    const emit = (b) => { trailer.update(b); return b; };
+    const header = Buffer.alloc(12);
+    header.write('PACK', 0, 'latin1');
+    header.writeUInt32BE(2, 4);
+    header.writeUInt32BE(files.length, 8);
+    yield emit(header);
+    for (const f of files) {
+        yield emit(objectHeader(3, f.size));
+        const id = createHash(format).update(`blob ${f.size}\0`), deflate = createDeflate({ level: 0 });
+        let file;
+        try {
+            file = await open(await projectFilePath(folder, f.path), constants.O_RDONLY | constants.O_NOFOLLOW);
+        }
+        catch { /* gone: zeros below */ }
+        const feed = (async () => {
+            const buffer = Buffer.allocUnsafe(1 << 20);
+            for (let at = 0; at < f.size;) {
+                const want = Math.min(buffer.length, f.size - at);
+                let got = file ? (await file.read(buffer, 0, want, at)).bytesRead : 0;
+                if (!got) {
+                    buffer.fill(0, 0, want);
+                    got = want;
+                }
+                const piece = Buffer.from(buffer.subarray(0, got));
+                id.update(piece);
+                at += got;
+                if (!deflate.write(piece))
+                    await new Promise(r => deflate.once('drain', r));
+            }
+            deflate.end();
+        })();
+        try {
+            for await (const z of deflate)
+                yield emit(z);
+            await feed;
+        }
+        finally {
+            await file?.close();
+        }
+        if (id.digest('hex') !== f.oid)
+            changed.push(f.path);
+    }
+    yield trailer.digest();
+}
+function packSource(folder, pack, format, changed, signal) {
+    if (pack.files)
+        return { chunks: mediaPack(folder, pack.files, format, changed), done: () => Promise.resolve() };
+    const child = spawn('git', ['-C', folder, '-c', 'core.hooksPath=/dev/null', ...pack.args], { signal, stdio: ['pipe', 'pipe', 'pipe'],
+        env: { ...process.env, GIT_OPTIONAL_LOCKS: '0', GIT_TERMINAL_PROMPT: '0' } });
+    let stderr = '';
+    child.stderr.on('data', (b) => { stderr = (stderr + String(b)).slice(-4000); });
+    const exited = new Promise((resolveExit, reject) => { child.once('error', reject); child.once('close', code => resolveExit(code ?? 1)); });
+    child.stdin.on('error', () => undefined);
+    child.stdin.end(pack.stdin);
+    return { chunks: child.stdout, done: async () => { if (await exited !== 0)
+            throw new Error(`Git could not pack the project: ${stderr.trim()}`); } };
+}
+/**
+ * Streams a pack into 8 MiB parts while the uploader sends them. Git's packs are spooled for a resumed transfer (their
+ * bytes may differ when generated again); the media pack is not: the same files always give the same bytes, so a
+ * resumed transfer generates it again and skips the parts the VM already holds.
+ */
+async function sendPack(folder, snapshot, pack, present, upload, stats, signal) {
+    const spool = join(snapshot.directory, 'spool'), recordPath = join(spool, `${pack.key}.json`), spooled = !pack.files;
+    let record;
+    try {
+        record = JSON.parse(await readFile(recordPath, 'utf8'));
+    }
+    catch { /* not generated yet */ }
+    if (record && (await Promise.all(record.chunks.map(key => stat(join(spool, key)).then(() => true, () => false)))).every(Boolean)) {
+        for (const key of record.chunks) {
+            if (present.has(key)) {
+                stats.reused += (await stat(join(spool, key))).size;
+                continue;
+            }
+            await upload.put(key, await readFile(join(spool, key)));
+            present.add(key);
+        }
+        return record;
+    }
+    const changed = [], source = packSource(folder, pack, snapshot.manifest.format, changed, signal);
+    const chunks = [];
+    let size = 0, held = [], heldBytes = 0;
+    const emit = async (data) => {
+        const key = sha256(data), target = join(spool, key);
+        if (spooled)
+            try {
+                await stat(target);
+            }
+            catch {
+                const temp = `${target}.${randomUUID()}.tmp`;
+                await writeFile(temp, data, { mode: 0o600, flag: 'wx' });
+                await rename(temp, target);
+            }
+        chunks.push(key);
+        size += data.length;
+        if (present.has(key))
+            stats.reused += data.length;
+        else {
+            await upload.put(key, data);
+            present.add(key);
+        }
+    };
+    for await (const raw of source.chunks) {
+        held.push(raw);
+        heldBytes += raw.length;
+        while (heldBytes >= PART_BYTES) {
+            const all = Buffer.concat(held, heldBytes);
+            await emit(all.subarray(0, PART_BYTES));
+            held = [all.subarray(PART_BYTES)];
+            heldBytes = all.length - PART_BYTES;
+        }
+    }
+    if (heldBytes)
+        await emit(Buffer.concat(held, heldBytes));
+    await source.done();
+    record = { chunks, size, ...(changed.length ? { changed } : {}) };
+    if (spooled)
+        await writeFile(recordPath, JSON.stringify(record), { mode: 0o600 });
+    // Media edited while they were sent: a later call learns of them even after the cache took this pack.
+    else if (changed.length)
+        await writeFile(changedPath(snapshot.directory, pack.key), JSON.stringify(changed), { mode: 0o600 });
+    return record;
+}
+const changedPath = (directory, key) => join(directory, 'spool', `${key}.changed.json`);
+async function sendFile(path, present, upload, stats) {
+    const file = await open(path, constants.O_RDONLY), keys = [];
+    try {
+        const size = (await file.stat()).size;
+        for (let at = 0; at < size;) {
+            const data = Buffer.allocUnsafe(Math.min(PART_BYTES, size - at));
+            for (let read = 0; read < data.length;) {
+                const { bytesRead } = await file.read(data, read, data.length - read, at + read);
+                if (!bytesRead)
+                    throw new Error('The project files archive changed.');
+                read += bytesRead;
+            }
+            const key = sha256(data);
+            keys.push(key);
+            if (present.has(key))
+                stats.reused += data.length;
+            else {
+                await upload.put(key, data);
+                present.add(key);
+            }
+            at += data.length;
+        }
+    }
+    finally {
+        await file.close();
+    }
+    return keys;
+}
+/** Where a conversation start's parts go: the VM's own disk, indexed there; the cache receives them after the reply. */
+export const PARTS_ROOT = '/home/user/.cache/shardflux/prepared-parts';
+/**
+ * Sends the revision, only what the project cache lacks. Without `apply` it goes into the cache (a preparation). With
+ * `apply` (a conversation start) it is checked out at that target: in the probe's own exec when the cache already
+ * holds the revision (a warm start is one exec), otherwise from parts uploaded to the VM's disk and indexed there, with
+ * no Git work on the shared cache before the reply; the cache receives them after it. With `defer`, media beyond
+ * DEFER_BYTES wait for a later call: the result names them in `pending`, and the worktree starts without them.
+ */
+export async function transferPreparedSnapshot(cell, folder, cache, snapshot, options = {}) {
+    const { signal } = options, m = snapshot.manifest, started = performance.now(), timings = {};
+    const markTime = (name) => { timings[name] = Math.round(performance.now() - started); };
+    const stats = noStats(), sizes = {}, changed = [];
+    let originFetched = false;
+    const result = (missing) => {
+        const lost = new Set(changed), absent = new Set(missing), pending = m.files.filter(f => absent.has(f.oid) && !lost.has(f.path)).map(f => f.path);
+        return { revision: snapshot.revision, complete: !pending.length, pending, history_bytes: sizes.history ?? 0, large_bytes: (sizes.large ?? 0) + (sizes.content ?? 0),
+            media_bytes: sizes.media ?? 0, extras_bytes: sizes.extras ?? 0, uploaded_bytes: stats.uploaded, reused_bytes: stats.reused, puts: stats.puts, execs: stats.execs,
+            max_in_flight_bytes: stats.maxInFlight, origin_fetched: originFetched, changed, timings };
+    };
+    const staging = options.apply !== undefined, partsDir = staging ? options.partsDir ?? `${PARTS_ROOT}/${sha256(cache).slice(0, 16)}` : `${cache}/chunks`;
+    let probe = await invoke(cell, 'probe', cache, { manifest: m, ...(options.pin ? { pin: options.pin } : {}),
+        ...(options.apply ? { apply: applyRequest(options.apply) } : {}), ...(staging ? { parts_dir: partsDir } : {}) }, stats, signal);
+    markTime('probe');
+    if (probe.revision !== snapshot.revision)
+        throw new Error('The VM computed a different project revision.');
+    let applied = probe.applied;
+    if (probe.ready)
+        return { ...result([]), ...(applied ? { applied } : {}) };
+    const planPath = join(snapshot.directory, `plan-${snapshot.revision}.json`);
+    let plan;
+    try {
+        plan = JSON.parse(await readFile(planPath, 'utf8'));
+    }
+    catch { /* first attempt */ }
+    // A saved plan counts on the commits the cache held then; one the cache no longer holds (pruned, another project
+    // space) is negotiated again before anything is sent.
+    // A plan made while the cache had the revision carries no history: it holds only while the cache still does.
+    if (plan && (!plan.base || plan.base.some(oid => !probe.tips.includes(oid)) || (plan.published && !probe.published))) {
+        await cleanSpool(snapshot.directory, plan);
+        plan = undefined;
+    }
+    if (!plan) {
+        let known = await localCommits(folder, probe.tips, signal);
+        // History origin has, beyond what the cache holds, comes from origin inside the cloud rather than over this link:
+        // the VM's own fetch of the branch at boot when it made one (a cold cache takes it whatever its size; a cache
+        // behind by little gets the few commits from here), else a fetch from the cache.
+        if (!probe.published && m.origin && m.upstream?.ref.startsWith('refs/remotes/origin/') && !known.includes(m.upstream.oid)) {
+            const prefetch = staging ? probe.prefetch : undefined;
+            const estimate = prefetch && !known.length ? Infinity
+                : Number(await optional(folder, ['rev-list', '--objects', '--disk-usage', m.upstream.oid, ...known.map(oid => `^${oid}`)], signal)) || 0;
+            // A prefetch still running is waited for only when the history is worth a fetch of its own.
+            const adopt = prefetch === 'ready' ? estimate > PREFETCH_ADOPT_BYTES : prefetch === 'running' && estimate > (options.originFirstBytes ?? ORIGIN_FIRST_BYTES);
+            if (adopt || estimate > (options.originFirstBytes ?? ORIGIN_FIRST_BYTES)) {
+                try {
+                    const fetched = await invoke(cell, 'origin', cache, { url: m.origin,
+                        ref: `refs/heads/${m.upstream.ref.slice(20)}`, name: sha256(m.upstream.ref).slice(0, 16), format: m.format, expect: m.upstream.oid,
+                        ...(adopt ? { prefetch: options.apply.root } : {}) }, stats, signal);
+                    probe = { ...probe, tips: fetched.tips };
+                    // This computer's upstream commit counts as held only when the cache now has it (origin may have rewritten it).
+                    known = await localCommits(folder, [...fetched.tips, ...(fetched.has_expect !== false ? [m.upstream.oid] : [])], signal);
+                    originFetched = true;
+                    if (fetched.adopted)
+                        markTime('prefetch_adopted');
+                }
+                catch (error) {
+                    if (signal?.aborted || !(error instanceof PreparedRefusal))
+                        throw error;
+                }
+                markTime('origin');
+            }
+        }
+        plan = { ...await planTransfer(folder, snapshot, known, probe.missing_files, !probe.published, signal), base: known.filter(oid => probe.tips.includes(oid)),
+            published: probe.published };
+        await writeFile(planPath, JSON.stringify(plan), { mode: 0o600 });
+    }
+    markTime('plan');
+    // What the cache imported is its own record; packs staged into another VM count only once the cache has them.
+    const present = new Set(probe.chunks), done = new Set(probe.imported);
+    const send = async (packs, extras) => {
+        for (let attempt = 0;; attempt++) {
+            const upload = new Uploader(cell, partsDir, stats, signal), sent = [];
+            const staged = [];
+            let extrasChunks;
+            for (const pack of packs) {
+                if (done.has(pack.key)) {
+                    const edited = await readFile(changedPath(snapshot.directory, pack.key), 'utf8').then(text => JSON.parse(text), () => []);
+                    for (const path of edited)
+                        if (!changed.includes(path))
+                            changed.push(path);
+                    continue;
+                }
+                const record = await sendPack(folder, snapshot, pack, present, upload, stats, signal);
+                markTime(`packed_${pack.kind}`);
+                sizes[pack.kind] = record.size;
+                sent.push({ key: pack.key, chunks: record.chunks, thin: pack.thin });
+                if (pack.files)
+                    staged.push(...pack.files.map(f => f.oid).filter(oid => !record.changed?.length || !m.files.some(f => f.oid === oid && record.changed.includes(f.path))));
+                for (const path of record.changed ?? [])
+                    if (!changed.includes(path))
+                        changed.push(path);
+            }
+            if (extras && m.extras_key && !probe.extras_present) {
+                const archive = join(snapshot.directory, `extras-${m.extras_key}.tar`);
+                sizes.extras = (await stat(archive)).size;
+                extrasChunks = await sendFile(archive, present, upload, stats);
+            }
+            await upload.drain();
+            markTime('uploaded');
+            try {
+                let missing;
+                if (staging) {
+                    const out = await invoke(cell, 'stage', cache, { manifest: m, packs: sent, ...(extrasChunks ? { extras_chunks: extrasChunks } : {}),
+                        parts_dir: partsDir, apply: applyRequest(options.apply), media_staged: staged, ...(options.pin ? { pin: options.pin } : {}) }, stats, signal);
+                    applied ??= out.applied;
+                    const arrived = new Set(staged);
+                    missing = probe.missing_files.filter(oid => !arrived.has(oid));
+                    probe = { ...probe, missing_files: missing };
+                }
+                else {
+                    missing = (await invoke(cell, 'import', cache, { manifest: m, packs: sent, ...(extrasChunks ? { extras_chunks: extrasChunks } : {}) }, stats, signal)).missing_files;
+                }
+                markTime(staging ? 'staged' : 'imported');
+                for (const pack of sent)
+                    done.add(pack.key);
+                return missing;
+            }
+            catch (error) {
+                if (error instanceof PreparedRefusal && ['missing', 'corrupt'].includes(error.reason) && attempt < 2) {
+                    for (const key of error.chunks)
+                        present.delete(key);
+                    continue;
+                }
+                // Anything else (a delta base the cache no longer holds, a snapshot it cannot complete): the next attempt
+                // negotiates again from what the cache holds then, instead of repeating this plan.
+                if (!signal?.aborted)
+                    await cleanSpool(snapshot.directory, plan);
+                throw error;
+            }
+        }
+    };
+    // A revision another start published meanwhile needs only its media.
+    const packs = probe.published ? plan.packs.filter(p => p.kind === 'media') : plan.packs;
+    const media = packs.find(p => p.kind === 'media' && !done.has(p.key)), later = options.defer === true && (media?.files ?? []).reduce((n, f) => n + f.size, 0) > DEFER_BYTES;
+    // Already checked out (the probe did, or an earlier stage): only what is still missing goes, and nothing to stage
+    // when nothing is left.
+    const remaining = packs.filter(p => !done.has(p.key) && (!later || p.kind !== 'media'));
+    const missing = !probe.published || remaining.length || (m.extras_key && !probe.extras_present) || (staging && !applied) ? await send(remaining, true) : probe.missing_files;
+    markTime(later ? 'start_ready' : 'sent');
+    const out = result(missing);
+    if (!later)
+        await cleanSpool(snapshot.directory, plan);
+    return { ...out, ...(applied ? { applied } : {}) };
+}
+async function cleanSpool(directory, plan) {
+    for (const pack of plan.packs) {
+        const recordPath = join(directory, 'spool', `${pack.key}.json`);
+        try {
+            const record = JSON.parse(await readFile(recordPath, 'utf8'));
+            for (const key of record.chunks)
+                await rm(join(directory, 'spool', key), { force: true });
+        }
+        catch { /* already gone */ }
+        await rm(recordPath, { force: true });
+        await rm(changedPath(directory, pack.key), { force: true });
+    }
+    await rm(join(directory, `plan-${plan.revision}.json`), { force: true });
+}
+/** Git settings the VM repository copies from this one: origin, identity, the branch's upstream, exclude rules. */
+export async function preparedGitConfig(folder, manifest, signal) {
+    const user = {};
+    for (const key of ['name', 'email']) {
+        const value = await optional(folder, ['config', '--get', `user.${key}`], signal);
+        if (value)
+            user[key] = value;
+    }
+    const readOptional = async (path) => { try {
+        return await readFile(path, 'utf8');
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return '';
+        throw error;
+    } };
+    const info = await optional(folder, ['rev-parse', '--git-path', 'info/exclude'], signal);
+    const global = await optional(folder, ['config', '--path', '--get', 'core.excludesFile'], signal);
+    const excludes = (info ? await readOptional(resolve(folder, info)) : '') + '\n' + await readOptional(global ? resolve(folder, global) : join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'git', 'ignore'));
+    const tracking = manifest.branch && manifest.upstream?.ref.startsWith('refs/remotes/origin/') ? { branch: manifest.branch.slice(11), merge: `refs/heads/${manifest.upstream.ref.slice(20)}` } : undefined;
+    return { ...(manifest.origin ? { origin_url: manifest.origin } : {}), ...(Object.keys(user).length ? { user } : {}), ...(tracking ? { tracking } : {}), excludes };
+}
+const applyRequest = ({ root, branch, options = {} }) => ({ root, branch, ...options.git,
+    ...(options.preparation ? { preparation: true } : {}), ...(options.replacePreparation ? { replace_preparation: true } : {}), ...(options.adopt ? { adopt: true } : {}),
+    ...(options.dependencySource ? { dependency_source: options.dependencySource } : {}),
+    ...(options.dissociate !== undefined && options.dissociate !== true ? { dissociate: options.dissociate } : {}),
+    ...(options.background === false ? { background: false } : {}), ...(options.releasePin ? { release_pin: options.releasePin } : {}),
+    ...(options.after ? { after: options.after } : {}),
+    ...(options.setupIntent ? { setup_intent: { conversation: options.setupIntent.conversation, deps_hash: options.setupIntent.depsHash } } : {}),
+    ...(options.workspace ? { workspace: options.workspace } : {}) });
+/**
+ * Checks out the revision as `branch` at `root`, in one exec. Media not in the cache yet stay pending
+ * (landDeferredFiles). A root holding the same revision is left as it is; one holding another is refused, unless it is
+ * a project template's checkout or `replacePreparation`.
+ */
+export async function applyPreparedSnapshot(cell, cache, revision, root, branch, options = {}) {
+    return invoke(cell, 'apply', cache, { revision, ...applyRequest({ root, branch, options }) }, noStats(), options.signal);
+}
+/** Lands media that arrived after the start (where nothing is yet: the agent's own file stands); `drop` gives up on blobs that changed. */
+export async function landDeferredFiles(cell, cache, revision, root, options = {}) {
+    return invoke(cell, 'materialize', cache, { revision, root, drop: options.drop ?? [] }, noStats(), options.signal);
+}
+export async function prunePreparedCache(cell, cache, signal) {
+    await invoke(cell, 'prune', cache, {}, noStats(), signal);
+}
+/**
+ * The local spool stays bounded: a finished transfer's parts go at once; other parts and plans after a day or past
+ * the budget, oldest first; a revision's manifest and files archive (what a pending start resumes from) after a week.
+ */
+export async function pruneLocalPrepared(stateDir, limit = SPOOL_BYTES, every = 0) {
+    const root = join(stateDir, 'prepared');
+    let worktrees;
+    try {
+        worktrees = await readdir(root);
+    }
+    catch {
+        return;
+    }
+    // A start prunes at most every `every` ms (the time is in the file, written before the pass).
+    const marker = join(root, '.pruned-at'), last = Number(await readFile(marker, 'utf8').catch(() => '0')) || 0;
+    if (every && Date.now() - last < every)
+        return;
+    await writeFile(marker, String(Date.now()), { mode: 0o600 });
+    await withTargetLock(join(root, '.prune-lock'), async () => {
+        const files = [];
+        for (const id of worktrees) {
+            if (!/^[a-f0-9]{16}$/.test(id))
+                continue;
+            for (const dir of [join(root, id), join(root, id, 'spool')]) {
+                let names;
+                try {
+                    names = await readdir(dir);
+                }
+                catch {
+                    continue;
+                }
+                for (const name of names) {
+                    if (name === 'spool')
+                        continue;
+                    const path = join(dir, name), st = await stat(path).catch(() => undefined);
+                    if (st?.isFile())
+                        files.push({ path, size: st.size, at: st.mtimeMs });
+                }
+            }
+        }
+        let used = files.reduce((n, f) => n + f.size, 0);
+        for (const f of files.sort((a, b) => a.at - b.at)) {
+            const age = Date.now() - f.at, kept = /\/(?:manifest-[a-f0-9]{64}\.json|extras-[a-f0-9]{64}\.tar|blob-ids-sha(?:1|256)\.json|folder)$/.test(f.path);
+            if (age < 60_000)
+                continue; // a transfer in progress is writing it
+            if (kept ? age < 7 * 86_400_000 : used <= limit && age < 86_400_000)
+                continue;
+            await rm(f.path, { force: true });
+            used -= f.size;
+            // A dropped revision's snapshot no longer pins objects in the user's repository.
+            const revision = /\/manifest-([a-f0-9]{64})\.json$/.exec(f.path)?.[1];
+            const folder = revision ? await readFile(join(f.path, '..', 'folder'), 'utf8').catch(() => '') : '';
+            if (revision && folder)
+                await releaseLocalSnapshot(folder, revision);
+        }
+    });
+}