@shardflux/cli 0.18.0 → 0.19.0

package/dist/project-sync-transfer.js

@@ -1,22 +1,31 @@
 import { createHash, randomUUID } from 'node:crypto';
 import { constants } from 'node:fs';
-import { chmod, lstat, mkdir, open, readlink, rename, rm, symlink, utimes } from 'node:fs/promises';
+import { chmod, copyFile, lstat, mkdir, open, readlink, rename, rm, rmdir, symlink, utimes } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { tarEnd, tarHeader, tarPadding } from '@shardflux/sdk';
-import { contentIdentity, projectFilePath, validSyncPath } from "./project-sync-manifest.js";
+import { contentIdentity, localEntry, projectFilePath, validSyncPath } from "./project-sync-manifest.js";
+/** One request's worth of file data. A file larger than this streams through its own archive, one chunk at a time. */
 export const SYNC_CHUNK_BYTES = 8 << 20;
+/** Small files per batch archive (and at most SYNC_CHUNK_BYTES of their data). */
+export const SYNC_BATCH_FILES = 128;
+/** A seed (overwrite) checks no previous versions, so one archive carries more small files (still SYNC_CHUNK_BYTES). */
+const SYNC_SEED_BATCH_FILES = 4096;
+/** Deletions per apply run. */
+const SYNC_BATCH_DELETIONS = 4096;
+const changed = (path) => new Error(`${path} changed during sync; compare manifests again`);
 export function verifySyncBytes(entry, data) {
     const hash = createHash('sha256').update(entry.symlink ? `symlink\0${entry.link_target}` : data).digest('hex');
     if (hash !== entry.sha256 || (!entry.symlink && data.byteLength !== entry.size))
-        throw new Error(`${entry.path} changed during sync; compare manifests again`);
+        throw changed(entry.path);
 }
+/** A small file's verified bytes (whole in memory: callers stream anything above SYNC_CHUNK_BYTES). */
 export async function readLocalSyncFile(folder, entry) {
     const path = await projectFilePath(folder, entry.path);
     const st = await lstat(path);
     let data = new Uint8Array();
     if (entry.symlink) {
         if (!st.isSymbolicLink() || await readlink(path) !== entry.link_target)
-            throw new Error(`${entry.path} changed during sync`);
+            throw changed(entry.path);
     }
     else {
         const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
@@ -26,11 +35,11 @@ export async function readLocalSyncFile(folder, entry) {
             while (offset < data.length) {
                 const { bytesRead } = await file.read(data, offset, data.length - offset, offset);
                 if (!bytesRead)
-                    throw new Error(`${entry.path} changed during sync`);
+                    throw changed(entry.path);
                 offset += bytesRead;
             }
             if ((await file.stat()).size !== entry.size)
-                throw new Error(`${entry.path} changed during sync`);
+                throw changed(entry.path);
         }
         finally {
             await file.close();
@@ -41,34 +50,19 @@ export async function readLocalSyncFile(folder, entry) {
     verifySyncBytes(entry, data);
     return { entry, data };
 }
-export async function readRemoteSyncFile(cell, root, entry) {
-    const data = new Uint8Array(entry.symlink ? 0 : entry.size);
-    for (let offset = 0; offset < data.length;) {
-        const chunk = await cell.files.read(`${root}/${entry.path}`, { offset, length: Math.min(SYNC_CHUNK_BYTES, data.length - offset) });
-        if (chunk.length === 0 || chunk.length > data.length - offset)
-            throw new Error(`${entry.path} changed during sync`);
-        data.set(chunk, offset);
-        offset += chunk.length;
-    }
-    verifySyncBytes(entry, data);
-    return { entry, data };
+// ---------------------------------------------------------------------------------------------------------------
+// VM → laptop
+async function localIdentity(folder, path) {
+    const found = await localEntry(folder, path);
+    if (found && !('sha256' in found))
+        throw new Error(`${path} is not a regular file or symbolic link`);
+    return contentIdentity(found);
 }
 async function checkLocal(folder, path, expected) {
-    if (expected) {
-        await readLocalSyncFile(folder, expected);
-        return;
-    }
-    try {
-        await lstat(await projectFilePath(folder, path));
-    }
-    catch (error) {
-        if (error.code === 'ENOENT')
-            return;
-        throw error;
-    }
-    throw new Error(`${path} was created during sync`);
+    if (await localIdentity(folder, path) !== contentIdentity(expected))
+        throw new Error(expected ? `${path} changed during sync` : `${path} was created during sync`);
 }
-export async function applyLocalSyncFile(folder, path, file, expected) {
+async function makeParents(folder, path) {
     if (!validSyncPath(path))
         throw new Error(`unsafe sync path: ${path}`);
     // Create each parent separately and refuse links before descending.
@@ -86,31 +80,30 @@ export async function applyLocalSyncFile(folder, path, file, expected) {
         if (!st.isDirectory() || st.isSymbolicLink())
             throw new Error(`sync parent is not a directory: ${parent}`);
     }
-    const target = await projectFilePath(folder, path);
-    await checkLocal(folder, path, expected);
-    if (!file) {
-        await rm(target, { force: true });
-        return;
+}
+/** Directories a deletion left empty go too (as Git does), so a path can turn from a directory into a file. */
+async function pruneLocalParents(folder, path) {
+    for (let dir = dirname(path); dir !== '.'; dir = dirname(dir)) {
+        try {
+            await rmdir(join(folder, dir));
+        }
+        catch {
+            return;
+        }
     }
-    verifySyncBytes(file.entry, file.data);
+}
+/** Writes `tmp` with `write(fd)`, then re-checks the destination and renames over it; `tmp` never outlives the call. */
+async function replaceLocal(folder, path, entry, expected, write) {
+    const target = await projectFilePath(folder, path);
     const tmp = `${target}.shard-tmp-${randomUUID()}`;
     try {
-        if (file.entry.symlink)
-            await symlink(file.entry.link_target, tmp);
-        else {
-            const fd = await open(tmp, 'wx', file.entry.mode);
-            try {
-                await fd.writeFile(file.data);
-                await fd.sync();
-            }
-            finally {
-                await fd.close();
-            }
-            await chmod(tmp, file.entry.mode);
-            await utimes(tmp, file.entry.mtime_unix_ms / 1000, file.entry.mtime_unix_ms / 1000);
+        await write(tmp);
+        if (!entry.symlink) {
+            await chmod(tmp, entry.mode);
+            await utimes(tmp, entry.mtime_unix_ms / 1000, entry.mtime_unix_ms / 1000);
         }
-        // The read above is deliberately just before replacement; concurrent
-        // changes detected here fail the pass instead of advancing its baseline.
+        // The check is deliberately just before replacement; concurrent changes
+        // detected here fail the pass instead of advancing its baseline.
         await checkLocal(folder, path, expected);
         await rename(tmp, target);
     }
@@ -118,25 +111,114 @@ export async function applyLocalSyncFile(folder, path, file, expected) {
         await rm(tmp, { force: true });
     }
 }
-// Extract our existing SDK tar format with dirfd + O_NOFOLLOW, so an agent's
-// parent symlink cannot redirect a batch out of the project. Compare previous
-// hashes at the mutation boundary, retain mtimes, and fsync every replacement.
+async function writeFileAt(tmp, mode, fill) {
+    const fd = await open(tmp, 'wx', mode);
+    try {
+        await fill(fd);
+        await fd.sync();
+    }
+    finally {
+        await fd.close();
+    }
+}
+/** Applies a file already in memory (or deletes `path` when `file` is undefined). `expected` is the local version it replaces. */
+export async function applyLocalSyncFile(folder, path, file, expected) {
+    await makeParents(folder, path);
+    await checkLocal(folder, path, expected);
+    if (!file) {
+        await rm(await projectFilePath(folder, path), { force: true });
+        await pruneLocalParents(folder, path);
+        return;
+    }
+    verifySyncBytes(file.entry, file.data);
+    await replaceLocal(folder, path, file.entry, expected, async (tmp) => {
+        if (file.entry.symlink)
+            await symlink(file.entry.link_target, tmp);
+        else
+            await writeFileAt(tmp, file.entry.mode, async (fd) => { await fd.writeFile(file.data); });
+    });
+}
+/**
+ * Applies the VM's version of one file here (or deletes it when `entry` is undefined), streaming it chunk by chunk
+ * into a temporary file next to the target: any size, never whole in memory. The bytes must hash to `entry`.
+ * `from` reads another VM path (a conflict copy); `expected` is the local version it replaces.
+ */
+export async function pullRemoteFile(cell, root, folder, path, entry, options = {}) {
+    const { expected, signal } = options;
+    await makeParents(folder, path);
+    await checkLocal(folder, path, expected);
+    if (!entry) {
+        await rm(await projectFilePath(folder, path), { force: true });
+        await pruneLocalParents(folder, path);
+        return;
+    }
+    if (entry.symlink) {
+        verifySyncBytes(entry, new Uint8Array());
+        await replaceLocal(folder, path, entry, expected, (tmp) => symlink(entry.link_target, tmp));
+        return;
+    }
+    const source = `${root}/${options.from ?? path}`;
+    await replaceLocal(folder, path, entry, expected, (tmp) => writeFileAt(tmp, entry.mode, async (fd) => {
+        const hash = createHash('sha256');
+        for (let offset = 0; offset < entry.size;) {
+            signal?.throwIfAborted();
+            const chunk = await cell.files.read(source, { offset, length: Math.min(SYNC_CHUNK_BYTES, entry.size - offset) });
+            if (chunk.length === 0 || chunk.length > entry.size - offset)
+                throw changed(path);
+            hash.update(chunk);
+            for (let at = 0; at < chunk.length;)
+                at += (await fd.write(chunk, at, chunk.length - at, offset + at)).bytesWritten;
+            offset += chunk.length;
+        }
+        if (hash.digest('hex') !== entry.sha256)
+            throw changed(path);
+    }));
+}
+/** Copies a local file to another project path (a conflict copy), verified against `entry`; any size. */
+export async function copyLocalSyncFile(folder, entry, to) {
+    await makeParents(folder, to);
+    await checkLocal(folder, to, undefined);
+    const source = await projectFilePath(folder, entry.path);
+    await replaceLocal(folder, to, entry, undefined, async (tmp) => {
+        if (entry.symlink) {
+            if (await readlink(source) !== entry.link_target)
+                throw changed(entry.path);
+            await symlink(entry.link_target, tmp);
+            return;
+        }
+        await copyFile(source, tmp, constants.COPYFILE_EXCL);
+        const copied = await localEntry(dirname(tmp), tmp.slice(dirname(tmp).length + 1));
+        if (!copied || !('sha256' in copied) || copied.sha256 !== entry.sha256 || copied.size !== entry.size)
+            throw changed(entry.path);
+    });
+}
+// ---------------------------------------------------------------------------------------------------------------
+// Laptop → VM
+// Extract our SDK tar format with dirfd + O_NOFOLLOW, so an agent's parent
+// symlink cannot redirect a batch out of the project. The first member is the
+// plan: [path, previous identity (null with overwrite), next entry (null deletes)].
+// Compare previous hashes at the mutation boundary, retain mtimes, fsync every
+// replacement and each touched directory before exiting, except with overwrite (a seed into a fresh checkout, which a
+// retry repeats; an fsync per file cost 5.7 ms in a VM, 3.4 s for a 600-file seed). No size limits: members
+// stream through a 1 MiB buffer. A deletion removes the directories it empties.
 export const APPLY_SYNC_TAR = String.raw `
 import os,sys,json,hashlib,tarfile,uuid,stat
-root,archive,changes=sys.argv[1:]
-changes=json.loads(changes)
+root,archive=sys.argv[1:]
 rootfd=os.open(root,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
-def parent(p):
+def split(p):
  parts=p.split('/')
  if not p or any(x in ('','..','.','.git','.shard') for x in parts) or '\\' in p: raise ValueError('unsafe project path')
+ return parts
+def walk(parts,make):
  fd=os.dup(rootfd)
  try:
-  for x in parts[:-1]:
-   try: os.mkdir(x,dir_fd=fd)
-   except FileExistsError: pass
+  for x in parts:
+   if make:
+    try: os.mkdir(x,dir_fd=fd)
+    except FileExistsError: pass
    child=os.open(x,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=fd)
    os.close(fd);fd=child
-  return fd,parts[-1]
+  return fd
  except: os.close(fd);raise
 def identity(fd,name):
  try: st=os.stat(name,dir_fd=fd,follow_symlinks=False)
@@ -144,26 +226,40 @@ def identity(fd,name):
  if stat.S_ISLNK(st.st_mode):
   digest=hashlib.sha256(('symlink\0'+os.readlink(name,dir_fd=fd)).encode()).hexdigest();kind='link'
  elif stat.S_ISREG(st.st_mode):
-  if st.st_size>67108864: raise ValueError('destination grew beyond sync limit')
   f=os.open(name,os.O_RDONLY|os.O_NOFOLLOW,dir_fd=fd)
   with os.fdopen(f,'rb') as reader: digest=hashlib.file_digest(reader,'sha256').hexdigest()
   kind='file'
  else: raise ValueError('destination is not a file')
  return digest+':'+str(stat.S_IMODE(st.st_mode)&511)+':'+kind
+dirty=set()
+def prune(parts):
+ for n in range(len(parts)-1,0,-1):
+  try: fd=walk(parts[:n-1],False)
+  except OSError: return
+  try: os.rmdir(parts[n-1],dir_fd=fd)
+  except OSError: return
+  finally: os.close(fd)
+  dirty.add('/'.join(parts[:n-1]))
+tar=tarfile.open(archive,'r:')
 try:
+ ms=tar.getmembers()
+ if not ms or ms[0].name!='.shard-plan' or not ms[0].isfile(): raise ValueError('invalid sync archive')
+ plan=json.loads(tar.extractfile(ms[0]).read())
+ overwrite=plan.get('overwrite') is True
+ wanted={c[0] for c in plan['changes'] if c[2] is not None}
  members={}
- tar=None
- if archive:
-  tar=tarfile.open(archive,'r:')
-  for member in tar:
-   if member.name in members or member.name not in changes or not(member.isfile() or member.issym()) or member.size>67108864: raise ValueError('invalid sync archive')
-   members[member.name]=member
- for p,change in changes.items():
-  fd,name=parent(p)
+ for member in ms[1:]:
+  if member.name in members or member.name not in wanted or not(member.isfile() or member.issym()): raise ValueError('invalid sync archive')
+  members[member.name]=member
+ for p,previous,entry in plan['changes']:
+  parts=split(p);name=parts[-1]
+  try: fd=walk(parts[:-1],entry is not None)
+  except (FileNotFoundError,NotADirectoryError):
+   if overwrite or previous=='absent': continue
+   raise ValueError(p+' changed during sync')
   tmp=name+'.shard-tmp-'+uuid.uuid4().hex
   try:
-   if identity(fd,name)!=change['previous']: raise ValueError(p+' changed during sync')
-   entry=change['next']
+   if not overwrite and identity(fd,name)!=previous: raise ValueError(p+' changed during sync')
    if entry is None:
     try: os.unlink(name,dir_fd=fd)
     except FileNotFoundError: pass
@@ -182,55 +278,141 @@ try:
        b=reader.read(1048576)
        if not b: break
        digest.update(b);writer.write(b)
-      writer.flush();os.fchmod(writer.fileno(),entry['mode']);os.fsync(writer.fileno())
+      writer.flush();os.fchmod(writer.fileno(),entry['mode'])
+      if not overwrite: os.fsync(writer.fileno())
      digest=digest.hexdigest()
      os.utime(tmp,ns=(int(entry['mtime_unix_ms']*1000000),)*2,dir_fd=fd,follow_symlinks=False)
     if digest!=entry['sha256']: raise ValueError('content mismatch')
-    if identity(fd,name)!=change['previous']: raise ValueError(p+' changed during sync')
-    os.replace(tmp,name,src_dir_fd=fd,dst_dir_fd=fd)
-   os.fsync(fd)
+    if not overwrite and identity(fd,name)!=previous: raise ValueError(p+' changed during sync')
+    try: os.replace(tmp,name,src_dir_fd=fd,dst_dir_fd=fd)
+    except IsADirectoryError:
+     if not overwrite: raise
+     os.rmdir(name,dir_fd=fd);os.replace(tmp,name,src_dir_fd=fd,dst_dir_fd=fd)
+   dirty.add('/'.join(parts[:-1]))
   finally:
    try: os.unlink(tmp,dir_fd=fd)
    except FileNotFoundError: pass
    os.close(fd)
+  if entry is None: prune(parts)
+ for d in ([] if overwrite else dirty):
+  try: fd=walk(d.split('/') if d else [],False)
+  except OSError: continue
+  try: os.fsync(fd)
+  finally: os.close(fd)
 finally:
- if tar: tar.close()
+ tar.close()
  os.close(rootfd)
 `;
+/** Appends to a VM file one SYNC_CHUNK_BYTES request at a time, holding at most one chunk. */
+class SyncBlobWriter {
+    #chunk = new Uint8Array(SYNC_CHUNK_BYTES);
+    #held = 0;
+    #written = 0;
+    #cell;
+    #path;
+    #signal;
+    constructor(cell, path, signal) { this.#cell = cell; this.#path = path; this.#signal = signal; }
+    get size() { return this.#written + this.#held; }
+    async push(data) {
+        for (let at = 0; at < data.length;) {
+            const take = Math.min(SYNC_CHUNK_BYTES - this.#held, data.length - at);
+            this.#chunk.set(data.subarray(at, at + take), this.#held);
+            this.#held += take;
+            at += take;
+            if (this.#held === SYNC_CHUNK_BYTES)
+                await this.#flush();
+        }
+    }
+    /** Copies `size` bytes of `file` from `position`, hashing them; a short file fails as changed. */
+    async pushFrom(file, size, hash, path, position = 0) {
+        for (let left = size; left > 0;) {
+            this.#signal?.throwIfAborted();
+            const { bytesRead } = await file.read(this.#chunk, this.#held, Math.min(SYNC_CHUNK_BYTES - this.#held, left), position);
+            if (!bytesRead)
+                throw changed(path);
+            hash?.update(this.#chunk.subarray(this.#held, this.#held + bytesRead));
+            this.#held += bytesRead;
+            position += bytesRead;
+            left -= bytesRead;
+            if (this.#held === SYNC_CHUNK_BYTES)
+                await this.#flush();
+        }
+    }
+    async close() { if (this.#held > 0 || this.#written === 0)
+        await this.#flush(); }
+    async #flush() {
+        this.#signal?.throwIfAborted();
+        await this.#cell.files.write(this.#path, this.#chunk.subarray(0, this.#held), { mode: '0600', append: this.#written > 0 });
+        this.#written += this.#held;
+        this.#held = 0;
+    }
+}
 export async function writeSyncBlob(cell, path, data) {
-    if (data.byteLength === 0)
-        await cell.files.write(path, data, { mode: '0600' });
-    for (let offset = 0; offset < data.byteLength; offset += SYNC_CHUNK_BYTES)
-        await cell.files.write(path, data.subarray(offset, offset + SYNC_CHUNK_BYTES), { mode: '0600', append: offset > 0 });
-}
-export async function applyRemoteSyncBatch(cell, root, files, expected, signal) {
-    const parts = [];
-    const changes = Object.create(null);
-    let size = 0;
-    for (const [path, file] of files) {
-        if (!validSyncPath(path))
-            throw new Error(`unsafe sync path: ${path}`);
-        changes[path] = { previous: contentIdentity(expected[path]), next: file?.entry ?? null };
-        if (file) {
-            verifySyncBytes(file.entry, file.data);
-            const header = tarHeader({ path, type: file.entry.symlink ? 'symlink' : 'file', size: file.data.length, mode: file.entry.mode, linkname: file.entry.link_target, mtime: Math.floor(file.entry.mtime_unix_ms / 1000) });
-            const padding = tarPadding(file.data.length);
-            parts.push(header, file.data, padding);
-            size += header.length + file.data.length + padding.length;
+    const writer = new SyncBlobWriter(cell, path);
+    await writer.push(data);
+    await writer.close();
+}
+/** Uploads a local file of any size to a VM path, SYNC_CHUNK_BYTES at a time straight from disk. */
+export async function writeSyncFileFrom(cell, path, local, signal) {
+    const file = await open(local, constants.O_RDONLY);
+    try {
+        const writer = new SyncBlobWriter(cell, path, signal);
+        await writer.pushFrom(file, (await file.stat()).size, undefined, local);
+        await writer.close();
+    }
+    finally {
+        await file.close();
+    }
+}
+/** Downloads a VM file of `size` bytes to a new local file, SYNC_CHUNK_BYTES at a time. */
+export async function readSyncFileTo(cell, path, size, local, signal) {
+    await writeFileAt(local, 0o600, async (fd) => {
+        for (let offset = 0; offset < size;) {
+            signal?.throwIfAborted();
+            const chunk = await cell.files.read(path, { offset, length: Math.min(SYNC_CHUNK_BYTES, size - offset) });
+            if (chunk.length === 0 || chunk.length > size - offset)
+                throw new Error(`${path} changed while it was read`);
+            for (let at = 0; at < chunk.length;)
+                at += (await fd.write(chunk, at, chunk.length - at, offset + at)).bytesWritten;
+            offset += chunk.length;
         }
+    });
+}
+/** Copies a file of `size` bytes from one VM to another, SYNC_CHUNK_BYTES at a time. */
+export async function copySyncBlob(from, path, size, to, target, signal) {
+    if (size === 0) {
+        await to.files.write(target, new Uint8Array(), { mode: '0600' });
+        return;
     }
-    const end = tarEnd(size);
-    const archive = new Uint8Array(size + end.length);
-    let at = 0;
-    for (const part of parts) {
-        archive.set(part, at);
-        at += part.length;
+    for (let offset = 0; offset < size;) {
+        signal?.throwIfAborted();
+        const chunk = await from.files.read(path, { offset, length: Math.min(SYNC_CHUNK_BYTES, size - offset) });
+        if (chunk.length === 0 || chunk.length > size - offset)
+            throw new Error(`${path} changed while it was copied`);
+        await to.files.write(target, chunk, { mode: '0600', append: offset > 0 });
+        offset += chunk.length;
     }
-    archive.set(end, at);
+}
+const noMembers = () => Promise.resolve();
+const fileHeader = (entry) => tarHeader({ path: entry.path, type: entry.symlink ? 'symlink' : 'file', size: entry.symlink ? 0 : entry.size, mode: entry.mode, linkname: entry.link_target, mtime: Math.floor(entry.mtime_unix_ms / 1000) });
+/** Writes one archive (plan first, then `body` adds the members) to the VM and applies it there. */
+async function applyArchive(cell, root, overwrite, changes, signal, body) {
+    for (const [path] of changes)
+        if (!validSyncPath(path))
+            throw new Error(`unsafe sync path: ${path}`);
     const temp = `/tmp/shard-sync-${randomUUID()}.tar`;
+    const writer = new SyncBlobWriter(cell, temp, signal);
     try {
-        await writeSyncBlob(cell, temp, archive);
-        const result = await cell.exec.run(['python3', '-c', APPLY_SYNC_TAR, root, temp, JSON.stringify(changes)], { timeoutMs: 60_000, maxOutputBytes: 4096, signal });
+        const plan = new TextEncoder().encode(JSON.stringify({ overwrite, changes }));
+        await writer.push(tarHeader({ path: '.shard-plan', type: 'file', size: plan.length, mode: 0o600 }));
+        await writer.push(plan);
+        await writer.push(tarPadding(plan.length));
+        await body(writer);
+        await writer.push(tarEnd(writer.size));
+        await writer.close();
+        // Time grows with the work: 60 s, plus 50 ms a change, plus a second per 16 MiB.
+        const timeoutMs = 60_000 + changes.length * 50 + Math.ceil(writer.size / (16 << 20)) * 1000;
+        const result = await cell.exec.run(['python3', '-c', APPLY_SYNC_TAR, root, temp], { timeoutMs, maxOutputBytes: 4096, signal });
         if (result.exitCode !== 0)
             throw new Error(`project sync batch failed: ${result.stderr.trim()}`);
     }
@@ -238,3 +420,119 @@ export async function applyRemoteSyncBatch(cell, root, files, expected, signal)
         await cell.files.remove(temp).catch(() => undefined);
     }
 }
+async function pushSmallFiles(cell, root, overwrite, files, previous, signal) {
+    for (const file of files)
+        verifySyncBytes(file.entry, file.data);
+    await applyArchive(cell, root, overwrite, files.map((file) => [file.entry.path, previous(file.entry.path), file.entry]), signal, async (writer) => {
+        for (const file of files) {
+            await writer.push(fileHeader(file.entry));
+            await writer.push(file.data);
+            await writer.push(tarPadding(file.data.length));
+        }
+    });
+}
+/** One file of any size, from disk to the VM through its own archive, hashed on the way; differing bytes fail the pass. */
+async function pushLargeFile(cell, root, folder, entry, overwrite, previous, signal) {
+    const path = await projectFilePath(folder, entry.path);
+    const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        const st = await file.stat();
+        if (!st.isFile() || st.size !== entry.size)
+            throw changed(entry.path);
+        if ((st.mode & 0o777) !== entry.mode)
+            throw new Error(`${entry.path} mode changed during sync`);
+        await applyArchive(cell, root, overwrite, [[entry.path, previous, entry]], signal, async (writer) => {
+            await writer.push(fileHeader(entry));
+            const hash = createHash('sha256');
+            await writer.pushFrom(file, entry.size, hash, entry.path);
+            if (hash.digest('hex') !== entry.sha256 || (await file.stat()).size !== entry.size)
+                throw changed(entry.path);
+            await writer.push(tarPadding(entry.size));
+        });
+    }
+    finally {
+        await file.close();
+    }
+}
+/**
+ * Writes local files into the VM project at `root`: `deletions` first, then `files` (manifest entries of `folder`),
+ * small ones batched (at most SYNC_BATCH_FILES and SYNC_CHUNK_BYTES per archive), larger ones streamed from disk one
+ * chunk at a time. Every file is verified against its entry. Without `overwrite`, each VM path must still be the
+ * `expected` version (absent when missing) or the pass fails as changed; `overwrite: true` replaces whatever is there.
+ * A deletion removes the directories it empties, so a directory can become a file.
+ */
+export async function pushLocalFiles(cell, root, folder, files, deletions, options = {}) {
+    const { signal, overwrite = false } = options;
+    const expected = options.expected ?? {};
+    const previous = (path) => overwrite ? null : contentIdentity(Object.hasOwn(expected, path) ? expected[path] : undefined);
+    for (let at = 0; at < deletions.length; at += SYNC_BATCH_DELETIONS) {
+        await applyArchive(cell, root, overwrite, deletions.slice(at, at + SYNC_BATCH_DELETIONS).map((path) => [path, previous(path), null]), signal, noMembers);
+    }
+    let batch = [], bytes = 0;
+    const flush = async () => { if (batch.length)
+        await pushSmallFiles(cell, root, overwrite, batch, previous, signal); batch = []; bytes = 0; };
+    for (const entry of files) {
+        signal?.throwIfAborted();
+        if (!validSyncPath(entry.path))
+            throw new Error(`unsafe sync path: ${entry.path}`);
+        if (!entry.symlink && entry.size > SYNC_CHUNK_BYTES) {
+            await pushLargeFile(cell, root, folder, entry, overwrite, previous(entry.path), signal);
+            continue;
+        }
+        if (batch.length >= (overwrite ? SYNC_SEED_BATCH_FILES : SYNC_BATCH_FILES) || bytes + entry.size > SYNC_CHUNK_BYTES)
+            await flush();
+        const file = await readLocalSyncFile(folder, entry);
+        batch.push(file);
+        bytes += file.data.length;
+    }
+    await flush();
+}
+// ---------------------------------------------------------------------------------------------------------------
+// VM manifest
+// Hashes VM files a guest left out of its manifest for size (guests before unbounded manifests stop at 64 MiB).
+const HASH_SYNC_FILES = String.raw `
+import os,sys,json,hashlib,stat
+rootfd=os.open(sys.argv[1],os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
+out=[]
+for p in json.loads(sys.argv[2]):
+ parts=p.split('/')
+ fd=os.dup(rootfd)
+ try:
+  try:
+   for x in parts[:-1]:
+    c=os.open(x,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=fd);os.close(fd);fd=c
+   st=os.stat(parts[-1],dir_fd=fd,follow_symlinks=False)
+  except (FileNotFoundError,NotADirectoryError): continue
+  if not stat.S_ISREG(st.st_mode): continue
+  f=os.open(parts[-1],os.O_RDONLY|os.O_NOFOLLOW,dir_fd=fd)
+  with os.fdopen(f,'rb') as r:
+   d=hashlib.file_digest(r,'sha256').hexdigest();after=os.fstat(r.fileno())
+  if after.st_size!=st.st_size or after.st_mtime_ns!=st.st_mtime_ns or after.st_ino!=st.st_ino: raise ValueError(p+' changed while hashing; retry project sync')
+  out.append({'path':p,'sha256':d,'size':st.st_size,'mode':stat.S_IMODE(st.st_mode)&511,'mtime_unix_ms':st.st_mtime_ns//1000000,'symlink':False,'link_target':''})
+ finally: os.close(fd)
+print(json.dumps(out))
+`;
+/**
+ * The VM project's manifest with every file hashed, whatever its size: no `maxFileBytes` is sent, and files an
+ * older guest still skips for size (`max_file_bytes`) are hashed in the VM and returned as entries.
+ */
+export async function remoteManifest(cell, root, knownPaths, signal) {
+    const manifest = await cell.files.manifest(root, { ...(knownPaths ? { knownPaths } : {}), ...(signal ? { signal } : {}) });
+    const oversized = Array.isArray(manifest?.skipped) ? manifest.skipped.filter((skip) => skip?.reason === 'max_file_bytes' && validSyncPath(skip.path)).map((skip) => skip.path) : [];
+    if (oversized.length === 0)
+        return manifest;
+    for (let at = 0; at < oversized.length; at += 16) {
+        const paths = oversized.slice(at, at + 16);
+        // Hashing reads each file once in the VM: allow ten minutes per 16 files.
+        const result = await cell.exec.run(['python3', '-c', HASH_SYNC_FILES, root, JSON.stringify(paths)], { timeoutMs: 600_000, maxOutputBytes: 1 << 20, ...(signal ? { signal } : {}) });
+        if (result.exitCode !== 0)
+            throw new Error(`project sync could not hash VM files: ${result.stderr.trim()}`);
+        for (const entry of JSON.parse(result.stdout))
+            if (paths.includes(entry.path))
+                manifest.entries.push(entry);
+    }
+    // A file that is gone or no longer regular is simply absent from this pass.
+    const hashed = new Set(oversized);
+    manifest.skipped = manifest.skipped.filter((skip) => !(skip.reason === 'max_file_bytes' && hashed.has(skip.path)));
+    return manifest;
+}