@shardflux/cli 0.18.1 → 0.19.0

package/dist/agent-repository.js

@@ -1,12 +1,14 @@
 import { randomUUID } from 'node:crypto';
-import { mkdtemp, readFile, rm, stat } from 'node:fs/promises';
-import { homedir, tmpdir } from 'node:os';
-import { join, resolve } from 'node:path';
-import { CodedError, UsageError } from "./errors.js";
+import { spawn } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { basename, join, resolve } from 'node:path';
+import { UsageError } from "./errors.js";
 import { AgentStateStore } from "./agent-state.js";
 import { projectIdentity } from "./project-identity.js";
-import { localManifest, projectGit } from "./project-sync-manifest.js";
-import { applyRemoteSyncBatch, readLocalSyncFile, writeSyncBlob } from "./project-sync-transfer.js";
+import { excludedSyncPath, localEntry, projectGit, projectGitPaths, selectedProjectPaths, validSyncPath } from "./project-sync-manifest.js";
+import { pushLocalFiles, SYNC_CHUNK_BYTES, writeSyncBlob } from "./project-sync-transfer.js";
+import { seedPhase } from "./seed-phases.js";
 export function projectWorkspacePath(project) {
     if (!/^[a-z0-9][a-z0-9._-]{0,95}$/.test(project.slug))
         throw new Error('invalid sync project slug');
@@ -20,7 +22,28 @@ export function needsGitSeed(ctx, ws) {
     const seed = new AgentStateStore(ctx.store.dir).read().git_seeding?.[ws.id];
     return seed !== undefined && seed.api_url === ctx.apiUrl;
 }
-/** One history seed for C5 and C11. Bundles use the bounded files path, without a Git helper or second connection. */
+/**
+ * The checks a start can fail on, run on this computer before a VM is allocated (0.19.0+). A project's size, file
+ * count or history never refuses a start; only a folder inside another repository does.
+ */
+export async function preflightProjectSeed(folder, signal) {
+    let top;
+    try {
+        top = (await projectGit(folder, ['rev-parse', '--show-toplevel'], signal)).trim();
+    }
+    catch (error) {
+        if (signal?.aborted || error.code !== 128)
+            throw error;
+        return;
+    }
+    if (top !== folder)
+        throw new UsageError('project sync requires the Git repository root; run shard there');
+}
+/**
+ * One seed for every start (C5, C11). A Git project's files come from Git in the VM: HEAD is checked out there and only
+ * what differs from HEAD on this computer (changes, untracked files, deletions, .env files) is copied (0.19.0+). Its
+ * size, file count and history length do not change what is sent.
+ */
 export async function seedAgentRepository(ctx, ws, folder, options = {}) {
     const project = projectIdentity(folder);
     // A cloud-only `like` launch must never seed an unrelated laptop repository.
@@ -41,8 +64,11 @@ export async function seedAgentRepository(ctx, ws, folder, options = {}) {
     initial?.ready();
     state.update(doc => { const seeds = (doc.git_seeding ?? {}); delete seeds[ws.id]; doc.git_seeding = seeds; });
 }
-/** History above this goes through the project's origin instead of the bounded files path. */
-export const HISTORY_BUNDLE_LIMIT = 256 << 20;
+/**
+ * Above this (an estimate: the repository's packed and loose objects) the workspace fetches the history from origin as a
+ * blobless partial clone, and receives only the commits origin lacks (0.19.0+). Below it, one small bundle is faster.
+ */
+export const ORIGIN_FIRST_BYTES = 32 << 20;
 async function optionalGit(folder, args, signal) {
     try {
         return (await projectGit(folder, args, signal)).trim();
@@ -54,71 +80,137 @@ async function optionalGit(folder, args, signal) {
     }
 }
 const tail = (text) => text.trim().split('\n').slice(-3).join(' ').slice(-400);
-// Fetches one origin branch (or origin's HEAD) into an empty workspace repository and prints its commit.
-const FETCH_ORIGIN = 'set -eu; git init -q "$1"; case "$3" in HEAD) dst=refs/shard-sync/origin ;; *) dst="refs/remotes/origin/${3#refs/heads/}" ;; esac; ' +
-    'git -C "$1" -c core.hooksPath=/dev/null -c credential.interactive=false fetch -q --no-tags "$2" "+$3:$dst" >&2; git -C "$1" rev-parse --verify "$dst^{commit}"';
+/** The repository's object store size, an upper bound of any one branch's history bundle. Reads sizes only. */
+export async function historyEstimate(folder, signal) {
+    const counts = new Map((await optionalGit(folder, ['count-objects', '-v'], signal)).split('\n').map(line => line.split(': ')));
+    return (Number(counts.get('size') ?? 0) + Number(counts.get('size-pack') ?? 0)) * 1024;
+}
+// Fetches one origin branch (or origin's HEAD) into an empty workspace repository as a blobless partial clone (0.19.0+):
+// every commit and tree. Alongside it, a depth-1 fetch of the same ref brings the tip's files in one pack, moved in
+// beside the history, so the checkout reads them locally instead of asking for each blob. Prints the commit. A failed
+// history fetch leaves a plain repository, so the bundle fallback is not a partial clone of an origin it cannot reach.
+export const FETCH_ORIGIN = [
+    'set -eu',
+    'git init -q "$1"; git -C "$1" config remote.origin.url "$2"',
+    'case "$3" in HEAD) dst=refs/shard-sync/origin ;; *) dst="refs/remotes/origin/${3#refs/heads/}" ;; esac',
+    'tip="$1/.git/shard-sync-tip"; rm -rf "$tip"; git init -q --bare "$tip"',
+    'git -C "$tip" -c core.hooksPath=/dev/null -c credential.interactive=false -c fetch.unpackLimit=1 fetch -q --no-tags --depth=1 "$2" "+$3:refs/tip" >/dev/null 2>&1 &',
+    'job=$!',
+    'if ! git -C "$1" -c core.hooksPath=/dev/null -c credential.interactive=false fetch -q --no-tags --filter=blob:none origin "+$3:$dst" >&2; then',
+    '  wait "$job" || true; rm -rf "$tip"',
+    '  git -C "$1" config --unset remote.origin.promisor || true; git -C "$1" config --unset remote.origin.partialclonefilter || true; exit 1',
+    'fi',
+    'if wait "$job"; then for pack in "$tip"/objects/pack/pack-*.pack; do [ -f "$pack" ] || continue; base=${pack%.pack}',
+    '  for ext in idx rev pack; do if [ -f "$base.$ext" ]; then mv "$base.$ext" "$1/.git/objects/pack/"; fi; done; done; fi',
+    'rm -rf "$tip"',
+    'git -C "$1" rev-parse --verify "$dst^{commit}"',
+].join('\n');
+// Completes a blobless partial clone in the background, at the lowest priority, once the start is done: later pushes to
+// the project hub, `git log -p` and blame then read local objects.
+export const BACKFILL = 'set -eu; cd "$1"; git -c core.hooksPath=/dev/null -c credential.interactive=false fetch -q --no-tags --no-write-fetch-head --refetch --no-filter origin "$2"; ' +
+    'git config --unset remote.origin.partialclonefilter || true; git config --unset remote.origin.promisor || true';
 // Applies an optional bundle of HEAD (plus its branch and upstream) and points the workspace HEAD at the laptop's.
 const APPLY_HISTORY = 'set -eu; git init -q "$1"; if [ -n "$2" ]; then git -C "$1" -c core.hooksPath=/dev/null fetch -q --update-head-ok "$2" HEAD:refs/shard-sync/initial ' +
     '"+refs/heads/*:refs/heads/*" "+refs/tags/*:refs/tags/*" "+refs/remotes/*:refs/remotes/*"; fi; ' +
     'if [ -n "$4" ]; then git -C "$1" symbolic-ref HEAD "$4"; else git -C "$1" update-ref --no-deref HEAD "$3"; fi; git -C "$1" update-ref HEAD "$3"; git -C "$1" read-tree HEAD; ' +
     'for ref in refs/shard-sync/initial refs/shard-sync/origin; do if git -C "$1" rev-parse -q --verify "$ref" >/dev/null; then git -C "$1" update-ref -d "$ref"; fi; done';
-/** A history the first copy cannot carry (repository_too_large, 0.13.3+): the facts, and the next step as the hint. */
-export function tooLarge(message, hint, branch, origin) {
-    return new CodedError('repository_too_large', `${message}.`, hint, { limit_bytes: HISTORY_BUNDLE_LIMIT, branch, ...(origin ? { origin } : {}) });
+// Writes HEAD's files that are missing from the workspace tree (never replacing one), except those this computer
+// deleted. A partial clone first fetches the missing blobs in one request; Git LFS files are left to the copy.
+export const CHECKOUT_HEAD = String.raw `
+import subprocess,sys
+root,skipfile=sys.argv[1],sys.argv[2]
+def git(*a,data=None): return subprocess.run(['git','-C',root,'-c','core.hooksPath=/dev/null','-c','filter.lfs.smudge=cat','-c','filter.lfs.process=','-c','filter.lfs.required=false',*a],check=True,input=data,stdout=subprocess.PIPE).stdout
+skip=set(open(skipfile,'rb').read().split(b'\0')) if skipfile else set()
+missing=[p for p in git('ls-files','-z','--deleted').split(b'\0') if p and p not in skip]
+if missing and git('config','--bool','--default','false','remote.origin.promisor').strip()==b'true':
+ want,wanted=set(),set(missing)
+ for rec in git('ls-files','-z','-s').split(b'\0'):
+  if rec:
+   meta,path=rec.split(b'\t',1)
+   if path in wanted: want.add(meta.split()[1])
+ absent={l[1:] for l in git('rev-list','--objects','--no-walk','--missing=print','HEAD').split(b'\n') if l.startswith(b'?')}
+ want&=absent
+ if want: git('-c','fetch.negotiationAlgorithm=noop','fetch','-q','--no-tags','--no-write-fetch-head','--recurse-submodules=no','--filter=blob:none','--stdin','origin',data=b'\n'.join(sorted(want))+b'\n')
+for at in range(0,len(missing),4096): git('checkout-index','-z','--stdin',data=b'\0'.join(missing[at:at+4096])+b'\0')
+print(len(missing))
+`;
+/** `git bundle create` streamed straight into the workspace in chunks: any size, no temporary file, no time limit. */
+async function streamBundle(cell, folder, remote, revs, signal) {
+    const child = spawn('git', ['-C', folder, '-c', 'core.hooksPath=/dev/null', 'bundle', 'create', '-q', '-', ...revs], { stdio: ['ignore', 'pipe', 'pipe'], signal, env: { ...process.env, GIT_TERMINAL_PROMPT: '0', GIT_OPTIONAL_LOCKS: '0' } });
+    let stderr = '';
+    child.stderr.on('data', (chunk) => { stderr = (stderr + chunk.toString()).slice(-4000); });
+    const exited = new Promise((resolve, reject) => { child.on('error', reject); child.on('close', code => resolve(code ?? 1)); });
+    let pending = [], size = 0, total = 0;
+    const flush = async () => { if (!size && total)
+        return; await cell.files.write(remote, Buffer.concat(pending, size), { mode: '0600', append: total > 0 }); total += size; pending = []; size = 0; };
+    try {
+        for await (const chunk of child.stdout) {
+            pending.push(chunk);
+            size += chunk.length;
+            if (size >= SYNC_CHUNK_BYTES)
+                await flush();
+        }
+        await flush();
+    }
+    catch (error) {
+        child.kill();
+        await exited.catch(() => undefined);
+        throw error;
+    }
+    const code = await exited;
+    if (code !== 0)
+        throw new Error(`Git could not bundle the project history: ${tail(stderr)}`);
+    return total;
 }
 /**
  * Imports the checked-out branch's history into an empty workspace repository. Only HEAD, its branch and that branch's
- * upstream are bundled: other branches, worktrees and stashes stay on this computer. When that is still above `limit`,
- * the workspace fetches `origin` itself (with its own Git credentials) and receives only the commits origin lacks.
+ * upstream are sent: other branches, worktrees and stashes stay on this computer. A history above `originFirst` comes
+ * from origin as a blobless partial clone (the workspace's own Git credentials), plus a bundle of the commits origin
+ * lacks. Without a reachable origin the whole bundle is streamed in chunks: no history size refuses (0.19.0+).
  */
-export async function importProjectHistory(cell, folder, root, head, branch, origin, signal, limit = HISTORY_BUNDLE_LIMIT) {
+export async function importProjectHistory(cell, folder, root, head, branch, origin, signal, originFirst = ORIGIN_FIRST_BYTES, report = () => undefined) {
     const upstream = branch ? await optionalGit(folder, ['rev-parse', '--symbolic-full-name', `${branch.slice(11)}@{upstream}`], signal) : '';
     const tracked = upstream.startsWith('refs/remotes/') ? upstream : '';
     const tips = ['HEAD', ...(branch ? [branch] : []), ...(tracked ? [tracked] : [])];
-    const dir = await mkdtemp(join(tmpdir(), 'shard-sync-git-'));
-    const local = join(dir, 'history.bundle'), remote = `/tmp/shard-sync-${randomUUID()}.bundle`;
+    const remote = `/tmp/shard-sync-${randomUUID()}.bundle`;
     try {
-        await projectGit(folder, ['bundle', 'create', local, ...tips], signal);
-        let bundle = true;
-        if ((await stat(local)).size > limit) {
-            const name = branch ? branch.slice(11) : 'HEAD';
-            if (!origin)
-                throw tooLarge(`Git history of ${name} exceeds the 256 MiB sync limit and the project has no network origin the workspace can clone`, 'add an origin remote, then run shard again', name, origin);
+        let revs = tips, partial = null;
+        if (origin && await historyEstimate(folder, signal) > originFirst) {
             const ref = upstream.startsWith('refs/remotes/origin/') ? `refs/heads/${upstream.slice(20)}` : 'HEAD';
             const fetched = await cell.exec.run(['sh', '-c', FETCH_ORIGIN, 'shard-sync', root, origin, ref], { timeoutMs: 600_000, maxOutputBytes: 8192, env: { GIT_TERMINAL_PROMPT: '0' }, signal });
+            seedPhase('origin_fetch');
             const base = fetched.stdout.trim();
-            if (fetched.exitCode !== 0 || !/^[0-9a-f]{40}(?:[0-9a-f]{24})?$/.test(base)) {
-                throw tooLarge(`Git history of ${name} exceeds the 256 MiB sync limit and the workspace could not fetch ${origin}: ${tail(fetched.stderr)}`, 'give the machine access to the repository (GitHub: run gh auth login in shard machine), then run shard again', name, origin);
-            }
-            // Commits origin already has are not sent again; a base this computer does not know falls back to its upstream.
-            const exclude = await optionalGit(folder, ['cat-file', '-t', base], signal) === 'commit' ? base : tracked;
-            if (!exclude)
-                throw tooLarge(`Git history of ${name} exceeds the 256 MiB sync limit and shares no known commit with ${origin}`, 'push the branch, then run shard again', name, origin);
-            let onOrigin = true;
-            try {
-                await projectGit(folder, ['merge-base', '--is-ancestor', head, exclude], signal);
-            }
-            catch (error) {
-                if (signal?.aborted || error.code !== 1)
-                    throw error;
-                onOrigin = false;
-            }
-            bundle = !onOrigin;
-            if (bundle) {
-                await rm(local, { force: true });
-                await projectGit(folder, ['bundle', 'create', local, ...tips.filter(tip => tip !== tracked), '--not', exclude], signal);
-                if ((await stat(local)).size > limit)
-                    throw tooLarge(`Commits of ${name} that ${origin} does not have exceed the 256 MiB sync limit`, 'push them, then run shard again', name, origin);
+            if (fetched.exitCode === 0 && /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/.test(base)) {
+                // Commits origin already has are not sent again; a base this computer does not know falls back to its upstream.
+                const exclude = await optionalGit(folder, ['cat-file', '-t', base], signal) === 'commit' ? base : tracked;
+                let onOrigin = false;
+                if (exclude) {
+                    try {
+                        await projectGit(folder, ['merge-base', '--is-ancestor', head, exclude], signal);
+                        onOrigin = true;
+                    }
+                    catch (error) {
+                        if (signal?.aborted || error.code !== 1)
+                            throw error;
+                    }
+                }
+                revs = onOrigin ? null : exclude ? [...tips.filter(tip => tip !== tracked), '--not', exclude] : tips;
+                partial = ref;
             }
+            else
+                report(`Sending the history from this computer: the workspace could not fetch ${origin} (${tail(fetched.stderr)}).`);
+        }
+        if (revs) {
+            await streamBundle(cell, folder, remote, revs, signal);
+            seedPhase('bundle');
         }
-        if (bundle)
-            await writeSyncBlob(cell, remote, await readFile(local));
-        const result = await cell.exec.run(['sh', '-c', APPLY_HISTORY, 'shard-sync', root, bundle ? remote : '', head, branch], { timeoutMs: 60_000, maxOutputBytes: 4096, signal });
+        const result = await cell.exec.run(['sh', '-c', APPLY_HISTORY, 'shard-sync', root, revs ? remote : '', head, branch], { timeoutMs: 600_000, maxOutputBytes: 4096, signal });
         if (result.exitCode !== 0)
             throw new Error(`initial Git bundle failed: ${result.stderr.trim()}`);
+        seedPhase('apply_history');
+        return { partial };
     }
     finally {
-        await rm(dir, { recursive: true, force: true });
         await cell.files.remove(remote).catch(() => undefined);
     }
 }
@@ -131,7 +223,7 @@ async function initializeProjectGit(cell, folder, root, signal, populate = false
         if (signal?.aborted || error.code !== 128)
             throw error;
         if (populate)
-            await seedProjectFiles(cell, folder, root, '', new Set(), true, signal, report, initial);
+            await copyProjectFiles(cell, folder, root, await missingInWorkspace(cell, root, await selectedProjectPaths(folder, signal), signal), [], signal, report, initial, false);
         return;
     }
     if (top !== folder)
@@ -156,9 +248,10 @@ async function initializeProjectGit(cell, folder, root, signal, populate = false
             if (signal?.aborted)
                 throw error;
         }
+        let partial = null;
         if (head && importHistory) {
             const remote = projectIdentity(folder).remote;
-            await importProjectHistory(cell, folder, root, head, branch, remote && !remote.startsWith('file:') ? remote : null, signal);
+            ({ partial } = await importProjectHistory(cell, folder, root, head, branch, remote && !remote.startsWith('file:') ? remote : null, signal, ORIGIN_FIRST_BYTES, report));
         }
         else if (importHistory) {
             const result = await cell.exec.run(['git', 'init', '-q', ...(branch.startsWith('refs/heads/') ? ['--initial-branch', branch.slice(11)] : []), root], { timeoutMs: 5000, signal });
@@ -194,11 +287,13 @@ async function initializeProjectGit(cell, folder, root, signal, populate = false
                         throw new Error('could not copy Git upstream');
                 }
         }
-        // Ignore files must exist before the first remote scan; otherwise a
-        // template's dependency outputs would look like agent-created source.
-        // For unborn repos, intent-to-add preserves tracked ignored paths too.
-        const tracked = new Set((await projectGit(folder, ['ls-files', '-z', '--cached'], signal)).split('\0'));
-        await seedProjectFiles(cell, folder, root, head, tracked, populate, signal, report, initial);
+        if (head)
+            await seedFromHead(cell, folder, root, populate, signal, report, initial);
+        else
+            await seedUnborn(cell, folder, root, populate, signal, report, initial);
+        // Optional work, after the start's own: the lowest priority, and a failure only leaves the clone partial.
+        if (partial)
+            await cell.exec.start({ argv: ['nice', '-n', '19', 'sh', '-c', BACKFILL, 'shard-backfill', root, partial], env: { GIT_TERMINAL_PROMPT: '0' }, timeout_ms: 3_600_000 }, signal).catch(() => undefined);
     }
     // Git's info/exclude and laptop-wide excludes are selection metadata, copied
     // into a dedicated Git exclude file without syncing credentials or .git data.
@@ -226,44 +321,166 @@ async function initializeProjectGit(cell, folder, root, signal, populate = false
     const result = await cell.exec.run(['git', '-C', root, 'config', 'core.excludesFile', `${root}/.git/shard-sync-excludes`], { timeoutMs: 5000, signal });
     if (result.exitCode !== 0)
         throw new Error('could not apply project Git exclude rules');
+    seedPhase('excludes');
 }
-async function seedProjectFiles(cell, folder, root, head, tracked, populate, signal, report, initial) {
-    const raw = await localManifest(folder, signal);
-    for (const skip of raw.skipped)
-        report(`Project seed skipped ${skip.path}: ${skip.reason}`);
-    const seed = raw.entries.filter((entry) => populate || entry.path.split('/').at(-1) === '.gitignore' || (!head && tracked.has(entry.path)));
-    const groups = initial ? [seed.filter(e => initial.paths.has(e.path)), seed.filter(e => !initial.paths.has(e.path))] : [seed];
+const nul = (text) => text.split('\0').filter(Boolean);
+/** Git's NUL-separated output of any size, streamed (no output buffer or time bound). */
+async function gitList(folder, args, signal) {
+    const out = [];
+    await projectGitPaths(folder, args, path => { out.push(path); }, { signal });
+    return out;
+}
+/**
+ * What differs between HEAD and this computer's working tree, from Git's own status (0.19.0+): changed, added and
+ * untracked files Git does not ignore, the .env files we always carry, Git LFS files (their content lives outside Git),
+ * and deletions. Ignored files, nested repositories and submodule contents stay out.
+ */
+export async function localChanges(folder, signal) {
+    const [fields, untracked, env, lfs] = await Promise.all([
+        gitList(folder, ['diff', '--name-status', '-z', '--no-renames', '--ignore-submodules=all', 'HEAD', '--'], signal),
+        gitList(folder, ['ls-files', '-z', '--others', '--exclude-standard', '--', '.'], signal),
+        gitList(folder, ['ls-files', '-z', '--others', '--ignored', '--exclude-standard', '--', '.env*', '**/.env*'], signal),
+        gitList(folder, ['ls-files', '-z', '--cached', '--', ':(attr:filter=lfs)'], signal).catch((error) => { if (signal?.aborted)
+            throw error; return []; }),
+    ]);
+    const selected = new Set([...untracked, ...env.filter(path => basename(path).startsWith('.env')), ...lfs]);
+    const removed = new Set();
+    for (let at = 0; at + 1 < fields.length; at += 2) {
+        const status = fields[at], path = fields[at + 1];
+        if (!status || !path)
+            continue;
+        if (status === 'D' && !selected.has(path))
+            removed.add(path);
+        else
+            selected.add(path);
+    }
+    const copy = [], remove = [], skipped = [];
+    for (const path of [...selected].sort()) {
+        if (path.endsWith('/')) {
+            skipped.push({ path, reason: 'nested_repository' });
+            continue;
+        }
+        if (excludedSyncPath(path))
+            continue;
+        if (!validSyncPath(path)) {
+            skipped.push({ path, reason: 'unsupported_path' });
+            continue;
+        }
+        copy.push(path);
+    }
+    for (const path of [...removed].sort())
+        if (validSyncPath(path) && !excludedSyncPath(path))
+            remove.push(path);
+    return { copy, remove, skipped };
+}
+/** HEAD's files come from the workspace's own Git; then only this computer's changes are copied. */
+async function seedFromHead(cell, folder, root, populate, signal, report, initial) {
+    if (!populate) {
+        // Ignore files must exist before the first remote scan; otherwise a template's dependency outputs would look like
+        // agent-created source.
+        const ignores = await gitList(folder, ['ls-files', '-z', '--cached', '--others', '--exclude-standard', '--', '.gitignore', '**/.gitignore'], signal);
+        await copyProjectFiles(cell, folder, root, await missingInWorkspace(cell, root, ignores, signal), [], signal, report, undefined, false);
+        return;
+    }
+    const changes = await localChanges(folder, signal);
+    seedPhase('local_changes');
+    const skip = changes.remove.length ? `/tmp/shard-sync-${randomUUID()}.skip` : '';
+    try {
+        if (skip)
+            await writeSyncBlob(cell, skip, Buffer.from(changes.remove.join('\0')));
+        const checkout = await cell.exec.run(['python3', '-c', CHECKOUT_HEAD, root, skip], { timeoutMs: 600_000, maxOutputBytes: 8192, env: { GIT_TERMINAL_PROMPT: '0' }, signal });
+        if (checkout.exitCode !== 0)
+            throw new Error(`the workspace could not check out HEAD: ${tail(checkout.stderr)}`);
+    }
+    finally {
+        if (skip)
+            await cell.files.remove(skip).catch(() => undefined);
+    }
+    seedPhase('checkout_head');
+    for (const skipped of changes.skipped)
+        report(`Project seed skipped ${skipped.path}: ${skipped.reason}`);
+    await copyProjectFiles(cell, folder, root, changes.copy, changes.remove, signal, report, initial);
+}
+/** An unborn repository has no history to check out: its files are copied, and tracked ones are marked intent-to-add. */
+async function seedUnborn(cell, folder, root, populate, signal, report, initial) {
+    const tracked = (await gitList(folder, ['ls-files', '-z', '--cached'], signal)).filter(path => validSyncPath(path) && !excludedSyncPath(path));
+    const paths = populate ? await selectedProjectPaths(folder, signal) : [...new Set([...tracked, ...await gitList(folder, ['ls-files', '-z', '--others', '--exclude-standard', '--', '.gitignore', '**/.gitignore'], signal)])];
+    await copyProjectFiles(cell, folder, root, await missingInWorkspace(cell, root, paths, signal), [], signal, report, initial, false);
+    const present = new Set(paths);
+    const add = tracked.filter(path => present.has(path));
+    if (!add.length)
+        return;
+    const list = `/tmp/shard-sync-${randomUUID()}.add`;
+    try {
+        await writeSyncBlob(cell, list, Buffer.from(add.join('\0')));
+        const result = await cell.exec.run(['git', '-C', root, '-c', 'core.hooksPath=/dev/null', 'add', '-N', '-f', `--pathspec-from-file=${list}`, '--pathspec-file-nul'], { timeoutMs: 600_000, signal });
+        if (result.exitCode !== 0)
+            throw new Error(`initial tracked project files failed: ${result.stderr.trim()}`);
+    }
+    finally {
+        await cell.files.remove(list).catch(() => undefined);
+    }
+}
+// Rewrites the list file with only the paths the workspace tree lacks and prints its size: the answer is read back in
+// chunks, so any number of paths fits.
+const MISSING = 'import os,sys;root=sys.argv[1].encode();f=sys.argv[2];paths=[p for p in open(f,"rb").read().split(b"\\0") if p];' +
+    'out=b"\\0".join(p for p in paths if not os.path.lexists(os.path.join(root,p)));open(f,"wb").write(out);print(len(out))';
+/** The paths not yet in the workspace tree: a retried or partial seed never replaces what is there. */
+async function missingInWorkspace(cell, root, paths, signal) {
+    if (!paths.length)
+        return [];
+    const list = `/tmp/shard-sync-${randomUUID()}.paths`;
+    try {
+        await writeSyncBlob(cell, list, Buffer.from(paths.join('\0')));
+        const probe = await cell.exec.run(['python3', '-c', MISSING, root, list], { timeoutMs: 600_000, maxOutputBytes: 4096, signal });
+        const size = Number(probe.stdout.trim());
+        if (probe.exitCode !== 0 || !Number.isSafeInteger(size))
+            throw new Error(`could not inspect the workspace project files: ${tail(probe.stderr)}`);
+        const parts = [];
+        for (let offset = 0; offset < size;) {
+            const chunk = await cell.files.read(list, { offset, length: Math.min(SYNC_CHUNK_BYTES, size - offset) });
+            if (!chunk.length)
+                throw new Error('the workspace file list changed while it was read');
+            parts.push(chunk);
+            offset += chunk.length;
+        }
+        return nul(Buffer.concat(parts).toString('utf8'));
+    }
+    finally {
+        await cell.files.remove(list).catch(() => undefined);
+    }
+}
+/**
+ * Copies these files from this computer (any number, any size: small ones batched, big ones streamed) and applies the
+ * deletions. `replace` (a seed over a fresh checkout) overwrites what the checkout wrote; otherwise only missing paths
+ * are passed in. Files a harness reads first (initial) go before the rest.
+ */
+async function copyProjectFiles(cell, folder, root, paths, deletions, signal, report, initial, replace = true) {
+    const groups = initial ? [paths.filter(path => initial.paths.has(path)), paths.filter(path => !initial.paths.has(path))] : [paths];
     for (let group = 0; group < groups.length; group++) {
-        for (let offset = 0; offset < groups[group].length; offset += 32) {
-            const entries = groups[group].slice(offset, offset + 32);
-            const probe = await cell.exec.run(['python3', '-c', 'import os,sys,json;root=sys.argv[1];print(json.dumps([p for p in json.loads(sys.argv[2]) if not os.path.lexists(root+"/"+p)]))', root, JSON.stringify(entries.map((entry) => entry.path))], { timeoutMs: 5000, signal });
-            if (probe.exitCode !== 0)
-                throw new Error('could not inspect initial project ignore files');
-            const missing = new Set(JSON.parse(probe.stdout));
-            let seeds = new Map(), bytes = 0;
-            for (const entry of entries)
-                if (missing.has(entry.path)) {
-                    if (bytes + entry.size > 8 << 20 && seeds.size) {
-                        await applyRemoteSyncBatch(cell, root, seeds, {}, signal);
-                        seeds = new Map();
-                        bytes = 0;
-                    }
-                    const file = await readLocalSyncFile(folder, entry);
-                    seeds.set(entry.path, file);
-                    bytes += file.data.length;
-                }
-            if (seeds.size)
-                await applyRemoteSyncBatch(cell, root, seeds, {}, signal);
-            if (!head) {
-                const add = entries.filter((entry) => tracked.has(entry.path)).map((entry) => entry.path);
-                if (add.length) {
-                    const result = await cell.exec.run(['git', '-C', root, '-c', 'core.hooksPath=/dev/null', 'add', '-N', '-f', '--', ...add], { timeoutMs: 5000, signal });
-                    if (result.exitCode !== 0)
-                        throw new Error(`initial tracked project files failed: ${result.stderr.trim()}`);
+        const entries = [];
+        let next = 0;
+        const list = groups[group];
+        await Promise.all(Array.from({ length: Math.min(16, list.length) }, async () => {
+            for (;;) {
+                const path = list[next++];
+                if (path === undefined)
+                    break;
+                const entry = await localEntry(folder, path, signal);
+                if (entry === undefined)
+                    continue;
+                if (!('sha256' in entry)) {
+                    report(`Project seed skipped ${entry.path}: ${entry.reason}`);
+                    continue;
                 }
+                entries.push(entry);
             }
-        }
+        }));
+        entries.sort((a, b) => a.path.localeCompare(b.path));
+        // Deletions go first: a directory this computer replaced with a file must be gone before the file arrives.
+        await pushLocalFiles(cell, root, folder, entries, group === 0 ? deletions : [], { signal, overwrite: replace });
         if (group === 0)
             initial?.ready();
     }
+    seedPhase('copy_changes');
 }

package/dist/agent-state.d.ts

@@ -22,6 +22,9 @@ export declare class AgentStateStore {
         host: string;
         api_url: string;
     }): void;
+    /** The APIs whose account has its T3 Connect sign-in, as the last live check saw it: every project can start in T3. */
+    t3Connected(apiUrl: string): boolean;
+    saveT3Connected(apiUrl: string, connected: boolean): void;
     saveTarget(project: string, target: string): void;
     update(change: (state: State) => void): void;
     save(project: string, app: 'claude' | 'codex'): void;

package/dist/agent-state.js

@@ -41,6 +41,19 @@ export class AgentStateStore {
             doc.t3 = [...list.filter(e => !(e.host === entry.host && e.api_url === entry.api_url)), entry];
         });
     }
+    /** The APIs whose account has its T3 Connect sign-in, as the last live check saw it: every project can start in T3. */
+    t3Connected(apiUrl) {
+        const list = this.read().t3_connect;
+        return Array.isArray(list) && list.includes(apiUrl);
+    }
+    saveT3Connected(apiUrl, connected) {
+        if (this.t3Connected(apiUrl) === connected)
+            return;
+        this.update(doc => {
+            const list = (Array.isArray(doc.t3_connect) ? doc.t3_connect : []).filter((url) => typeof url === 'string' && url !== apiUrl);
+            doc.t3_connect = connected ? [...list, apiUrl] : list;
+        });
+    }
     saveTarget(project, target) {
         this.update(doc => {
             doc.projects ??= {};

package/dist/agent-target-commands.d.ts

@@ -33,5 +33,7 @@ export declare function openClaudeDesktopSession(ctx: Ctx, ws: Workspace, conver
  * server in the VM (SSH mode), and T3 is brought to the front with --json too (`opened: true`).
  */
 export declare function openDesktopWorkspace(ctx: Ctx, ws: Workspace, target: DesktopTarget, t3Version?: string, prewarmed?: boolean, thread?: T3ThreadRequest, install?: T3Install, conversation?: string | null): Promise<number>;
+/** T3 Code's Connections page (its t3code:// link), where a project linked with T3 Connect is listed. */
+export declare const T3_CONNECTIONS_PAGE = "t3code://app/settings/connections";
 export declare function queueT3Prewarm(ctx: Ctx, ws: Workspace, version: string): Promise<void>;
 export declare const TARGET_HANDLERS: Record<string, Handler>;