@shardflux/cli 0.11.0 → 0.13.0

package/dist/project-sync-manifest.js

@@ -0,0 +1,225 @@
+import { createHash } from 'node:crypto';
+import { execFile } from 'node:child_process';
+import { constants } from 'node:fs';
+import { lstat, open, readdir, readlink, realpath } from 'node:fs/promises';
+import { basename, dirname, join, relative, resolve, sep } from 'node:path';
+export const MAX_SYNC_FILE_BYTES = 64 * 1024 * 1024;
+export const MAX_SYNC_ENTRIES = 20_000;
+export function validSyncPath(path) {
+    return path.length > 0 && Buffer.byteLength(path) <= 4096 && !path.includes('\0') && !path.includes('\\') && !path.startsWith('/') && !path.split('/').some((part) => !part || part === '.' || part === '..' || part === '.git' || part === '.shard');
+}
+export function excludedSyncPath(path) { return path.split('/').some((part) => part === '.git' || part === '.shard' || part.includes('.shard-tmp-')); }
+export function contentIdentity(entry) {
+    return entry ? `${entry.sha256}:${entry.mode}:${entry.symlink ? 'link' : 'file'}` : 'absent';
+}
+/** Deletions have an observation timestamp: a detached filesystem has no deletion clock. Ties favor the laptop. */
+export function diffManifests(local, remote, common, observedAt = Date.now(), blocked = new Set()) {
+    const changes = [];
+    for (const path of [...new Set([...Object.keys(local), ...Object.keys(remote), ...Object.keys(common)])].sort()) {
+        if (blocked.has(path))
+            continue;
+        const localEntry = Object.hasOwn(local, path) ? local[path] : undefined;
+        const remoteEntry = Object.hasOwn(remote, path) ? remote[path] : undefined;
+        const l = contentIdentity(localEntry), r = contentIdentity(remoteEntry);
+        if (l === r)
+            continue;
+        const base = Object.hasOwn(common, path) ? common[path] : 'absent';
+        const lc = l !== base, rc = r !== base;
+        const winner = lc && !rc ? 'local' : rc && !lc ? 'remote' : (localEntry?.mtime_unix_ms ?? observedAt) >= (remoteEntry?.mtime_unix_ms ?? observedAt) ? 'local' : 'remote';
+        changes.push({ path, winner, conflict: lc && rc });
+    }
+    return changes;
+}
+export function parseManifest(manifest) {
+    if (!manifest || !Array.isArray(manifest.entries) || !Array.isArray(manifest.skipped) || manifest.entries.length + manifest.skipped.length > MAX_SYNC_ENTRIES)
+        throw new Error('invalid project manifest');
+    const entries = Object.create(null);
+    for (const entry of manifest.entries) {
+        if (!validSyncPath(entry.path) || !/^[a-f0-9]{64}$/.test(entry.sha256) || !Number.isSafeInteger(entry.size) || entry.size < 0 || entry.size > MAX_SYNC_FILE_BYTES || !Number.isInteger(entry.mode) || entry.mode < 0 || entry.mode > 0o777 || !Number.isFinite(entry.mtime_unix_ms) || typeof entry.symlink !== 'boolean' || typeof entry.link_target !== 'string' || entries[entry.path])
+            throw new Error(`invalid project manifest entry: ${entry.path}`);
+        entries[entry.path] = entry;
+    }
+    for (const skip of manifest.skipped)
+        if (!validSyncPath(skip.path) || typeof skip.reason !== 'string' || !skip.reason)
+            throw new Error('invalid skipped project path');
+    return entries;
+}
+export async function projectGit(folder, args, signal, input = '') {
+    return new Promise((resolve, reject) => {
+        const child = execFile('git', ['-C', folder, '-c', 'core.fsmonitor=false', '-c', 'core.untrackedCache=false', '-c', 'core.hooksPath=/dev/null', ...args], { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 << 20, signal, env: { ...process.env, GIT_TERMINAL_PROMPT: '0', GIT_OPTIONAL_LOCKS: '0' } }, (error, stdout) => {
+            if (error) {
+                const failure = Object.assign(new Error(error.message, { cause: error }), { code: error.code, stdout });
+                reject(failure);
+            }
+            else
+                resolve(stdout);
+        });
+        child.stdin?.on('error', () => undefined); // Early Git refusal is reported by the callback.
+        child.stdin?.end(input);
+    });
+}
+/** Uses the actual Git implementation rather than an approximation of gitignore. */
+export async function selectedProjectPaths(folder, signal) {
+    let git = false;
+    try {
+        git = (await projectGit(folder, ['rev-parse', '--is-inside-work-tree'], signal)).trim() === 'true';
+    }
+    catch (error) {
+        if (signal?.aborted || error.code === 'ENOENT')
+            throw error;
+    }
+    const paths = new Set();
+    if (git) {
+        for (const path of (await projectGit(folder, ['ls-files', '-z', '--cached', '--others', '--exclude-standard', '--', '.'], signal)).split('\0'))
+            if (path && !excludedSyncPath(path))
+                paths.add(path);
+        for (const path of (await projectGit(folder, ['ls-files', '-z', '--others', '--ignored', '--exclude-standard', '--', '.env*', '**/.env*'], signal)).split('\0')) {
+            if (path && basename(path).startsWith('.env') && !excludedSyncPath(path))
+                paths.add(path);
+        }
+    }
+    else {
+        const walk = async (dir) => {
+            signal?.throwIfAborted();
+            for (const entry of await readdir(join(folder, dir), { withFileTypes: true })) {
+                const path = dir ? `${dir}/${entry.name}` : entry.name;
+                if (excludedSyncPath(path))
+                    continue;
+                if (entry.isDirectory())
+                    await walk(path);
+                else
+                    paths.add(path);
+                if (paths.size > MAX_SYNC_ENTRIES)
+                    throw new Error(`project exceeds ${MAX_SYNC_ENTRIES} sync entries`);
+            }
+        };
+        await walk('');
+    }
+    if (paths.size > MAX_SYNC_ENTRIES)
+        throw new Error(`project exceeds ${MAX_SYNC_ENTRIES} sync entries`);
+    for (const path of paths)
+        if (!validSyncPath(path))
+            throw new Error(`unsupported project path: ${JSON.stringify(path)}`);
+    return [...paths].sort();
+}
+export async function ignoredProjectPaths(folder, paths, signal) {
+    paths = paths.filter((path) => !(basename(path).startsWith('.env')));
+    if (paths.length === 0)
+        return [];
+    try {
+        await projectGit(folder, ['rev-parse', '--is-inside-work-tree'], signal);
+    }
+    catch (error) {
+        if (signal?.aborted)
+            throw error;
+        return [];
+    }
+    const ignored = [];
+    for (let at = 0; at < paths.length; at += 32) {
+        let output;
+        try {
+            output = await projectGit(folder, ['check-ignore', '-z', '--stdin'], signal, paths.slice(at, at + 32).join('\0') + '\0');
+        }
+        catch (error) {
+            const failure = error;
+            if (failure.code !== 1)
+                throw error;
+            output = failure.stdout ?? '';
+        }
+        ignored.push(...output.split('\0').filter(Boolean));
+    }
+    return ignored;
+}
+/** Refuse parent symlinks, including a tracked file beneath a submodule/link. */
+export async function projectFilePath(folder, path) {
+    if (!validSyncPath(path))
+        throw new Error(`unsafe project path: ${JSON.stringify(path)}`);
+    folder = await realpath(folder);
+    const target = resolve(folder, path);
+    const parent = await realpath(dirname(target));
+    const rel = relative(folder, parent);
+    if (rel === '..' || rel.startsWith(`..${sep}`) || resolve(folder, dirname(path)) !== parent)
+        throw new Error(`project path traverses a symbolic link: ${path}`);
+    return target;
+}
+export async function localManifest(folder, signal, cache, known = []) {
+    const paths = await selectedProjectPaths(folder, signal);
+    const result = { entries: [], skipped: [] };
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(16, paths.length) }, async () => {
+        for (;;) {
+            const path = paths[next++];
+            if (path === undefined)
+                break;
+            signal?.throwIfAborted();
+            let target;
+            let st;
+            try {
+                target = await projectFilePath(folder, path);
+                st = await lstat(target);
+            }
+            catch (error) {
+                if (error.code === 'ENOENT')
+                    continue;
+                throw error;
+            }
+            const metadata = `${st.dev}:${st.ino}:${st.size}:${st.mode}:${st.mtimeMs}:${st.ctimeMs}`;
+            const cached = cache?.get(path);
+            if (cached?.metadata === metadata && st.size <= MAX_SYNC_FILE_BYTES) {
+                result.entries.push(cached.entry);
+                continue;
+            }
+            let sha256, link_target = '';
+            if (st.isSymbolicLink()) {
+                link_target = await readlink(target);
+                sha256 = createHash('sha256').update(`symlink\0${link_target}`).digest('hex');
+            }
+            else if (st.isFile()) {
+                if (st.size > MAX_SYNC_FILE_BYTES) {
+                    result.skipped.push({ path, reason: 'max_file_bytes' });
+                    continue;
+                }
+                const file = await open(target, constants.O_RDONLY | constants.O_NOFOLLOW);
+                try {
+                    const hash = createHash('sha256');
+                    let bytes = 0;
+                    for await (const raw of file.createReadStream({ autoClose: false })) {
+                        const chunk = raw;
+                        signal?.throwIfAborted();
+                        bytes += chunk.length;
+                        if (bytes > MAX_SYNC_FILE_BYTES)
+                            throw new Error(`${path} grew beyond the sync limit`);
+                        hash.update(chunk);
+                    }
+                    const after = await file.stat();
+                    if (bytes !== st.size || after.ino !== st.ino || after.mtimeMs !== st.mtimeMs || after.ctimeMs !== st.ctimeMs)
+                        throw new Error(`${path} changed while hashing; retry project sync`);
+                    sha256 = hash.digest('hex');
+                }
+                finally {
+                    await file.close();
+                }
+            }
+            else {
+                result.skipped.push({ path, reason: 'unsupported_type' });
+                continue;
+            }
+            const entry = { path, sha256, size: st.size, mode: st.mode & 0o777, mtime_unix_ms: st.mtimeMs, symlink: st.isSymbolicLink(), link_target };
+            result.entries.push(entry);
+            if (cache) {
+                if (cache.size >= MAX_SYNC_ENTRIES)
+                    cache.clear();
+                cache.set(path, { metadata, entry });
+            }
+        }
+    }));
+    const selected = new Set(paths);
+    for (const path of known)
+        if (!validSyncPath(path))
+            throw new Error('invalid common sync path');
+    for (const path of await ignoredProjectPaths(folder, known.filter((path) => !selected.has(path)), signal))
+        result.skipped.push({ path, reason: 'ignored' });
+    result.entries.sort((a, b) => a.path.localeCompare(b.path));
+    result.skipped.sort((a, b) => a.path.localeCompare(b.path));
+    return result;
+}

package/dist/project-sync-transfer.d.ts

@@ -0,0 +1,14 @@
+import type { CellClient, FileManifestEntry } from '@shardflux/sdk';
+export type SyncCell = Pick<CellClient, 'files' | 'exec'>;
+export interface SyncFile {
+    entry: FileManifestEntry;
+    data: Uint8Array;
+}
+export declare const SYNC_CHUNK_BYTES: number;
+export declare function verifySyncBytes(entry: FileManifestEntry, data: Uint8Array): void;
+export declare function readLocalSyncFile(folder: string, entry: FileManifestEntry): Promise<SyncFile>;
+export declare function readRemoteSyncFile(cell: SyncCell, root: string, entry: FileManifestEntry): Promise<SyncFile>;
+export declare function applyLocalSyncFile(folder: string, path: string, file: SyncFile | undefined, expected?: FileManifestEntry): Promise<void>;
+export declare const APPLY_SYNC_TAR: string;
+export declare function writeSyncBlob(cell: SyncCell, path: string, data: Uint8Array): Promise<void>;
+export declare function applyRemoteSyncBatch(cell: SyncCell, root: string, files: Map<string, SyncFile | undefined>, expected: Record<string, FileManifestEntry | undefined>, signal?: AbortSignal): Promise<void>;

package/dist/project-sync-transfer.js

@@ -0,0 +1,240 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { chmod, lstat, mkdir, open, readlink, rename, rm, symlink, utimes } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { tarEnd, tarHeader, tarPadding } from '@shardflux/sdk';
+import { contentIdentity, projectFilePath, validSyncPath } from "./project-sync-manifest.js";
+export const SYNC_CHUNK_BYTES = 8 << 20;
+export function verifySyncBytes(entry, data) {
+    const hash = createHash('sha256').update(entry.symlink ? `symlink\0${entry.link_target}` : data).digest('hex');
+    if (hash !== entry.sha256 || (!entry.symlink && data.byteLength !== entry.size))
+        throw new Error(`${entry.path} changed during sync; compare manifests again`);
+}
+export async function readLocalSyncFile(folder, entry) {
+    const path = await projectFilePath(folder, entry.path);
+    const st = await lstat(path);
+    let data = new Uint8Array();
+    if (entry.symlink) {
+        if (!st.isSymbolicLink() || await readlink(path) !== entry.link_target)
+            throw new Error(`${entry.path} changed during sync`);
+    }
+    else {
+        const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+        try {
+            data = new Uint8Array(entry.size);
+            let offset = 0;
+            while (offset < data.length) {
+                const { bytesRead } = await file.read(data, offset, data.length - offset, offset);
+                if (!bytesRead)
+                    throw new Error(`${entry.path} changed during sync`);
+                offset += bytesRead;
+            }
+            if ((await file.stat()).size !== entry.size)
+                throw new Error(`${entry.path} changed during sync`);
+        }
+        finally {
+            await file.close();
+        }
+    }
+    if ((st.mode & 0o777) !== entry.mode)
+        throw new Error(`${entry.path} mode changed during sync`);
+    verifySyncBytes(entry, data);
+    return { entry, data };
+}
+export async function readRemoteSyncFile(cell, root, entry) {
+    const data = new Uint8Array(entry.symlink ? 0 : entry.size);
+    for (let offset = 0; offset < data.length;) {
+        const chunk = await cell.files.read(`${root}/${entry.path}`, { offset, length: Math.min(SYNC_CHUNK_BYTES, data.length - offset) });
+        if (chunk.length === 0 || chunk.length > data.length - offset)
+            throw new Error(`${entry.path} changed during sync`);
+        data.set(chunk, offset);
+        offset += chunk.length;
+    }
+    verifySyncBytes(entry, data);
+    return { entry, data };
+}
+async function checkLocal(folder, path, expected) {
+    if (expected) {
+        await readLocalSyncFile(folder, expected);
+        return;
+    }
+    try {
+        await lstat(await projectFilePath(folder, path));
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return;
+        throw error;
+    }
+    throw new Error(`${path} was created during sync`);
+}
+export async function applyLocalSyncFile(folder, path, file, expected) {
+    if (!validSyncPath(path))
+        throw new Error(`unsafe sync path: ${path}`);
+    // Create each parent separately and refuse links before descending.
+    let parent = folder;
+    for (const part of dirname(path).split('/').filter((part) => part !== '.')) {
+        parent = join(parent, part);
+        try {
+            await mkdir(parent);
+        }
+        catch (error) {
+            if (error.code !== 'EEXIST')
+                throw error;
+        }
+        const st = await lstat(parent);
+        if (!st.isDirectory() || st.isSymbolicLink())
+            throw new Error(`sync parent is not a directory: ${parent}`);
+    }
+    const target = await projectFilePath(folder, path);
+    await checkLocal(folder, path, expected);
+    if (!file) {
+        await rm(target, { force: true });
+        return;
+    }
+    verifySyncBytes(file.entry, file.data);
+    const tmp = `${target}.shard-tmp-${randomUUID()}`;
+    try {
+        if (file.entry.symlink)
+            await symlink(file.entry.link_target, tmp);
+        else {
+            const fd = await open(tmp, 'wx', file.entry.mode);
+            try {
+                await fd.writeFile(file.data);
+                await fd.sync();
+            }
+            finally {
+                await fd.close();
+            }
+            await chmod(tmp, file.entry.mode);
+            await utimes(tmp, file.entry.mtime_unix_ms / 1000, file.entry.mtime_unix_ms / 1000);
+        }
+        // The read above is deliberately just before replacement; concurrent
+        // changes detected here fail the pass instead of advancing its baseline.
+        await checkLocal(folder, path, expected);
+        await rename(tmp, target);
+    }
+    finally {
+        await rm(tmp, { force: true });
+    }
+}
+// Extract our existing SDK tar format with dirfd + O_NOFOLLOW, so an agent's
+// parent symlink cannot redirect a batch out of the project. Compare previous
+// hashes at the mutation boundary, retain mtimes, and fsync every replacement.
+export const APPLY_SYNC_TAR = String.raw `
+import os,sys,json,hashlib,tarfile,uuid,stat
+root,archive,changes=sys.argv[1:]
+changes=json.loads(changes)
+rootfd=os.open(root,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
+def parent(p):
+ parts=p.split('/')
+ if not p or any(x in ('','..','.','.git','.shard') for x in parts) or '\\' in p: raise ValueError('unsafe project path')
+ fd=os.dup(rootfd)
+ try:
+  for x in parts[:-1]:
+   try: os.mkdir(x,dir_fd=fd)
+   except FileExistsError: pass
+   child=os.open(x,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=fd)
+   os.close(fd);fd=child
+  return fd,parts[-1]
+ except: os.close(fd);raise
+def identity(fd,name):
+ try: st=os.stat(name,dir_fd=fd,follow_symlinks=False)
+ except FileNotFoundError: return 'absent'
+ if stat.S_ISLNK(st.st_mode):
+  digest=hashlib.sha256(('symlink\0'+os.readlink(name,dir_fd=fd)).encode()).hexdigest();kind='link'
+ elif stat.S_ISREG(st.st_mode):
+  if st.st_size>67108864: raise ValueError('destination grew beyond sync limit')
+  f=os.open(name,os.O_RDONLY|os.O_NOFOLLOW,dir_fd=fd)
+  with os.fdopen(f,'rb') as reader: digest=hashlib.file_digest(reader,'sha256').hexdigest()
+  kind='file'
+ else: raise ValueError('destination is not a file')
+ return digest+':'+str(stat.S_IMODE(st.st_mode)&511)+':'+kind
+try:
+ members={}
+ tar=None
+ if archive:
+  tar=tarfile.open(archive,'r:')
+  for member in tar:
+   if member.name in members or member.name not in changes or not(member.isfile() or member.issym()) or member.size>67108864: raise ValueError('invalid sync archive')
+   members[member.name]=member
+ for p,change in changes.items():
+  fd,name=parent(p)
+  tmp=name+'.shard-tmp-'+uuid.uuid4().hex
+  try:
+   if identity(fd,name)!=change['previous']: raise ValueError(p+' changed during sync')
+   entry=change['next']
+   if entry is None:
+    try: os.unlink(name,dir_fd=fd)
+    except FileNotFoundError: pass
+   else:
+    member=members[p]
+    if entry['symlink']:
+     if not member.issym() or member.linkname!=entry['link_target']: raise ValueError('link mismatch')
+     digest=hashlib.sha256(('symlink\0'+member.linkname).encode()).hexdigest()
+     os.symlink(member.linkname,tmp,dir_fd=fd)
+    else:
+     if not member.isfile() or member.size!=entry['size']: raise ValueError('file mismatch')
+     dst=os.open(tmp,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,entry['mode'],dir_fd=fd)
+     with os.fdopen(dst,'wb') as writer,tar.extractfile(member) as reader:
+      digest=hashlib.sha256()
+      while True:
+       b=reader.read(1048576)
+       if not b: break
+       digest.update(b);writer.write(b)
+      writer.flush();os.fchmod(writer.fileno(),entry['mode']);os.fsync(writer.fileno())
+     digest=digest.hexdigest()
+     os.utime(tmp,ns=(int(entry['mtime_unix_ms']*1000000),)*2,dir_fd=fd,follow_symlinks=False)
+    if digest!=entry['sha256']: raise ValueError('content mismatch')
+    if identity(fd,name)!=change['previous']: raise ValueError(p+' changed during sync')
+    os.replace(tmp,name,src_dir_fd=fd,dst_dir_fd=fd)
+   os.fsync(fd)
+  finally:
+   try: os.unlink(tmp,dir_fd=fd)
+   except FileNotFoundError: pass
+   os.close(fd)
+finally:
+ if tar: tar.close()
+ os.close(rootfd)
+`;
+export async function writeSyncBlob(cell, path, data) {
+    if (data.byteLength === 0)
+        await cell.files.write(path, data, { mode: '0600' });
+    for (let offset = 0; offset < data.byteLength; offset += SYNC_CHUNK_BYTES)
+        await cell.files.write(path, data.subarray(offset, offset + SYNC_CHUNK_BYTES), { mode: '0600', append: offset > 0 });
+}
+export async function applyRemoteSyncBatch(cell, root, files, expected, signal) {
+    const parts = [];
+    const changes = Object.create(null);
+    let size = 0;
+    for (const [path, file] of files) {
+        if (!validSyncPath(path))
+            throw new Error(`unsafe sync path: ${path}`);
+        changes[path] = { previous: contentIdentity(expected[path]), next: file?.entry ?? null };
+        if (file) {
+            verifySyncBytes(file.entry, file.data);
+            const header = tarHeader({ path, type: file.entry.symlink ? 'symlink' : 'file', size: file.data.length, mode: file.entry.mode, linkname: file.entry.link_target, mtime: Math.floor(file.entry.mtime_unix_ms / 1000) });
+            const padding = tarPadding(file.data.length);
+            parts.push(header, file.data, padding);
+            size += header.length + file.data.length + padding.length;
+        }
+    }
+    const end = tarEnd(size);
+    const archive = new Uint8Array(size + end.length);
+    let at = 0;
+    for (const part of parts) {
+        archive.set(part, at);
+        at += part.length;
+    }
+    archive.set(end, at);
+    const temp = `/tmp/shard-sync-${randomUUID()}.tar`;
+    try {
+        await writeSyncBlob(cell, temp, archive);
+        const result = await cell.exec.run(['python3', '-c', APPLY_SYNC_TAR, root, temp, JSON.stringify(changes)], { timeoutMs: 60_000, maxOutputBytes: 4096, signal });
+        if (result.exitCode !== 0)
+            throw new Error(`project sync batch failed: ${result.stderr.trim()}`);
+    }
+    finally {
+        await cell.files.remove(temp).catch(() => undefined);
+    }
+}

package/dist/project-sync.d.ts

@@ -0,0 +1,38 @@
+import type { Workspace } from '@shardflux/sdk';
+import type { Ctx } from './commands.js';
+import type { projectIdentity } from './project-identity.js';
+import type { Common } from './project-sync-manifest.js';
+import type { SyncCell } from './project-sync-transfer.js';
+export interface ProjectSyncConflict {
+    path: string;
+    copy: string | null;
+    losingDeleted: boolean;
+    agent: string;
+    winner: 'local' | 'remote';
+    at: number;
+}
+export interface ProjectSyncState {
+    common: Common;
+    conflicts: ProjectSyncConflict[];
+}
+export interface ProjectSync {
+    root: string;
+    conflicts(): readonly ProjectSyncConflict[];
+    reconcile(): Promise<void>;
+    stop(): Promise<void>;
+}
+export { projectWorkspacePath } from './agent-repository.js';
+/** L8 can read these reports without creating a watcher or touching a VM. */
+export declare function projectSyncConflicts(dir: string, key?: string): ProjectSyncConflict[];
+/** Core reconciler used by the attach module and by the real Linux harness. */
+export declare function projectReconciler(options: {
+    folder: string;
+    root: string;
+    agent: string;
+    cell: SyncCell;
+    state: ProjectSyncState;
+    save: (state: ProjectSyncState) => void;
+    report: (message: string) => void;
+    signal?: AbortSignal;
+}): () => Promise<void>;
+export declare function startProjectSync(ctx: Ctx, ws: Workspace, project: ReturnType<typeof projectIdentity>): Promise<ProjectSync>;