@shardflux/cli 0.14.2 → 0.14.3

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.14.3
+
+- Poll device requests and deliver approvals independently of transcript mirroring, so a slow or failed copy cannot delay file-push approvals. Isolate mirror failures by conversation, retry with exponential backoff, and log each failure once per conversation revision with the affected entry name.
+- Skip unknown, nested, symlink and oversized transcript entries while mirroring safe top-level JSONL and JSON files. Log each skipped name once per conversation, preserve the empty append-lock exception, and refuse symlink destinations or ancestors inside the stash.
+
 ## 0.14.2
 
 - `shard agents open` accepts conversation keys, VM keys and conversation ids, reattaching the existing Claude Code or Codex session in its conversation worktree. The menu bar's Open action uses the same path; legacy agents retain their existing behavior.

package/README.md

@@ -81,6 +81,7 @@ Your branch opens in `.shard/worktrees/<name>`, syncs both ways, and its dev ser
 Ctrl-C stops local sync. Your main checkout stays available for your own work.
 Device tools ask Allow once or Always for this project on your Mac before preparing the conversation’s local worktree. Live transcripts stay in
 `~/.config/shardflux/conversations/`; ended sessions enter Claude Code’s or Codex’s native history. Transcript files up to 64 MiB are supported (0.14.0+).
+From 0.14.3, transcript copying runs independently of device approvals. Unknown entries are skipped, and failed copies retry with backoff; diagnostics name the conversation and affected entry.
 
 For application workspace commands, `shard signup` creates an account and saves a project key, or set
 `SHARDFLUX_API_KEY` to a key from the console and run `shard login`. `shard --help` lists every command.

package/dist/args.d.ts

@@ -1,4 +1,4 @@
-export declare const CLI_VERSION = "0.14.2";
+export declare const CLI_VERSION = "0.14.3";
 export interface OptionSpec {
     type: 'string' | 'boolean';
     short?: string;

package/dist/args.js

@@ -12,7 +12,7 @@
 import { parseArgs } from 'node:util';
 import { ACCOUNT_COMMANDS } from "./account-args.js";
 import { UsageError } from "./errors.js";
-export const CLI_VERSION = '0.14.2';
+export const CLI_VERSION = '0.14.3';
 export const GLOBAL_OPTIONS = {
     'api-url': { type: 'string', value: '<url>', description: 'API base URL (default $SHARDFLUX_API_URL, else https://api.shardflux.dev).' },
     json: { type: 'boolean', description: 'Print machine-readable JSON on stdout (errors as JSON on stderr).' },

package/dist/conversation-stash.d.ts

@@ -4,8 +4,18 @@ import type { Conversation, LocalConversation } from './conversations.js';
 export declare function rewriteTranscript(bytes: Uint8Array, remoteCwd: string, localCwd: string): Uint8Array;
 export declare function transcriptDestination(c: Conversation, folder: string, filename: string, env: Record<string, string | undefined>): string;
 /** Keep growing copies away from native session discovery until the control plane ends the conversation. */
-export declare function mirrorConversation(ctx: Ctx, record: LocalConversation, source: Workspace): Promise<void>;
+export declare function mirrorConversation(ctx: Ctx, record: LocalConversation, source: Workspace, skipped?: Set<string>): Promise<void>;
 export declare function handoffConversation(ctx: Pick<Ctx, 'io' | 'store' | 'apiUrl' | 'signal'>, c: Conversation, folder: string, dir: string, source?: Pick<Workspace, 'key'> | {
     hub: string;
 }): Promise<void>;
-export declare function mirrorProjectConversations(ctx: Ctx, mirrored?: Map<string, string>): Promise<void>;
+export declare class ConversationMirrorState {
+    readonly mirrored: Map<string, string>;
+    readonly skipped: Map<string, Set<string>>;
+    readonly retries: Map<string, {
+        revision: string;
+        failures: number;
+        retryAt: number;
+        reported: Set<string>;
+    }>;
+}
+export declare function mirrorProjectConversations(ctx: Ctx, state?: ConversationMirrorState): Promise<void>;

package/dist/conversation-stash.js

@@ -1,5 +1,5 @@
 import { createHash } from 'node:crypto';
-import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync } from 'node:fs';
+import { closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
 import { homedir } from 'node:os';
 import { conversationName, listConversations, localConversations, rememberConversation } from "./conversations.js";
@@ -45,35 +45,79 @@ function privateDirectory(path) {
         at = parent;
     }
     if (existsSync(at) && lstatSync(at).isSymbolicLink())
-        throw new Error('Transcript directory must not be a symlink.');
+        throw new Error(`Transcript directory must not be a symlink: ${at}`);
     mkdirSync(path, { recursive: true, mode: 0o700 });
     if (lstatSync(path).isSymbolicLink())
-        throw new Error('Transcript directory must not be a symlink.');
+        throw new Error(`Transcript directory must not be a symlink: ${path}`);
+}
+function readStashFile(file) {
+    const fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+    try {
+        if (!fstatSync(fd).isFile())
+            throw new Error(`Stash entry is not a regular file: ${file}`);
+        return readFileSync(fd);
+    }
+    finally {
+        closeSync(fd);
+    }
 }
 /** Keep growing copies away from native session discovery until the control plane ends the conversation. */
-export async function mirrorConversation(ctx, record, source) {
+export async function mirrorConversation(ctx, record, source, skipped = new Set()) {
     const c = record.conversation;
-    const dir = join(ctx.store.dir, 'conversations', c.project_id, c.id), root = `/shardflux/project/transcripts/${c.id}`;
+    if (!/^[a-zA-Z0-9_-]+$/.test(c.project_id) || !/^conv_[a-z0-9]{26}$/.test(c.id))
+        throw new Error('Transcript stash identity is unsafe.');
+    let dir = ctx.store.dir;
+    // Check each stash component before creating children; no ancestor inside the stash may be a symlink.
     privateDirectory(dir);
+    for (const part of ['conversations', c.project_id, c.id]) {
+        dir = join(dir, part);
+        privateDirectory(dir);
+    }
+    const root = `/shardflux/project/transcripts/${c.id}`;
+    const skip = (name, reason) => {
+        if (skipped.has(name))
+            return;
+        skipped.add(name);
+        ctx.io.stderr.write(`Transcript mirror ${c.id}: skipped entry ${JSON.stringify(name)} (${reason}).\n`);
+    };
     const cell = source.cell({ wake: null });
+    const close = () => cell.close();
+    ctx.signal.addEventListener('abort', close, { once: true });
     try {
+        ctx.signal.throwIfAborted();
         const manifest = await cell.files.manifest(root, { maxFileBytes: MAX, signal: ctx.signal });
-        if (manifest.skipped.length)
-            throw new Error('Project transcript copy includes files outside the mirror bound.');
+        for (const entry of manifest.skipped)
+            skip(entry.path, entry.reason);
         let count = 0;
         for (const entry of manifest.entries) {
             // Flush holds this empty regular file while appending the transcript.
             // It is synchronization metadata, not native history to mirror.
             if (entry.path === '.lock' && !entry.symlink && entry.size === 0 && entry.sha256 === 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
                 continue;
-            if (entry.symlink || entry.path.includes('/') || !/^[a-zA-Z0-9_.-]+\.(jsonl|json)$/.test(entry.path))
-                throw new Error('Project transcript copy has an unsafe entry.');
-            const file = join(dir, entry.path), expected = existsSync(file) ? createHash('sha256').update(readFileSync(file)).digest('hex') : '';
-            if (expected !== entry.sha256) {
-                const bytes = await cell.files.read(`${root}/${entry.path}`, { length: Math.min(MAX + 1, entry.size + 1) });
-                if (bytes.length !== entry.size || createHash('sha256').update(bytes).digest('hex') !== entry.sha256)
-                    throw new Error('Transcript changed during copying; retaining the previous stash.');
-                writePrivateFile(dir, file, new TextDecoder('utf8', { fatal: true }).decode(bytes));
+            if (entry.symlink || entry.path.includes('..') || entry.path.includes('/') || !/^[a-zA-Z0-9_.-]+\.(jsonl|json)$/.test(entry.path)) {
+                skip(entry.path, 'not a safe top-level JSONL or JSON file');
+                continue;
+            }
+            if (!Number.isSafeInteger(entry.size) || entry.size < 0 || entry.size > MAX) {
+                skip(entry.path, 'outside the 64 MiB mirror bound');
+                continue;
+            }
+            try {
+                const file = join(dir, entry.path), existing = lstatSync(file, { throwIfNoEntry: false });
+                if (existing && !existing.isFile()) {
+                    skip(entry.path, 'stash destination is not a regular file');
+                    continue;
+                }
+                const expected = existing ? createHash('sha256').update(readStashFile(file)).digest('hex') : '';
+                if (expected !== entry.sha256) {
+                    const bytes = await cell.files.read(`${root}/${entry.path}`, { length: entry.size + 1 });
+                    if (bytes.length !== entry.size || createHash('sha256').update(bytes).digest('hex') !== entry.sha256)
+                        throw new Error('Transcript changed during copying; retaining the previous stash.');
+                    writePrivateFile(dir, file, new TextDecoder('utf8', { fatal: true }).decode(bytes));
+                }
+            }
+            catch (e) {
+                throw new Error(`Entry ${JSON.stringify(entry.path)}: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
             }
             if (entry.path.endsWith('.jsonl'))
                 count++;
@@ -84,26 +128,27 @@ export async function mirrorConversation(ctx, record, source) {
             const binding = await conversationClient(ctx).workspaces.get(c.workspace_id);
             if (!binding.data.deleted_at)
                 return;
-            const meta = JSON.parse(readFileSync(join(dir, 'meta.json'), 'utf8'));
-            if (!c.harness_session_id || meta.harness !== c.harness || meta.harness_session_id !== c.harness_session_id || !/^[a-zA-Z0-9_.-]+\.jsonl$/.test(meta.file))
-                throw new Error('Transcript handoff metadata does not match the ended conversation.');
-            const bytes = readFileSync(join(dir, meta.file));
+            const meta = JSON.parse(readStashFile(join(dir, 'meta.json')).toString('utf8'));
+            if (!meta || !c.harness_session_id || meta.harness !== c.harness || meta.harness_session_id !== c.harness_session_id || typeof meta.file !== 'string' || meta.file.includes('..') || !/^[a-zA-Z0-9_.-]+\.jsonl$/.test(meta.file))
+                throw new Error('Entry "meta.json": transcript handoff metadata does not match the ended conversation.');
+            const bytes = readStashFile(join(dir, meta.file));
             if (bytes.length !== meta.bytes || createHash('sha256').update(bytes).digest('hex') !== meta.sha256)
-                throw new Error('Transcript handoff checkpoint differs from its durable metadata.');
+                throw new Error(`Entry ${JSON.stringify(meta.file)}: transcript handoff checkpoint differs from its durable metadata.`);
             await handoffConversation(ctx, { ...c, worktree_path: meta.cwd }, record.folder, dir, source);
         }
     }
     finally {
+        ctx.signal.removeEventListener('abort', close);
         cell.close();
     }
 }
 export async function handoffConversation(ctx, c, folder, dir, source) {
     if (c.state !== 'done' && c.state !== 'archived')
         throw new Error('Only an ended conversation can enter native history.');
-    const files = readdirSync(dir).filter(f => f.endsWith('.jsonl'));
+    const files = readdirSync(dir).filter(f => !f.includes('..') && /^[a-zA-Z0-9_.-]+\.jsonl$/.test(f) && lstatSync(join(dir, f)).isFile());
     if (!files.length)
         throw new Error('Conversation has no transcript to hand off.');
-    const writes = files.map(name => ({ dest: transcriptDestination(c, folder, name, ctx.io.env), bytes: rewriteTranscript(readFileSync(join(dir, name)), c.worktree_path, folder) }));
+    const writes = files.map(name => ({ dest: transcriptDestination(c, folder, name, ctx.io.env), bytes: rewriteTranscript(readStashFile(join(dir, name)), c.worktree_path, folder) }));
     // Check every destination before the first write. Preserve local native history on any collision.
     for (const { dest, bytes } of writes) {
         if (existsSync(dest) && (lstatSync(dest).isSymbolicLink() || !readFileSync(dest).equals(Buffer.from(bytes))))
@@ -123,7 +168,12 @@ export async function handoffConversation(ctx, c, folder, dir, source) {
     rememberConversation(ctx, c, folder, true);
     writePrivateFile(dir, join(dir, 'handoff.json'), JSON.stringify({ conversation_id: c.id, branch: c.branch, destinations: writes.map(w => w.dest), at: new Date().toISOString() }) + '\n');
 }
-export async function mirrorProjectConversations(ctx, mirrored = new Map()) {
+export class ConversationMirrorState {
+    mirrored = new Map();
+    skipped = new Map();
+    retries = new Map();
+}
+export async function mirrorProjectConversations(ctx, state = new ConversationMirrorState()) {
     const records = localConversations(ctx);
     if (!Object.keys(records).length)
         return;
@@ -133,30 +183,57 @@ export async function mirrorProjectConversations(ctx, mirrored = new Map()) {
         if (!c || record.handed_off || !c.harness_session_id)
             continue;
         const revision = JSON.stringify([c.workspace_id, c.harness_session_id, c.state, c.durable, c.last_activity_at, c.summary]);
-        if (mirrored.get(c.id) === revision)
+        if (state.mirrored.get(c.id) === revision)
             continue;
-        rememberConversation(ctx, c, record.folder);
-        let source;
-        for (const candidate of [c, ...all.filter(other => other.project_id === c.project_id && other.id !== c.id)]) {
-            try {
-                const ws = await conversationClient(ctx).workspaces.get(candidate.workspace_id);
-                if (!ws.data.deleted_at && ws.ready) {
-                    source = ws;
-                    break;
+        let retry = state.retries.get(c.id);
+        if (!retry || retry.revision !== revision) {
+            retry = { revision, failures: 0, retryAt: 0, reported: new Set() };
+            state.retries.set(c.id, retry);
+        }
+        if (Date.now() < retry.retryAt)
+            continue;
+        try {
+            rememberConversation(ctx, c, record.folder);
+            let source;
+            for (const candidate of [c, ...all.filter(other => other.project_id === c.project_id && other.id !== c.id)]) {
+                try {
+                    const ws = await conversationClient(ctx).workspaces.get(candidate.workspace_id);
+                    if (!ws.data.deleted_at && ws.ready) {
+                        source = ws;
+                        break;
+                    }
                 }
+                catch { /* Another binding may be available. */ }
+            }
+            if (!source) {
+                const client = conversationClient(ctx);
+                const reader = await client.conversations.spaceReader(c.project_id, { signal: ctx.signal });
+                source = await client.workspaces.get(reader.workspace.id);
+                if (!source.ready)
+                    await source.wake({ timeoutMs: ctx.wakeTimeoutMs, signal: ctx.signal });
+            }
+            let skipped = state.skipped.get(c.id);
+            if (!skipped) {
+                skipped = new Set();
+                state.skipped.set(c.id, skipped);
             }
-            catch { /* Another binding may be available. */ }
+            await mirrorConversation(ctx, { ...record, conversation: c }, source, skipped);
+            retry.failures = 0;
+            retry.retryAt = 0;
+            // An ended record may precede its VM tombstone; retry until the handoff receipt is written.
+            if (c.state !== 'done' && c.state !== 'archived')
+                state.mirrored.set(c.id, revision);
         }
-        if (!source) {
-            const client = conversationClient(ctx);
-            const reader = await client.conversations.spaceReader(c.project_id, { signal: ctx.signal });
-            source = await client.workspaces.get(reader.workspace.id);
-            if (!source.ready)
-                await source.wake({ timeoutMs: ctx.wakeTimeoutMs, signal: ctx.signal });
+        catch (e) {
+            if (ctx.signal.aborted)
+                return;
+            const message = e instanceof Error ? e.message : String(e);
+            const backoff = Math.min(300_000, 5000 * 2 ** Math.min(retry.failures++, 6));
+            retry.retryAt = Date.now() + backoff;
+            if (!retry.reported.has(message)) {
+                retry.reported.add(message);
+                ctx.io.stderr.write(`Transcript mirror ${c.id}: ${message} (retrying in ${backoff / 1000}s).\n`);
+            }
         }
-        await mirrorConversation(ctx, { ...record, conversation: c }, source);
-        // An ended record may precede its VM tombstone; retry until the handoff receipt is written.
-        if (c.state !== 'done' && c.state !== 'archived')
-            mirrored.set(c.id, revision);
     }
 }

package/dist/device-helper.js

@@ -1,6 +1,6 @@
 import { setTimeout as delay } from 'node:timers/promises';
 import { DeviceExecutor, askConnectedAcp } from "./device-bridge.js";
-import { mirrorProjectConversations } from "./conversation-stash.js";
+import { ConversationMirrorState, mirrorProjectConversations } from "./conversation-stash.js";
 import { withTargetLock } from "./target-lock.js";
 import { join } from 'node:path';
 import { conversationsUnavailable } from "./conversations.js";
@@ -49,7 +49,7 @@ export async function runDeviceHelper(ctx) {
         });
     };
     const working = new Map();
-    const mirrored = new Map();
+    const mirrored = new ConversationMirrorState();
     const deviceFile = join(ctx.store.dir, 'device-id');
     let deviceId;
     try {
@@ -61,8 +61,32 @@ export async function runDeviceHelper(ctx) {
     }
     const executor = new DeviceExecutor(ctx, ask, deviceId);
     const report = (e) => { ctx.io.stderr.write(`Device bridge: ${e instanceof Error ? e.message : String(e)}\n`); };
+    let mirroring;
     try {
         return await withTargetLock(join(ctx.store.dir, 'device-helper-lock'), async () => {
+            // A slow or failed transcript copy must not delay heartbeats, request polling or approvals.
+            mirroring = (async () => {
+                let failures = 0;
+                const reported = new Set();
+                while (!signal.aborted) {
+                    let wait = 1000;
+                    try {
+                        await mirrorProjectConversations(ctx, mirrored);
+                        failures = 0;
+                    }
+                    catch (e) {
+                        if (signal.aborted)
+                            break;
+                        wait = Math.min(300_000, 5000 * 2 ** Math.min(failures++, 6));
+                        const message = e instanceof Error ? e.message : String(e);
+                        if (!conversationsUnavailable(e) && !reported.has(message)) {
+                            reported.add(message);
+                            ctx.io.stderr.write(`Transcript mirror discovery: ${message} (retrying in ${wait / 1000}s).\n`);
+                        }
+                    }
+                    await delay(wait, undefined, { signal }).catch(() => undefined);
+                }
+            })();
             while (!signal.aborted) {
                 try {
                     for (const projectId of new Set(Object.values(localConversations(ctx)).map(r => r.project_id))) {
@@ -80,7 +104,6 @@ export async function runDeviceHelper(ctx) {
                         } while (cursor && !signal.aborted);
                     }
                     await executor.reap();
-                    await mirrorProjectConversations(ctx, mirrored);
                 }
                 catch (e) {
                     if (!signal.aborted && !conversationsUnavailable(e))
@@ -99,6 +122,7 @@ export async function runDeviceHelper(ctx) {
         pending.clear();
         await executor.close();
         await Promise.all([...working.values()]);
+        await mirroring;
         await input.catch(() => undefined);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shardflux/cli",
-  "version": "0.14.2",
+  "version": "0.14.3",
   "type": "module",
   "description": "shard: the Shardflux command line. Create an account, sign in, set up keys and billing, then open, run commands in, move files to, suspend, resume and fork serverless agent workspaces.",
   "license": "Apache-2.0",