@lifestreamdynamics/vault-cli 1.3.10 → 1.4.1

package/dist/commands/audit.js

@@ -2,7 +2,7 @@ import chalk from 'chalk';
 import fs from 'node:fs';
 import path from 'node:path';
 import os from 'node:os';
-import { AuditLogger } from '@lifestreamdynamics/vault-sdk';
+import { AuditLogger } from '@lifestreamdynamics/vault-sdk/audit';
 import { addGlobalFlags, resolveFlags } from '../utils/flags.js';
 import { createOutput, handleError } from '../utils/output.js';
 const DEFAULT_LOG_PATH = path.join(os.homedir(), '.lsvault', 'audit.log');

package/dist/commands/sync.js

@@ -9,7 +9,7 @@ import { formatUptime } from '../utils/format.js';
 import { loadSyncConfigs, createSyncConfig, deleteSyncConfig, getSyncConfig, } from '../sync/config.js';
 import { deleteSyncState, loadSyncState, saveSyncState, hashFileContent, buildRemoteFileState } from '../sync/state.js';
 import { resolveIgnorePatterns } from '../sync/ignore.js';
-import { scanLocalFiles, scanRemoteFiles, executePull, executePush, computePullDiff, computePushDiff, } from '../sync/engine.js';
+import { scanLocalFiles, scanRemoteFiles, executePull, executePush, computePullDiff, computePushDiff, resolveConcurrency, } from '../sync/engine.js';
 import { formatDiff } from '../sync/diff.js';
 import { createWatcher } from '../sync/watcher.js';
 import { createRemotePoller } from '../sync/remote-poller.js';
@@ -153,7 +153,8 @@ Sync modes:
     // sync pull <syncId>
     addGlobalFlags(sync.command('pull')
         .description('Pull remote changes to local directory')
-        .argument('<syncId>', 'Sync configuration ID'))
+        .argument('<syncId>', 'Sync configuration ID')
+        .option('--concurrency <n>', 'Max concurrent file transfers (1-16, default 4)', (v) => parseInt(v, 10)))
         .action(async (syncId, _opts) => {
         const flags = resolveFlags(_opts);
         const out = createOutput(flags);
@@ -211,12 +212,13 @@ Sync modes:
             if (flags.verbose) {
                 out.status(formatDiff(diff));
             }
+            const concurrency = resolveConcurrency(_opts.concurrency);
             out.startSpinner(`Pulling ${totalOps} file(s)...`);
             const result = await executePull(client, config, diff, (progress) => {
                 if (progress.phase === 'transferring' && progress.currentFile) {
                     out.startSpinner(`[${progress.current}/${progress.total}] ${progress.currentFile}`);
                 }
-            });
+            }, concurrency);
             if (result.errors.length > 0) {
                 out.failSpinner(`Pull completed with ${result.errors.length} error(s)`);
                 for (const err of result.errors) {
@@ -241,7 +243,8 @@ Sync modes:
     // sync push <syncId>
     addGlobalFlags(sync.command('push')
         .description('Push local changes to remote vault')
-        .argument('<syncId>', 'Sync configuration ID'))
+        .argument('<syncId>', 'Sync configuration ID')
+        .option('--concurrency <n>', 'Max concurrent file transfers (1-16, default 4)', (v) => parseInt(v, 10)))
         .action(async (syncId, _opts) => {
         const flags = resolveFlags(_opts);
         const out = createOutput(flags);
@@ -299,12 +302,13 @@ Sync modes:
             if (flags.verbose) {
                 out.status(formatDiff(diff));
             }
+            const concurrency = resolveConcurrency(_opts.concurrency);
             out.startSpinner(`Pushing ${totalOps} file(s)...`);
             const result = await executePush(client, config, diff, (progress) => {
                 if (progress.phase === 'transferring' && progress.currentFile) {
                     out.startSpinner(`[${progress.current}/${progress.total}] ${progress.currentFile}`);
                 }
-            });
+            }, concurrency);
             if (result.errors.length > 0) {
                 out.failSpinner(`Push completed with ${result.errors.length} error(s)`);
                 for (const err of result.errors) {

package/dist/sync/engine.d.ts

@@ -30,12 +30,17 @@ export declare function scanLocalFiles(localPath: string, ignorePatterns: string
  * Returns a map of doc paths -> FileState.
  */
 export declare function scanRemoteFiles(client: LifestreamVaultClient, vaultId: string, ignorePatterns: string[]): Promise<Record<string, FileState>>;
+/**
+ * Validates and clamps a user-supplied concurrency value. Throws on invalid
+ * values so the CLI can surface a clear error before kicking off any I/O.
+ */
+export declare function resolveConcurrency(value: number | undefined): number;
 /**
  * Execute a pull operation: download remote changes to local.
  */
-export declare function executePull(client: LifestreamVaultClient, config: SyncConfig, diff: SyncDiff, onProgress?: ProgressCallback): Promise<SyncResult>;
+export declare function executePull(client: LifestreamVaultClient, config: SyncConfig, diff: SyncDiff, onProgress?: ProgressCallback, concurrency?: number): Promise<SyncResult>;
 /**
  * Execute a push operation: upload local changes to remote.
  */
-export declare function executePush(client: LifestreamVaultClient, config: SyncConfig, diff: SyncDiff, onProgress?: ProgressCallback): Promise<SyncResult>;
+export declare function executePush(client: LifestreamVaultClient, config: SyncConfig, diff: SyncDiff, onProgress?: ProgressCallback, concurrency?: number): Promise<SyncResult>;
 export { computePullDiff, computePushDiff, type SyncDiff, type SyncDiffEntry };

package/dist/sync/engine.js

@@ -54,7 +54,7 @@ export async function scanRemoteFiles(client, vaultId, ignorePatterns) {
         if (!shouldIgnore(doc.path, ignorePatterns)) {
             files[doc.path] = {
                 path: doc.path,
-                hash: '', // We don't have content hash from list; will use mtime for comparison
+                hash: doc.contentHash,
                 mtime: doc.fileModifiedAt,
                 size: doc.sizeBytes,
             };
@@ -71,12 +71,31 @@ function atomicWriteFileSync(targetPath, content, encoding = 'utf-8') {
     fs.writeFileSync(tmpFile, content, encoding);
     fs.renameSync(tmpFile, targetPath);
 }
+/**
+ * Default in-flight transfer count. A small number flattens the load1 spike
+ * a full-vault `sync pull` causes on the API host without making single
+ * transfers measurably slower.
+ */
+const DEFAULT_TRANSFER_CONCURRENCY = 4;
+const MAX_TRANSFER_CONCURRENCY = 16;
+/**
+ * Validates and clamps a user-supplied concurrency value. Throws on invalid
+ * values so the CLI can surface a clear error before kicking off any I/O.
+ */
+export function resolveConcurrency(value) {
+    if (value === undefined)
+        return DEFAULT_TRANSFER_CONCURRENCY;
+    if (!Number.isInteger(value) || value < 1 || value > MAX_TRANSFER_CONCURRENCY) {
+        throw new Error(`--concurrency must be an integer between 1 and ${MAX_TRANSFER_CONCURRENCY} (got ${value})`);
+    }
+    return value;
+}
 /**
  * Shared sync operation executor used by both pull and push.
  * Handles result initialization, state loading, progress callbacks,
  * quota error handling, state saving, and lastSync update.
  */
-async function executeSyncOperation(config, diff, handlers, onProgress) {
+async function executeSyncOperation(config, diff, handlers, onProgress, concurrency = DEFAULT_TRANSFER_CONCURRENCY) {
     const result = {
         filesUploaded: 0,
         filesDownloaded: 0,
@@ -87,7 +106,12 @@ async function executeSyncOperation(config, diff, handlers, onProgress) {
     const state = loadSyncState(config.id);
     const allOps = [...handlers.transfers, ...handlers.deletes];
     let current = 0;
-    for (const entry of handlers.transfers) {
+    // Once a quota error is hit anywhere in the pool we stop submitting new
+    // work but let in-flight transfers drain to keep state consistent.
+    let stopSubmitting = false;
+    async function runOne(entry) {
+        if (stopSubmitting)
+            return;
         current++;
         onProgress?.({
             phase: 'transferring',
@@ -112,13 +136,25 @@ async function executeSyncOperation(config, diff, handlers, onProgress) {
         }
         catch (err) {
             const message = err instanceof Error ? err.message : String(err);
+            result.errors.push({ path: entry.path, error: message });
             if (isQuotaError(message)) {
-                result.errors.push({ path: entry.path, error: message });
-                break; // Stop immediately on quota errors
+                stopSubmitting = true;
             }
-            result.errors.push({ path: entry.path, error: message });
         }
     }
+    // Bounded async pool. Workers race for entries off the queue tail; once
+    // the queue is empty (or stopSubmitting is set), each worker exits and
+    // Promise.all resolves only after every in-flight transfer has settled.
+    const queue = handlers.transfers.slice();
+    const poolSize = Math.min(Math.max(1, concurrency), Math.max(1, queue.length));
+    await Promise.all(Array.from({ length: poolSize }, async () => {
+        while (!stopSubmitting) {
+            const entry = queue.shift();
+            if (!entry)
+                return;
+            await runOne(entry);
+        }
+    }));
     for (const entry of handlers.deletes) {
         current++;
         onProgress?.({
@@ -154,7 +190,7 @@ async function executeSyncOperation(config, diff, handlers, onProgress) {
 /**
  * Execute a pull operation: download remote changes to local.
  */
-export async function executePull(client, config, diff, onProgress) {
+export async function executePull(client, config, diff, onProgress, concurrency) {
     return executeSyncOperation(config, diff, {
         transfers: diff.downloads,
         deletes: diff.deletes,
@@ -175,12 +211,12 @@ export async function executePull(client, config, diff, onProgress) {
                 fs.unlinkSync(localFile);
             }
         },
-    }, onProgress);
+    }, onProgress, concurrency);
 }
 /**
  * Execute a push operation: upload local changes to remote.
  */
-export async function executePush(client, config, diff, onProgress) {
+export async function executePush(client, config, diff, onProgress, concurrency) {
     return executeSyncOperation(config, diff, {
         transfers: diff.uploads,
         deletes: diff.deletes,
@@ -194,7 +230,7 @@ export async function executePush(client, config, diff, onProgress) {
         async deleteFile(entry, cfg) {
             await retryWithBackoff(() => client.documents.delete(cfg.vaultId, entry.path));
         },
-    }, onProgress);
+    }, onProgress, concurrency);
 }
 /**
  * Retry a function with exponential backoff (max 3 retries).

package/dist/sync/remote-poller.js

@@ -22,62 +22,91 @@ export function createRemotePoller(client, config, options) {
             return; // Skip if previous poll still in progress
         polling = true;
         try {
-            const remoteDocs = await client.documents.list(config.vaultId);
             const state = loadSyncState(config.id);
             let changes = 0;
-            for (const doc of remoteDocs) {
-                if (shouldIgnore(doc.path, ignorePatterns))
-                    continue;
-                const lastRemote = state.remote[doc.path];
-                // Detect remote changes by comparing mtime
-                const remoteChanged = !lastRemote || doc.fileModifiedAt !== lastRemote.mtime;
-                if (!remoteChanged)
-                    continue;
-                // Fetch the full content
-                const { content } = await client.documents.get(config.vaultId, doc.path);
-                const remoteHash = hashFileContent(content);
-                // Skip if hash hasn't actually changed
-                if (lastRemote && remoteHash === lastRemote.hash) {
-                    // Update mtime in state but skip file operations
-                    state.remote[doc.path] = buildRemoteFileState(doc.path, content, doc.fileModifiedAt);
+            let stateMutated = false;
+            // Build the known state from persisted remote hashes + stored list ETag.
+            const known = {
+                hashes: Object.fromEntries(Object.entries(state.remote).map(([k, v]) => [k, v.hash])),
+                listEtag: state.remoteListEtag,
+            };
+            const sync = await client.documents.syncList(config.vaultId, known);
+            // Steady-state: server confirmed nothing changed — skip all per-doc work.
+            if (sync.vaultUnchanged)
+                return;
+            // Persist the new list ETag so the next poll can 304.
+            state.remoteListEtag = sync.listEtag;
+            stateMutated = true;
+            // Process added/changed documents.
+            for (const change of sync.changes) {
+                if (shouldIgnore(change.path, ignorePatterns))
                     continue;
+                const lastRemote = state.remote[change.path];
+                let content;
+                let remoteHash;
+                if (lastRemote) {
+                    // Conditional GET — server can still 304 us if our hash is current
+                    // (rare: syncList classified this as 'changed' but the GET hash matches
+                    // our last-known hash — possible race between list and get).
+                    const result = await client.documents.get(config.vaultId, change.path, {
+                        ifNoneMatch: `"${lastRemote.hash}"`,
+                    });
+                    if (result.notModified) {
+                        // Server confirmed our copy is current despite differing list hash.
+                        // Update mtime only; no file write needed.
+                        if (lastRemote.mtime !== change.fileModifiedAt) {
+                            state.remote[change.path] = { ...lastRemote, mtime: change.fileModifiedAt };
+                            stateMutated = true;
+                        }
+                        continue;
+                    }
+                    content = result.content;
+                    remoteHash = hashFileContent(content);
                 }
-                const localFile = path.join(config.localPath, doc.path);
+                else {
+                    // First-time entry: unconditional GET.
+                    const fetched = await client.documents.get(config.vaultId, change.path);
+                    content = fetched.content;
+                    remoteHash = hashFileContent(content);
+                }
+                const localFile = path.join(config.localPath, change.path);
                 const localExists = fs.existsSync(localFile);
                 if (localExists) {
                     const localContent = fs.readFileSync(localFile, 'utf-8');
                     const localHash = hashFileContent(localContent);
                     if (localHash === remoteHash) {
                         // Content is already the same — just update state
-                        state.local[doc.path] = { path: doc.path, hash: localHash, mtime: new Date().toISOString(), size: Buffer.byteLength(localContent) };
-                        state.remote[doc.path] = buildRemoteFileState(doc.path, content, doc.fileModifiedAt);
+                        state.local[change.path] = { path: change.path, hash: localHash, mtime: new Date().toISOString(), size: Buffer.byteLength(localContent) };
+                        state.remote[change.path] = buildRemoteFileState(change.path, content, change.fileModifiedAt);
+                        stateMutated = true;
                         continue;
                     }
                     // Check for conflict
-                    const lastLocal = state.local[doc.path];
-                    const localState = { path: doc.path, hash: localHash, mtime: fs.statSync(localFile).mtime.toISOString(), size: Buffer.byteLength(localContent) };
-                    const remoteState = { path: doc.path, hash: remoteHash, mtime: doc.fileModifiedAt, size: Buffer.byteLength(content) };
+                    const lastLocal = state.local[change.path];
+                    const localState = { path: change.path, hash: localHash, mtime: fs.statSync(localFile).mtime.toISOString(), size: Buffer.byteLength(localContent) };
+                    const remoteState = { path: change.path, hash: remoteHash, mtime: change.fileModifiedAt, size: Buffer.byteLength(content) };
                     if (detectConflict(localState, remoteState, lastLocal, lastRemote)) {
                         const resolution = resolveConflict(config.onConflict, localState, remoteState);
                         let conflictFile = null;
                         if (resolution === 'remote') {
-                            conflictFile = createConflictFile(config.localPath, doc.path, localContent, 'local');
-                            onLocalWrite?.(doc.path);
+                            conflictFile = createConflictFile(config.localPath, change.path, localContent, 'local');
+                            onLocalWrite?.(change.path);
                             const tmpConflict = localFile + '.tmp';
                             fs.writeFileSync(tmpConflict, content, 'utf-8');
                             fs.renameSync(tmpConflict, localFile);
-                            log(`Conflict: ${doc.path} — used remote, saved local as ${conflictFile}`);
+                            log(`Conflict: ${change.path} — used remote, saved local as ${conflictFile}`);
                         }
                         else {
-                            conflictFile = createConflictFile(config.localPath, doc.path, content, 'remote');
-                            await client.documents.put(config.vaultId, doc.path, localContent);
-                            log(`Conflict: ${doc.path} — used local, saved remote as ${conflictFile}`);
+                            conflictFile = createConflictFile(config.localPath, change.path, content, 'remote');
+                            await client.documents.put(config.vaultId, change.path, localContent);
+                            log(`Conflict: ${change.path} — used local, saved remote as ${conflictFile}`);
                         }
-                        onConflictLog?.(formatConflictLog(doc.path, resolution, conflictFile));
-                        state.local[doc.path] = resolution === 'remote' ? remoteState : localState;
-                        state.remote[doc.path] = resolution === 'remote'
-                            ? buildRemoteFileState(doc.path, content, doc.fileModifiedAt)
-                            : buildRemoteFileState(doc.path, localContent, new Date().toISOString());
+                        onConflictLog?.(formatConflictLog(change.path, resolution, conflictFile));
+                        state.local[change.path] = resolution === 'remote' ? remoteState : localState;
+                        state.remote[change.path] = resolution === 'remote'
+                            ? buildRemoteFileState(change.path, content, change.fileModifiedAt)
+                            : buildRemoteFileState(change.path, localContent, new Date().toISOString());
+                        stateMutated = true;
                         changes++;
                         continue;
                     }
@@ -87,40 +116,41 @@ export function createRemotePoller(client, config, options) {
                 if (!fs.existsSync(dir)) {
                     fs.mkdirSync(dir, { recursive: true });
                 }
-                onLocalWrite?.(doc.path);
+                onLocalWrite?.(change.path);
                 const tmpFile = localFile + '.tmp';
                 fs.writeFileSync(tmpFile, content, 'utf-8');
                 fs.renameSync(tmpFile, localFile);
-                log(`Pulled: ${doc.path}`);
+                log(`Pulled: ${change.path}`);
                 changes++;
-                state.local[doc.path] = {
-                    path: doc.path,
+                state.local[change.path] = {
+                    path: change.path,
                     hash: remoteHash,
                     mtime: new Date().toISOString(),
                     size: Buffer.byteLength(content),
                 };
-                state.remote[doc.path] = buildRemoteFileState(doc.path, content, doc.fileModifiedAt);
+                state.remote[change.path] = buildRemoteFileState(change.path, content, change.fileModifiedAt);
+                stateMutated = true;
             }
-            // Check for remote deletions
-            for (const docPath of Object.keys(state.remote)) {
-                if (shouldIgnore(docPath, ignorePatterns))
+            // Process remote deletions.
+            for (const removedPath of sync.removed) {
+                if (shouldIgnore(removedPath, ignorePatterns))
                     continue;
-                const stillExists = remoteDocs.some(d => d.path === docPath);
-                if (!stillExists) {
-                    const localFile = path.join(config.localPath, docPath);
-                    if (fs.existsSync(localFile)) {
-                        fs.unlinkSync(localFile);
-                        log(`Deleted local: ${docPath} (removed from remote)`);
-                        changes++;
-                    }
-                    delete state.local[docPath];
-                    delete state.remote[docPath];
+                const localFile = path.join(config.localPath, removedPath);
+                if (fs.existsSync(localFile)) {
+                    fs.unlinkSync(localFile);
+                    log(`Deleted local: ${removedPath} (removed from remote)`);
+                    changes++;
                 }
+                delete state.local[removedPath];
+                delete state.remote[removedPath];
+                stateMutated = true;
             }
-            if (changes > 0) {
+            if (changes > 0 || stateMutated) {
                 saveSyncState(state);
-                updateLastSync(config.id);
-                log(`Poll complete: ${changes} change(s)`);
+                if (changes > 0) {
+                    updateLastSync(config.id);
+                    log(`Poll complete: ${changes} change(s)`);
+                }
             }
         }
         catch (err) {

package/dist/sync/types.d.ts

@@ -51,6 +51,8 @@ export interface SyncState {
     local: Record<string, FileState>;
     /** Map of document path -> file state for remote files */
     remote: Record<string, FileState>;
+    /** ETag from the most recent successful list response (for conditional polling). */
+    remoteListEtag?: string;
     /** ISO 8601 timestamp when state was last updated */
     updatedAt: string;
 }

package/dist/sync/watcher.js

@@ -101,18 +101,25 @@ export function createWatcher(client, config, options) {
             // Check remote for conflicts in bidirectional mode
             if (config.mode === 'sync' && lastRemote) {
                 try {
-                    const remote = await client.documents.get(config.vaultId, docPath);
-                    const remoteHash = hashFileContent(remote.content);
-                    if (remoteHash !== lastRemote.hash) {
-                        const result = await handleConflict({
-                            absPath, docPath, localContent: content, localHash,
-                            lastLocal, lastRemote,
-                            remoteContent: remote.content, remoteHash,
-                            remoteUpdatedAt: remote.document.updatedAt, state,
-                        });
-                        if (result !== 'skip')
-                            return;
+                    const result = await client.documents.get(config.vaultId, docPath, {
+                        ifNoneMatch: `"${lastRemote.hash}"`,
+                    });
+                    if (!result.notModified) {
+                        // Server returned the body — check if hash differs from our last-known state.
+                        const remoteHash = hashFileContent(result.content);
+                        if (remoteHash !== lastRemote.hash) {
+                            const conflictResult = await handleConflict({
+                                absPath, docPath, localContent: content, localHash,
+                                lastLocal, lastRemote,
+                                remoteContent: result.content, remoteHash,
+                                remoteUpdatedAt: result.document.updatedAt, state,
+                            });
+                            if (conflictResult !== 'skip')
+                                return;
+                        }
+                        // 200 + matching hash: rare list/cache mismatch — fall through to push.
                     }
+                    // 304: remote unchanged since last sync — no conflict possible. Fall through to push.
                 }
                 catch {
                     // Remote check failed — proceed with push

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lifestreamdynamics/vault-cli",
-  "version": "1.3.10",
+  "version": "1.4.1",
   "description": "Command-line interface for Lifestream Vault",
   "engines": {
     "node": ">=22"
@@ -44,7 +44,7 @@
     "prepublishOnly": "npm run build && npm test"
   },
   "dependencies": {
-    "@lifestreamdynamics/vault-sdk": "^2.1.0",
+    "@lifestreamdynamics/vault-sdk": "^2.2.0",
     "chalk": "^5.4.0",
     "chokidar": "^4.0.3",
     "commander": "^13.0.0",