@fizzyflow/wdoublesync_cli 1.0.2 → 1.0.3

package/lib/commands/rebate.js

@@ -0,0 +1,81 @@
+import EndlessVector from '@fizzyflow/endless-vector';
+import { WDoubleSync } from '@fizzyflow/wdoublesync';
+import { CDCStore } from '@fizzyflow/doublesync';
+import { DoubleSyncFSFolder } from '../DoubleSyncFSFolder.js';
+import {
+    makeWalrusSealClient,
+    resolvePath,
+    makeExcludes,
+    formatBytes,
+    countTree,
+} from './shared.js';
+
+export async function rebate(args) {
+    if (!args.vectorId) {
+        throw new Error('vector-id is required for rebate');
+    }
+
+    const destPath = resolvePath(args.path);
+    const fsFolder = new DoubleSyncFSFolder(destPath, makeExcludes(args));
+
+    const counts = await countTree(fsFolder);
+    console.log('  scanning...', counts.files, 'files,', counts.folders, 'folders');
+
+    const wsc = await makeWalrusSealClient(args);
+
+    const evParams = {
+        suiClient: wsc.suiClient,
+        id: args.vectorId,
+        packageId: args.chain,
+        signAndExecuteTransaction: (tx) => wsc.signAndExecuteTransaction(tx),
+        walrusClient: wsc.walrusClient,
+        aggregatorUrl: wsc.aggregatorUrl,
+        senderAddress: wsc.suiMaster.address,
+        signer: wsc.suiMaster._keypair,
+    };
+    if (wsc.sealClient && wsc.suiMaster) {
+        evParams.sealClient = wsc.sealClient;
+    }
+
+    const ev = new EndlessVector(evParams);
+    await ev.initialize();
+
+    console.log('step 1: archiving history...');
+    await ev.archive();
+    console.log('  archived');
+
+    console.log('step 2: burning archives...');
+    await ev.burnArchive();
+    console.log('  burned');
+
+    // archive() and burnArchive() both call ev.reInitialize() — refetch fresh state
+    await ev.initialize();
+
+    console.log('step 3: pushing local state as fresh snapshot...');
+
+    const wdsync = new WDoubleSync({
+        endlessVector: ev,
+        compress: args.compress || 'gzip',
+    });
+
+    // Skip wdsync.initialize() replay entirely: rebate always pushes a full snapshot,
+    // so there's no need to replay the existing (now partly burned) chain.
+    // _lastSnapshot = null forces a full snapshot; _isInitialized = true bypasses replay.
+    wdsync._senderStore = new CDCStore({ copyBytes: false });
+    wdsync._receiverMirror = new CDCStore();
+    wdsync._lastSnapshot = null;
+    wdsync._replayedCount = ev.length;
+    wdsync._isInitialized = true;
+
+    const result = await wdsync.push(fsFolder);
+
+    ev.reInitialize();
+    await ev.initialize();
+
+    console.log('  pushed version', result.version);
+    console.log('  [debug] after push: length=', ev.length, 'burned=', ev.burnedArchiveCount, 'version=', result.version);
+
+    console.log('\nrebate complete:');
+    console.log('  new version:', result.version);
+    console.log('  new on-chain size:', formatBytes(ev.binaryLength));
+}

package/lib/commands/shared.js

@@ -0,0 +1,237 @@
+import { mkdir, writeFile, readFile, readdir, rm } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { homedir } from 'node:os';
+
+import { SuiMaster } from 'suidouble';
+import { DoubleSync, CDCStore, DoubleSyncSnapshot } from '@fizzyflow/doublesync';
+import WalrusSealClient from '../WalrusSealClient.js';
+import { DoubleSyncFSFolder } from '../DoubleSyncFSFolder.js';
+
+const DEFAULT_FS_EXCLUDES = ['node_modules', '.git', '.env', '.DS_Store', '.wdoublesync', '.claude', 'pnpm-lock.yaml', 'package-lock.json'];
+
+export function makeExcludes(args) {
+    if (!args.exclude) return undefined;
+    const extra = args.exclude.split(',').map(s => s.trim());
+    return [...DEFAULT_FS_EXCLUDES, ...extra];
+}
+
+export function resolvePath(p) {
+    if (p.startsWith('~')) return join(homedir(), p.slice(1));
+    return resolve(p);
+}
+
+export function formatBytes(bytes) {
+    if (bytes < 1024) return bytes + ' B';
+    if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
+    return (bytes / (1024 * 1024)).toFixed(1) + ' MB';
+}
+
+export function formatPath(path) {
+    const home = homedir();
+    if (path.startsWith(home)) {
+        return '~' + path.slice(home.length);
+    }
+    return path;
+}
+
+export async function makeWalrusSealClient(args) {
+    const params = { client: args.chain };
+    const key = args.key || process.env.WDOUBLESYNC_KEY;
+    if (key) {
+        params.privateKey = key;
+    } else if (args.phrase) {
+        params.phrase = args.phrase;
+    } else {
+        throw new Error('No signing key. Use --key, --phrase, or WDOUBLESYNC_KEY env var');
+    }
+
+    const suiMaster = new SuiMaster(params);
+    await suiMaster.initialize();
+
+    const client = new WalrusSealClient({ network: args.chain, suiMaster });
+    await client.initialize();
+    return client;
+}
+
+export async function makeReadOnlyWalrusSealClient(chain) {
+    const client = new WalrusSealClient({ network: chain });
+    await client.initialize();
+    return client;
+}
+
+export async function makeClientForRead(args) {
+    const key = args.key || process.env.WDOUBLESYNC_KEY;
+    if (key || args.phrase) {
+        return makeWalrusSealClient(args);
+    }
+    return makeReadOnlyWalrusSealClient(args.chain);
+}
+
+export async function countTree(folder) {
+    let files = 0;
+    let folders = 0;
+    const children = await folder.list();
+    for (const child of children) {
+        if (child.getContent) {
+            files++;
+        } else {
+            folders++;
+            const sub = await countTree(child);
+            files += sub.files;
+            folders += sub.folders;
+        }
+    }
+    return { files, folders };
+}
+
+export function hashContent(bytes) {
+    return createHash('sha256').update(bytes).digest('hex');
+}
+
+export async function readManifest(destPath) {
+    try {
+        const raw = await readFile(join(destPath, '.wdoublesync'), 'utf8');
+        return JSON.parse(raw);
+    } catch { return null; }
+}
+
+export async function writeManifest(destPath, manifest) {
+    await writeFile(join(destPath, '.wdoublesync'), JSON.stringify(manifest, null, 2));
+}
+
+export const DISK_EXCLUDES = new Set(['.wdoublesync', '.git', '.DS_Store', 'node_modules']);
+
+export async function hashDiskTree(dirPath, prefix = []) {
+    const files = {};
+    let entries;
+    try { entries = await readdir(dirPath, { withFileTypes: true }); } catch { return files; }
+    for (const entry of entries) {
+        if (DISK_EXCLUDES.has(entry.name) || entry.name.startsWith('.')) continue;
+        const relParts = [...prefix, entry.name];
+        const relPath = relParts.join('/');
+        const fullPath = join(dirPath, entry.name);
+        if (entry.isFile()) {
+            const buf = await readFile(fullPath);
+            files[relPath] = hashContent(buf);
+        } else if (entry.isDirectory()) {
+            Object.assign(files, await hashDiskTree(fullPath, relParts));
+        }
+    }
+    return files;
+}
+
+export async function localTreeHash(dirPath) {
+    const fsFolder = new DoubleSyncFSFolder(dirPath);
+    const children = await fsFolder.list();
+    if (children.length === 0) return null;
+
+    const sync = new DoubleSync();
+    const store = new CDCStore();
+    const snapshotBytes = await sync.buildSnapshot({ root: fsFolder, store });
+    return new DoubleSyncSnapshot(snapshotBytes).treeHash;
+}
+
+export async function syncMemoryFolderToDisk(folder, destPath, oldFiles) {
+    const newFiles = {};
+    const newDirs = new Set();
+    const stats = { written: 0, skipped: 0, deleted: 0, folders: 0, bytes: 0 };
+
+    await walkAndSync(folder, destPath, [], oldFiles, newFiles, newDirs, stats);
+
+    if (oldFiles) {
+        for (const relPath of Object.keys(oldFiles)) {
+            if (!(relPath in newFiles)) {
+                const absPath = join(destPath, ...relPath.split('/'));
+                await rm(absPath, { force: true });
+                stats.deleted++;
+            }
+        }
+    }
+
+    await removeStaleDirectories(destPath, newDirs);
+
+    return { stats, newFiles };
+}
+
+export function hashMapsEqual(a, b) {
+    const keysA = Object.keys(a).sort();
+    const keysB = Object.keys(b).sort();
+    if (keysA.length !== keysB.length) return false;
+    return keysA.every((k, i) => k === keysB[i] && a[k] === b[k]);
+}
+
+export async function hashLocalTree(fsFolder, prefix = []) {
+    const files = {};
+    const children = await fsFolder.list();
+    for (const child of children) {
+        const relParts = [...prefix, child.name];
+        const relPath = relParts.join('/');
+        if (child.getContent) {
+            const content = await child.getContent();
+            files[relPath] = hashContent(content);
+        } else {
+            Object.assign(files, await hashLocalTree(child, relParts));
+        }
+    }
+    return files;
+}
+
+async function walkAndSync(folder, destPath, prefix, oldFiles, newFiles, newDirs, stats) {
+    const children = await folder.list();
+
+    for (const child of children) {
+        const relParts = [...prefix, child.name];
+        const relPath = relParts.join('/');
+        const absPath = join(destPath, child.name);
+
+        if (child.getContent) {
+            const content = await child.getContent();
+            const hash = hashContent(content);
+            newFiles[relPath] = hash;
+
+            if (oldFiles && oldFiles[relPath] === hash) {
+                stats.skipped++;
+            } else {
+                await writeFile(absPath, content);
+                stats.written++;
+                stats.bytes += content.length;
+            }
+        } else {
+            await mkdir(absPath, { recursive: true });
+            newDirs.add(relPath);
+            stats.folders++;
+            await walkAndSync(child, absPath, relParts, oldFiles, newFiles, newDirs, stats);
+        }
+    }
+}
+
+async function removeStaleDirectories(destPath, newDirs) {
+    let entries;
+    try { entries = await readdir(destPath, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries) {
+        if (!entry.isDirectory()) continue;
+        if (DISK_EXCLUDES.has(entry.name) || entry.name.startsWith('.')) continue;
+        const fullPath = join(destPath, entry.name);
+        if (!newDirs.has(entry.name)) {
+            await rm(fullPath, { recursive: true, force: true });
+        } else {
+            await removeStaleSubdirs(fullPath, entry.name, newDirs);
+        }
+    }
+}
+
+async function removeStaleSubdirs(dirPath, prefix, newDirs) {
+    let entries;
+    try { entries = await readdir(dirPath, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries) {
+        if (!entry.isDirectory()) continue;
+        const relPath = prefix + '/' + entry.name;
+        const fullPath = join(dirPath, entry.name);
+        if (!newDirs.has(relPath)) {
+            await rm(fullPath, { recursive: true, force: true });
+        } else {
+            await removeStaleSubdirs(fullPath, relPath, newDirs);
+        }
+    }
+}

package/lib/commands/watch.js

@@ -0,0 +1,181 @@
+import { watch as fsWatch } from 'node:fs';
+import EndlessVector from '@fizzyflow/endless-vector';
+import { WDoubleSync } from '@fizzyflow/wdoublesync';
+import { DoubleSyncFSFolder } from '../DoubleSyncFSFolder.js';
+import {
+    makeWalrusSealClient,
+    makeClientForRead,
+    resolvePath,
+    makeExcludes,
+    formatPath,
+    localTreeHash,
+    hashDiskTree,
+    syncMemoryFolderToDisk,
+    DISK_EXCLUDES,
+} from './shared.js';
+
+function timestamp() {
+    return new Date().toTimeString().slice(0, 8);
+}
+
+function hashesEqual(a, b) {
+    if (a === null && b === null) return true;
+    if (a === null || b === null) return false;
+    return a.length === b.length && a.every((byte, i) => byte === b[i]);
+}
+
+function isEnsureLengthError(err) {
+    return err?.message?.includes('abort code: 98') ||
+           err?.message?.includes('ensure_length') ||
+           err?.message?.includes('EUnexpectedLength');
+}
+
+export async function watch(args) {
+    if (!args.vectorId) throw new Error('vector-id is required for watch');
+
+    const cwd = resolvePath(args.path);
+    const chain = args.chain;
+    const debounceMs = args.debounce ?? 5000;
+    const pollIntervalMs = (args.pollInterval ?? 2) * 1000;
+    const pushEnabled = !args.pullOnly;
+    const pullEnabled = !args.pushOnly;
+
+    const excludes = makeExcludes(args);
+
+    const wsc = pushEnabled
+        ? await makeWalrusSealClient(args)
+        : await makeClientForRead(args);
+
+    const evParams = {
+        suiClient: wsc.suiClient,
+        id: args.vectorId,
+        packageId: chain,
+        walrusClient: wsc.walrusClient,
+        aggregatorUrl: wsc.aggregatorUrl,
+    };
+    if (wsc.sealClient && wsc.suiMaster) {
+        evParams.sealClient = wsc.sealClient;
+        evParams.senderAddress = wsc.suiMaster.address;
+        evParams.signer = wsc.suiMaster._keypair;
+        if (pushEnabled) {
+            evParams.signAndExecuteTransaction = (tx) => wsc.signAndExecuteTransaction(tx);
+        }
+    }
+
+    const ev = new EndlessVector({ ...evParams });
+    const wdsync = new WDoubleSync({
+        endlessVector: ev,
+        compress: args.compress || 'gzip',
+    });
+
+    await wdsync.initialize();
+
+    let remoteVersion = ev.length;
+    let lastSyncedHash = await localTreeHash(cwd);
+    let pushInFlight = false;
+    let debounceTimer = null;
+    let dirty = false;
+
+    console.log(`watching ${formatPath(cwd)}`);
+    console.log(`  vector:        ${args.vectorId}`);
+    console.log(`  remote version: ${remoteVersion}`);
+    console.log(`  push:          ${pushEnabled ? `enabled (debounce ${debounceMs}ms)` : 'disabled'}`);
+    console.log(`  pull:          ${pullEnabled ? `enabled (poll every ${pollIntervalMs / 1000}s)` : 'disabled'}`);
+    console.log('press Ctrl+C to stop\n');
+
+    async function runPush() {
+        if (!pushEnabled || pushInFlight) return;
+        dirty = false;
+        const newHash = await localTreeHash(cwd);
+        if (hashesEqual(newHash, lastSyncedHash)) return;
+
+        pushInFlight = true;
+        try {
+            const fsFolder = new DoubleSyncFSFolder(cwd, excludes);
+            console.log(`[${timestamp()}] change detected — pushing...`);
+            const result = await wdsync.push(fsFolder);
+            lastSyncedHash = newHash;
+            remoteVersion = result.version;
+            console.log(`[${timestamp()}] pushed version ${result.version}`);
+        } catch (err) {
+            if (isEnsureLengthError(err)) {
+                console.log(`[${timestamp()}] conflict: remote was updated while pushing — skipping push, will pull on next poll`);
+                dirty = true;
+            } else {
+                console.error(`[${timestamp()}] push error:`, err.message);
+            }
+        } finally {
+            pushInFlight = false;
+        }
+    }
+
+    async function runPoll() {
+        if (!pullEnabled || pushInFlight) return;
+        try {
+            ev.reInitialize();
+            await ev.initialize();
+            if (ev.length <= remoteVersion) return;
+
+            const newRemoteVersion = ev.length;
+
+            // Always resync wdsync when remote has advanced. Without this,
+            // wdsync._lastSnapshot stays at the old version while ev.length has
+            // advanced — the next push would build a diff against the wrong base
+            // snapshot, corrupting the chain.
+            wdsync.reInitialize();
+
+            const localNow = await localTreeHash(cwd);
+
+            if (!hashesEqual(localNow, lastSyncedHash)) {
+                console.log(`[${timestamp()}] remote has ${newRemoteVersion - remoteVersion} new version(s) but local has unpushed changes — skipping pull`);
+                return;
+            }
+
+            console.log(`[${timestamp()}] remote advanced to version ${newRemoteVersion} — pulling...`);
+            let folder;
+            try {
+                folder = await wdsync.restore();
+            } catch (err) {
+                console.error(`[${timestamp()}] pull failed (${err.message}) — skipping to version ${newRemoteVersion}; run \`pull\` manually`);
+                remoteVersion = newRemoteVersion;
+                return;
+            }
+            const oldFiles = await hashDiskTree(cwd);
+            const { stats } = await syncMemoryFolderToDisk(folder, cwd, oldFiles);
+            lastSyncedHash = await localTreeHash(cwd);
+            remoteVersion = newRemoteVersion;
+            console.log(`[${timestamp()}] pulled version ${remoteVersion} (${stats.written} written, ${stats.skipped} unchanged, ${stats.deleted} deleted)`);
+        } catch (err) {
+            console.error(`[${timestamp()}] poll error:`, err.message);
+        }
+    }
+
+    const watcher = fsWatch(cwd, { recursive: true }, (eventType, filename) => {
+        if (!pushEnabled) return;
+        if (!filename) return;
+        const base = filename.split('/')[0];
+        if (DISK_EXCLUDES.has(base) || base.startsWith('.')) return;
+        dirty = true;
+        clearTimeout(debounceTimer);
+        debounceTimer = setTimeout(runPush, debounceMs);
+    });
+
+    const poller = setInterval(runPoll, pollIntervalMs);
+
+    let shutdownResolve;
+    const shutdownPromise = new Promise((resolve) => {
+        shutdownResolve = resolve;
+    });
+
+    function shutdown() {
+        console.log('\nstopping watch...');
+        clearTimeout(debounceTimer);
+        clearInterval(poller);
+        watcher.close();
+        shutdownResolve();
+    }
+    process.once('SIGINT', shutdown);
+    process.once('SIGTERM', shutdown);
+
+    await shutdownPromise;
+}

package/lib/commands.js

@@ -1,13 +1,18 @@
 import { mkdir, writeFile, readFile, readdir, rm } from 'node:fs/promises';
 import { watch as fsWatch } from 'node:fs';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import { createHash } from 'node:crypto';
 import { homedir } from 'node:os';
+
+function resolvePath(p) {
+    if (p.startsWith('~')) return join(homedir(), p.slice(1));
+    return resolve(p);
+}
 import { SuiMaster } from 'suidouble';
 import EndlessVector from '@fizzyflow/endless-vector';
 import { WDoubleSync } from '@fizzyflow/wdoublesync';
 import { DoubleSync, CDCStore, DoubleSyncSnapshot } from '@fizzyflow/doublesync';
-import { WalrusSealClient } from 'walrus-seal-client-with-local';
+import WalrusSealClient from './WalrusSealClient.js';
 import { DoubleSyncFSFolder } from './DoubleSyncFSFolder.js';
 
 async function makeWalrusSealClient(args) {
@@ -81,7 +86,7 @@ export async function push(args) {
     const excludes = args.exclude
         ? args.exclude.split(',').map(s => s.trim())
         : undefined;
-    const destPath = process.cwd();
+    const destPath = resolvePath(args.path);
     const fsFolder = new DoubleSyncFSFolder(destPath, excludes);
 
     const counts = await countTree(fsFolder);
@@ -202,7 +207,7 @@ export async function pull(args) {
         throw err;
     }
 
-    const destPath = process.cwd();
+    const destPath = resolvePath(args.path);
     const oldFiles = await hashDiskTree(destPath);
 
     const { stats, newFiles } = await syncMemoryFolderToDisk(folder, destPath, oldFiles);
@@ -218,12 +223,16 @@ export async function pull(args) {
 }
 
 export async function info(args) {
+    const wsc = await makeClientForRead(args);
+
+    console.log('wdoublesync');
+    console.log('  chain:          ', args.chain);
+    console.log('  your wallet:    ', wsc.suiMaster?.address || '(none)');
+
     if (!args.vectorId) {
-        throw new Error('vector-id is required for info');
+        return;
     }
 
-    const wsc = await makeClientForRead(args);
-
     const evParams = {
         suiClient: wsc.suiClient,
         id: args.vectorId,
@@ -244,14 +253,35 @@ export async function info(args) {
 
     const totalVersions = ev.length;
 
-    console.log('EndlessVector:', args.vectorId);
-    console.log('  chain:          ', args.chain);
+    console.log('\nEndlessVector:', args.vectorId);
+
+    let owner = '(unknown)';
+    if (wsc.suiMaster) {
+        try {
+            const obj = await wsc.suiMaster.getObject(args.vectorId);
+            if (obj?._owner) {
+                const ownerData = obj._owner;
+                if (typeof ownerData === 'string') {
+                    owner = ownerData;
+                } else if (ownerData.AddressOwner) {
+                    owner = ownerData.AddressOwner;
+                } else if (ownerData.ObjectOwner) {
+                    owner = ownerData.ObjectOwner;
+                }
+            }
+        } catch (err) {
+            // silently fail, owner stays unknown
+        }
+    }
+
+    console.log('  owner:          ', owner);
+    console.log('  encrypted:      ', ev.sealEncryptedKey ? 'yes (Seal)' : 'no');
     console.log('  versions:       ', totalVersions);
     console.log('  binary size:    ', formatBytes(ev.binaryLength));
     console.log('  history items:  ', ev.historyItemsCount);
     console.log('  archives:       ', ev.archiveItemsCount);
 
-    const destPath = process.cwd();
+    const destPath = resolvePath(args.path);
     const localHash = await localTreeHash(destPath);
 
     if (!localHash) {
@@ -451,7 +481,7 @@ async function removeStaleSubdirs(dirPath, prefix, newDirs) {
 export async function watch(args) {
     if (!args.vectorId) throw new Error('vector-id is required for watch');
 
-    const cwd = process.cwd();
+    const cwd = resolvePath(args.path);
     const chain = args.chain;
     const debounceMs = args.debounce ?? 5000;
     const pollIntervalMs = (args.pollInterval ?? 2) * 1000;
@@ -624,7 +654,7 @@ export async function rebate(args) {
         throw new Error('vector-id is required for rebate');
     }
 
-    const destPath = process.cwd();
+    const destPath = resolvePath(args.path);
     const excludes = args.exclude
         ? args.exclude.split(',').map(s => s.trim())
         : undefined;
@@ -663,6 +693,9 @@ export async function rebate(args) {
     await ev.burnArchive();
     console.log('  burned');
 
+    // archive() and burnArchive() both call ev.reInitialize() — refetch fresh state
+    await ev.initialize();
+
     console.log('step 3: pushing local state as fresh snapshot...');
 
     const wdsync = new WDoubleSync({
@@ -670,12 +703,13 @@ export async function rebate(args) {
         compress: args.compress || 'gzip',
     });
 
-    // Force fresh snapshot by resetting internal state to prevent replay of broken diffs
+    // Skip wdsync.initialize() replay: rebate pushes a full snapshot, no need to replay chain.
+    // _lastSnapshot = null forces a full snapshot; _isInitialized = true bypasses replay.
     wdsync._senderStore = new CDCStore({ copyBytes: false });
     wdsync._receiverMirror = new CDCStore();
     wdsync._lastSnapshot = null;
-    wdsync._replayedCount = 0;
-    wdsync._isInitialized = false;
+    wdsync._replayedCount = ev.length;
+    wdsync._isInitialized = true;
 
     const result = await wdsync.push(fsFolder);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fizzyflow/wdoublesync_cli",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "type": "module",
   "description": "CLI tool built on the wdoublesync library to sync local folders to Walrus decentralised storage on the Sui network. Stores versioned gzip-compressed snapshots and diffs inside an EndlessVector on-chain object. Supports Seal encryption, manifest-based fast change detection, and full version history with point-in-time restore.",
   "bin": {
@@ -27,9 +27,8 @@
     "@fizzyflow/wdoublesync": "^0.1.7",
     "@fizzyflow/endless-vector": "^0.0.13",
     "@mysten/seal": "^1.1.3",
+    "@mysten/sui": "^2.19.0",
+    "@mysten/walrus": "^1.2.1",
     "@noble/ciphers": "^2.2.0"
-  },
-  "devDependencies": {
-    "walrus-seal-client-with-local": "file:../seal_walrus_localnet"
   }
 }