pal-explorer-cli 0.4.7 → 0.4.9

package/lib/core/extensions.js

@@ -17,12 +17,8 @@ const EXTENSIONS_DIR = path.join(
 
 const BUNDLED_DIR = path.join(EXTENSIONS_DIR, '@palexplorer');
 
-const CORE_EXTENSIONS = new Set([
-  'analytics',
-  'discovery',
-  'explorer-integration',
-  'vfs',
-]);
+// No "core" extensions — all extensions can be enabled/disabled by the user.
+// Extensions that should be on by default set "config.enabled.default": true in their manifest.
 
 // Ed25519 public key for verifying extension signatures (Palexplorer team key)
 const TEAM_PUBLIC_KEY = 'ba71d876238e24b1b230c567a6b7339abdf55030e98580a5a883b97ee0a3f084';
@@ -517,7 +513,6 @@ function getInstalledExtensions() {
         const manifest = readManifest(bundledPath);
         if (manifest) {
           const fullName = `@palexplorer/${bundled}`;
-          const isCore = CORE_EXTENSIONS.has(bundled);
           const explicitEnabled = (config.get('enabledExtensions') || []).includes(fullName);
           const explicitDisabled = disabled.includes(fullName);
           const manifestDefault = manifest.config?.enabled?.default === true;
@@ -525,9 +520,7 @@ function getInstalledExtensions() {
             ...manifest,
             path: bundledPath,
             bundled: true,
-            enabled: isCore
-              ? !explicitDisabled
-              : explicitDisabled ? false : (explicitEnabled || manifestDefault),
+            enabled: explicitDisabled ? false : (explicitEnabled || manifestDefault),
           });
         }
       }
@@ -896,12 +889,10 @@ function enableExtension(name) {
   const newDisabled = disabled.filter(d => d !== name && d !== full && d !== short);
   if (newDisabled.length !== disabled.length) config.set('disabledExtensions', newDisabled);
 
-  if (!CORE_EXTENSIONS.has(short)) {
-    const enabled = config.get('enabledExtensions') || [];
-    if (!enabled.includes(full)) {
-      enabled.push(full);
-      config.set('enabledExtensions', enabled);
-    }
+  const enabled = config.get('enabledExtensions') || [];
+  if (!enabled.includes(full)) {
+    enabled.push(full);
+    config.set('enabledExtensions', enabled);
   }
 }
 
@@ -915,11 +906,9 @@ function disableExtension(name) {
     config.set('disabledExtensions', disabled);
   }
 
-  if (!CORE_EXTENSIONS.has(short)) {
-    const enabled = config.get('enabledExtensions') || [];
-    const newEnabled = enabled.filter(e => e !== name && e !== full && e !== short);
-    if (newEnabled.length !== enabled.length) config.set('enabledExtensions', newEnabled);
-  }
+  const enabled = config.get('enabledExtensions') || [];
+  const newEnabled = enabled.filter(e => e !== name && e !== full && e !== short);
+  if (newEnabled.length !== enabled.length) config.set('enabledExtensions', newEnabled);
   unloadExtension(name);
 }
 
@@ -1012,6 +1001,11 @@ function getContributedPages() {
 
 function ensureDefaultExtensions() {
   if (!fs.existsSync(BUNDLED_DIR)) return;
+  // Ensure extensions dir has ESM package.json so import() works
+  const extPkg = path.join(EXTENSIONS_DIR, 'package.json');
+  if (!fs.existsSync(extPkg)) {
+    fs.writeFileSync(extPkg, '{ "type": "module" }\n');
+  }
   const enabled = config.get('enabledExtensions') || [];
   const disabled = config.get('disabledExtensions') || [];
   let changed = false;
@@ -1019,7 +1013,7 @@ function ensureDefaultExtensions() {
     const manifest = readManifest(path.join(BUNDLED_DIR, bundled));
     if (!manifest) continue;
     const full = `@palexplorer/${bundled}`;
-    if (manifest.config?.enabled?.default === true && !CORE_EXTENSIONS.has(bundled)) {
+    if (manifest.config?.enabled?.default === true) {
       if (!enabled.includes(full) && !disabled.includes(full)) {
         enabled.push(full);
         changed = true;

package/lib/core/syncEngine.js

@@ -1,7 +1,11 @@
 import crypto from 'crypto';
 import fs from 'fs';
 import path from 'path';
+import { EventEmitter } from 'events';
 import logger from '../utils/logger.js';
+import { buildProtocolManifest, computeDelta } from '../protocol/sync.js';
+import { syncManifest as createSyncManifestMsg, syncRequest as createSyncRequestMsg, syncConflict as createSyncConflictMsg } from '../protocol/messages.js';
+import { registerSession, unregisterSession } from './syncProtocolHandlers.js';
 
 // mtime+size hash cache: skip re-hashing files that haven't changed
 const _hashCache = new Map();
@@ -30,6 +34,25 @@ function shouldIgnore(relativePath) {
   return IGNORE_PATTERNS.some(p => p.test(relativePath));
 }
 
+function matchesAny(filePath, patterns) {
+  for (const pattern of patterns) {
+    // Simple glob: *.ext, dir/*, **/pattern
+    const regex = globToRegex(pattern);
+    if (regex.test(filePath)) return true;
+  }
+  return false;
+}
+
+function globToRegex(glob) {
+  const escaped = glob
+    .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+    .replace(/\*\*/g, '§DOUBLESTAR§')
+    .replace(/\*/g, '[^/]*')
+    .replace(/§DOUBLESTAR§/g, '.*')
+    .replace(/\?/g, '[^/]');
+  return new RegExp(`^${escaped}$|/${escaped}$|^${escaped}/`);
+}
+
 function hashFileStream(filePath) {
   return new Promise((resolve, reject) => {
     const h = crypto.createHash('sha256');
@@ -40,7 +63,8 @@ function hashFileStream(filePath) {
   });
 }
 
-export async function buildManifest(dirPath, cachedManifest) {
+export async function buildManifest(dirPath, cachedManifest, options = {}) {
+  const { exclude = [], include = [], checksum = false, sizeOnly = false } = options;
   const cachedMap = new Map();
   if (cachedManifest) {
     const entries = Array.isArray(cachedManifest) ? cachedManifest : Object.values(cachedManifest);
@@ -72,6 +96,10 @@ export async function buildManifest(dirPath, cachedManifest) {
       const relativePath = relBase ? `${relBase}/${dirent.name}` : dirent.name;
 
       if (shouldIgnore(relativePath)) continue;
+      // User exclude/include patterns
+      if (exclude.length > 0 && matchesAny(relativePath, exclude)) {
+        if (include.length === 0 || !matchesAny(relativePath, include)) continue;
+      }
 
       if (dirent.isDirectory()) {
         await walk(fullPath, relativePath);
@@ -186,8 +214,8 @@ export function detectConflicts(localManifest, remoteManifest, baseManifest) {
 }
 
 export function applyDiff(diff, sourcePath, destPath, options = {}) {
-  const { dryRun = false } = options;
-  const results = { copied: [], deleted: [], conflicted: [], errors: [] };
+  const { dryRun = false, backup = false, update = false, ignoreExisting = false } = options;
+  const results = { copied: [], deleted: [], conflicted: [], skipped: [], errors: [] };
 
   for (const entry of diff.added) {
     const src = path.join(sourcePath, entry.relativePath);
@@ -208,12 +236,35 @@ export function applyDiff(diff, sourcePath, destPath, options = {}) {
   for (const entry of diff.modified) {
     const src = path.join(sourcePath, entry.relativePath);
     const dst = path.join(destPath, entry.relativePath);
+
+    // --ignore-existing: skip files that exist on destination
+    if (ignoreExisting && fs.existsSync(dst)) {
+      results.skipped.push(entry.relativePath);
+      continue;
+    }
+
+    // --update: skip files newer on destination
+    if (update && fs.existsSync(dst)) {
+      try {
+        const dstStat = fs.statSync(dst);
+        const srcStat = fs.statSync(src);
+        if (dstStat.mtimeMs > srcStat.mtimeMs) {
+          results.skipped.push(entry.relativePath);
+          continue;
+        }
+      } catch {}
+    }
+
     if (dryRun) {
       results.copied.push(entry.relativePath);
       continue;
     }
     try {
       fs.mkdirSync(path.dirname(dst), { recursive: true });
+      // --backup: rename before overwriting
+      if (backup && fs.existsSync(dst)) {
+        fs.renameSync(dst, dst + '.bak');
+      }
       fs.copyFileSync(src, dst);
       results.copied.push(entry.relativePath);
     } catch (err) {
@@ -261,4 +312,226 @@ export function applyDiff(diff, sourcePath, destPath, options = {}) {
   return results;
 }
 
+// --- SyncSession: P2P sync over PAL/1.0 protocol ---
+
+export class SyncSession extends EventEmitter {
+  constructor({ localPath, peerPK, transport, keyPair, syncPairId, options = {} }) {
+    super();
+    this.localPath = path.resolve(localPath);
+    this.peerPK = peerPK;
+    this.transport = transport;
+    this.keyPair = keyPair;
+    this.syncPairId = syncPairId || crypto.randomUUID();
+    this.options = options;
+    this.state = 'idle'; // idle → scanning → exchanging → transferring → done/error
+    this.localManifest = null;
+    this.remoteManifest = null;
+    this.diff = null;
+    this.generation = Date.now();
+    this._resolveRemoteManifest = null;
+    this._resolveFileData = null;
+    this._receivedFiles = new Map();
+  }
+
+  async start(cachedManifest) {
+    try {
+      this.state = 'scanning';
+      this.emit('state', 'scanning');
+      registerSession(this.syncPairId, this);
+
+      // Build local manifest (respects exclude/include/checksum options)
+      this.localManifest = await buildManifest(this.localPath, cachedManifest, this.options);
+
+      // Build protocol manifest and send via transport
+      this.state = 'exchanging';
+      this.emit('state', 'exchanging');
+
+      const protoManifest = buildProtocolManifest(this.localManifest, this.syncPairId, this.generation);
+      const envelope = createSyncManifestMsg(this.keyPair, this.peerPK, protoManifest);
+      await this.transport.send(envelope);
+      this.emit('manifestSent', { entries: this.localManifest.length });
+
+      return { localManifest: this.localManifest, syncPairId: this.syncPairId };
+    } catch (err) {
+      this.state = 'error';
+      this.emit('error', err);
+      throw err;
+    }
+  }
+
+  handleRemoteManifest(data) {
+    // Convert protocol manifest entries back to engine format
+    const entries = (data.entries || []).map(e => ({
+      relativePath: e.path || e.relativePath,
+      size: e.size,
+      hash: e.hash,
+      mtime: e.mtime,
+    }));
+    this.remoteManifest = entries;
+    this.emit('manifestReceived', { entries: entries.length, from: data.from });
+
+    if (this._resolveRemoteManifest) {
+      this._resolveRemoteManifest(entries);
+      this._resolveRemoteManifest = null;
+    }
+  }
+
+  waitForRemoteManifest(timeoutMs = 30000) {
+    if (this.remoteManifest) return Promise.resolve(this.remoteManifest);
+    return new Promise((resolve, reject) => {
+      this._resolveRemoteManifest = resolve;
+      const timer = setTimeout(() => {
+        this._resolveRemoteManifest = null;
+        reject(new Error('Timed out waiting for remote manifest'));
+      }, timeoutMs);
+      // Clean up timer if resolved
+      const origResolve = this._resolveRemoteManifest;
+      this._resolveRemoteManifest = (val) => {
+        clearTimeout(timer);
+        origResolve(val);
+      };
+    });
+  }
+
+  computeSync(baseManifest) {
+    if (!this.localManifest || !this.remoteManifest) {
+      throw new Error('Both local and remote manifests required');
+    }
+
+    const localMap = manifestToMap(this.localManifest);
+    const remoteMap = manifestToMap(this.remoteManifest);
+    const baseMap = baseManifest ? (Array.isArray(baseManifest) ? manifestToMap(baseManifest) : baseManifest) : {};
+
+    // Compute what we need from remote (files remote has that we don't or are different)
+    const diff = diffManifests(localMap, remoteMap);
+    const { conflicts, safeModified } = detectConflicts(localMap, remoteMap, baseMap);
+
+    // Separate real conflicts from safe modifications in the diff
+    if (conflicts.length > 0) {
+      diff.conflicts = conflicts;
+      diff.modified = diff.modified.filter(
+        m => !conflicts.some(c => c.relativePath === m.relativePath)
+      );
+    }
+
+    this.diff = diff;
+    this.emit('diffComputed', {
+      added: diff.added.length,
+      modified: diff.modified.length,
+      deleted: diff.deleted.length,
+      conflicts: (diff.conflicts || []).length,
+    });
+
+    return diff;
+  }
+
+  async requestFiles(filesToRequest) {
+    if (!filesToRequest || filesToRequest.length === 0) return;
+
+    this.state = 'transferring';
+    this.emit('state', 'transferring');
+
+    const requestPayload = filesToRequest.map(f => ({
+      path: f.relativePath || f,
+      mode: 'full',
+    }));
+
+    const envelope = createSyncRequestMsg(this.keyPair, this.peerPK, {
+      syncPairId: this.syncPairId,
+      files: requestPayload,
+    });
+
+    await this.transport.send(envelope);
+    this.emit('filesRequested', { count: requestPayload.length });
+  }
+
+  handleSyncRequest(data) {
+    // Peer is requesting files from us — read and stream them
+    const files = data.files || [];
+    this.emit('syncRequest', { from: data.from, files: files.length });
+
+    const fileData = {};
+    for (const req of files) {
+      const filePath = path.join(this.localPath, req.path);
+      try {
+        if (fs.existsSync(filePath)) {
+          fileData[req.path] = fs.readFileSync(filePath);
+        }
+      } catch (err) {
+        logger.warn(`Cannot read requested file: ${req.path} (${err.message})`);
+      }
+    }
+
+    this._receivedFiles = new Map(Object.entries(fileData));
+    this.emit('filesReady', { count: Object.keys(fileData).length });
+    return fileData;
+  }
+
+  handleSyncDiff(data) {
+    this.emit('syncDiff', data);
+  }
+
+  handleConflict(data) {
+    const conflicts = data.conflicts || [];
+    this.emit('conflict', { count: conflicts.length, conflicts });
+  }
+
+  applyChanges(sourcePath, options = {}) {
+    if (!this.diff) throw new Error('No diff computed. Call computeSync() first.');
+    const mergedOpts = { ...this.options, ...options };
+    return applyDiff(this.diff, sourcePath || this.localPath, this.localPath, mergedOpts);
+  }
+
+  // Delta sync: request only changed blocks of modified files (4KB blocks)
+  async requestDelta(modifiedFiles) {
+    if (!modifiedFiles || modifiedFiles.length === 0) return [];
+
+    const deltaRequests = [];
+    for (const file of modifiedFiles) {
+      const localPath = path.join(this.localPath, file.relativePath);
+      if (!fs.existsSync(localPath)) continue;
+
+      try {
+        const { computeBlockHashes } = await import('../protocol/sync.js');
+        const localHashes = computeBlockHashes(localPath, 4096);
+        deltaRequests.push({
+          path: file.relativePath,
+          mode: 'delta',
+          blockSize: 4096,
+          localBlockHashes: localHashes.hashes,
+          localSize: localHashes.totalSize,
+        });
+      } catch {
+        // Fall back to full file if delta not available (e.g., not Pro)
+        deltaRequests.push({ path: file.relativePath, mode: 'full' });
+      }
+    }
+
+    if (deltaRequests.length > 0) {
+      const envelope = createSyncRequestMsg(this.keyPair, this.peerPK, {
+        syncPairId: this.syncPairId,
+        files: deltaRequests,
+      });
+      await this.transport.send(envelope);
+      this.emit('deltaRequested', { count: deltaRequests.length });
+    }
+
+    return deltaRequests;
+  }
+
+  finish() {
+    this.state = 'done';
+    this.emit('state', 'done');
+    unregisterSession(this.syncPairId);
+    this.transport.close();
+  }
+
+  abort(reason) {
+    this.state = 'error';
+    this.emit('error', new Error(reason || 'Session aborted'));
+    unregisterSession(this.syncPairId);
+    this.transport.close();
+  }
+}
+
 export { formatSize } from '../utils/format.js';

package/lib/core/syncOptions.js

@@ -0,0 +1,80 @@
+// Parse and validate sync CLI flags into engine options
+
+export const SYNC_FLAGS = {
+  dryRun: { flag: '--dry-run', description: 'Show changes without applying', default: false },
+  delete: { flag: '--delete', description: 'Delete destination files not in source', default: false },
+  exclude: { flag: '--exclude <pattern>', description: 'Glob patterns to skip (repeatable)', default: [] },
+  include: { flag: '--include <pattern>', description: 'Override exclude for specific patterns (repeatable)', default: [] },
+  progress: { flag: '--progress', description: 'Show transfer progress (bytes, speed, ETA)', default: false },
+  verbose: { flag: '-v, --verbose', description: 'Per-file operation output', default: false },
+  checksum: { flag: '--checksum', description: 'Force hash comparison (skip mtime shortcut)', default: false },
+  sizeOnly: { flag: '--size-only', description: 'Compare by size only', default: false },
+  update: { flag: '--update', description: 'Skip files newer on destination', default: false },
+  ignoreExisting: { flag: '--ignore-existing', description: 'Skip files that exist on destination', default: false },
+  backup: { flag: '--backup', description: 'Rename before overwriting (.bak)', default: false },
+  partial: { flag: '--partial', description: 'Keep partial files for resume', default: false },
+  bandwidthLimit: { flag: '--bandwidth-limit <KBps>', description: 'Throttle transfer speed', default: 0 },
+  compress: { flag: '--compress', description: 'zlib compress chunks before transfer', default: false },
+  recursive: { flag: '--no-recursive', description: 'Flat sync (recursive is default)', default: true },
+  archive: { flag: '--archive', description: 'Shorthand for --recursive --checksum --delete', default: false },
+};
+
+export function parseSyncOptions(opts) {
+  const options = {
+    dryRun: opts.dryRun || false,
+    delete: opts.delete || false,
+    exclude: normalizeArray(opts.exclude),
+    include: normalizeArray(opts.include),
+    progress: opts.progress || false,
+    verbose: opts.verbose || false,
+    checksum: opts.checksum || false,
+    sizeOnly: opts.sizeOnly || false,
+    update: opts.update || false,
+    ignoreExisting: opts.ignoreExisting || false,
+    backup: opts.backup || false,
+    partial: opts.partial || false,
+    bandwidthLimit: parseInt(opts.bandwidthLimit, 10) || 0,
+    compress: opts.compress || false,
+    recursive: opts.recursive !== false,
+  };
+
+  // --archive expands to --recursive --checksum --delete
+  if (opts.archive) {
+    options.recursive = true;
+    options.checksum = true;
+    options.delete = true;
+  }
+
+  // Validate: --checksum and --size-only are mutually exclusive
+  if (options.checksum && options.sizeOnly) {
+    throw new Error('Cannot use --checksum and --size-only together');
+  }
+
+  return options;
+}
+
+function normalizeArray(val) {
+  if (!val) return [];
+  if (Array.isArray(val)) return val;
+  return [val];
+}
+
+export function addSyncFlagsToCommand(cmd) {
+  return cmd
+    .option('--dry-run', 'Show changes without applying')
+    .option('--delete', 'Delete destination files not in source')
+    .option('--exclude <pattern...>', 'Glob patterns to skip')
+    .option('--include <pattern...>', 'Override exclude for specific patterns')
+    .option('--progress', 'Show transfer progress')
+    .option('-v, --verbose', 'Per-file operation output')
+    .option('--checksum', 'Force hash comparison (skip mtime shortcut)')
+    .option('--size-only', 'Compare by size only')
+    .option('--update', 'Skip files newer on destination')
+    .option('--ignore-existing', 'Skip files that exist on destination')
+    .option('--backup', 'Rename before overwriting (.bak)')
+    .option('--partial', 'Keep partial files for resume')
+    .option('--bandwidth-limit <KBps>', 'Throttle transfer speed (KB/s)')
+    .option('--compress', 'Compress chunks before transfer')
+    .option('--no-recursive', 'Flat sync (recursive by default)')
+    .option('--archive', 'Shorthand for --recursive --checksum --delete');
+}

package/lib/core/syncProtocolHandlers.js

@@ -0,0 +1,87 @@
+import { on, off } from '../protocol/handler.js';
+import logger from '../utils/logger.js';
+
+// Registry of active sync sessions by syncPairId
+const activeSessions = new Map();
+
+// Bridge protocol handler events to sync engine sessions
+const handlers = {
+  'sync.manifest': (data) => {
+    const session = activeSessions.get(data.syncPairId) || findSessionByPeer(data.from);
+    if (session) {
+      session.handleRemoteManifest(data);
+    } else {
+      logger.debug(`[sync] Received manifest from ${data.from?.slice(0, 8)} but no active session`);
+      pendingManifests.set(data.from, data);
+    }
+  },
+  'sync.diff': (data) => {
+    const session = activeSessions.get(data.syncPairId) || findSessionByPeer(data.from);
+    if (session?.handleSyncDiff) session.handleSyncDiff(data);
+  },
+  'sync.request': (data) => {
+    const session = activeSessions.get(data.syncPairId) || findSessionByPeer(data.from);
+    if (session?.handleSyncRequest) session.handleSyncRequest(data);
+  },
+  'sync.conflict': (data) => {
+    const session = activeSessions.get(data.syncPairId) || findSessionByPeer(data.from);
+    if (session?.handleConflict) session.handleConflict(data);
+  },
+};
+
+// Store manifests that arrive before a session is created
+const pendingManifests = new Map();
+
+function findSessionByPeer(peerPK) {
+  for (const session of activeSessions.values()) {
+    if (session.peerPK === peerPK) return session;
+  }
+  return null;
+}
+
+let registered = false;
+
+export function registerSyncHandlers() {
+  if (registered) return;
+  for (const [type, handler] of Object.entries(handlers)) {
+    on(type, handler);
+  }
+  registered = true;
+}
+
+export function unregisterSyncHandlers() {
+  if (!registered) return;
+  for (const [type, handler] of Object.entries(handlers)) {
+    off(type, handler);
+  }
+  registered = false;
+  activeSessions.clear();
+  pendingManifests.clear();
+}
+
+export function registerSession(syncPairId, session) {
+  activeSessions.set(syncPairId, session);
+  // Check for pending manifests from this peer
+  const pending = pendingManifests.get(session.peerPK);
+  if (pending) {
+    pendingManifests.delete(session.peerPK);
+    session.handleRemoteManifest(pending);
+  }
+}
+
+export function unregisterSession(syncPairId) {
+  activeSessions.delete(syncPairId);
+}
+
+export function getActiveSession(syncPairId) {
+  return activeSessions.get(syncPairId) || null;
+}
+
+export function getPendingManifest(peerPK) {
+  const manifest = pendingManifests.get(peerPK);
+  if (manifest) {
+    pendingManifests.delete(peerPK);
+    return manifest;
+  }
+  return null;
+}