@tb.p/dd 1.0.0

package/processors/optionETL.js

@@ -0,0 +1,82 @@
+import path from 'path';
+function parsePipeSeparated(pipeString) {
+  if (!pipeString) return [];
+  return pipeString.split('|').map(item => item.trim()).filter(item => item.length > 0);
+}
+
+function normalizePaths(paths) {
+  return paths.map(p => {
+    // Preserve original relative paths, only resolve if they don't start with ./
+    if (p.startsWith('./') || p.startsWith('../')) {
+      return p;
+    }
+    return path.resolve(p);
+  });
+}
+
+function normalizeExtensions(extensions) {
+  return extensions.map(ext => {
+    return ext.replace(/^\.+/, '').toLowerCase();
+  });
+}
+
+function determineMode(options) {
+  if (options.getExtensions) return 'get-extensions';
+  if (options.detectCandidates) return 'detect-candidates';
+  if (options.candidateStats) return 'candidate-stats';
+  if (options.resume) return 'resume';
+  if (options.save) return 'save';
+  return 'unknown';
+}
+
+function processOptions(rawOptions) {
+  const targets = parsePipeSeparated(rawOptions.targets);
+  const normalizedTargets = normalizePaths(targets);
+
+  // Parse extensions - support both comma and pipe separators
+  const extensions = rawOptions.extensions ? 
+    rawOptions.extensions.split(/[,|]/).map(item => item.trim()).filter(item => item.length > 0) : 
+    [];
+  const normalizedExtensions = normalizeExtensions(extensions);
+
+  const mode = determineMode(rawOptions);
+
+  // Determine path preservation setting
+  const preservePaths = rawOptions.flatDuplicates ? false : (rawOptions.preservePaths !== false);
+
+  const processedOptions = {
+    mode,
+    targets: normalizedTargets,
+    ...(extensions.length > 0 && { extensions: normalizedExtensions }),
+    ...(rawOptions.resume && { resumeDb: path.resolve(rawOptions.resume) }),
+    ...(rawOptions.save && { saveDb: path.resolve(rawOptions.save) }),
+    ...(rawOptions.detectCandidates && { detectCandidates: true }),
+    ...(rawOptions.candidateStats && { candidateStats: true }),
+    ...(rawOptions.hashAlgorithm && { hashAlgorithm: rawOptions.hashAlgorithm }),
+    ...(rawOptions.batchSize && { batchSize: rawOptions.batchSize }),
+    ...(rawOptions.maxConcurrency && { maxConcurrency: rawOptions.maxConcurrency }),
+    ...(rawOptions.verbose && { verbose: true }),
+    move: Boolean(rawOptions.move),
+    preservePaths,
+    _raw: rawOptions
+  };
+
+  // Determine database path for candidate operations
+  if (processedOptions.mode === 'detect-candidates' || processedOptions.mode === 'candidate-stats') {
+    if (rawOptions.resume) {
+      processedOptions.database = path.resolve(rawOptions.resume);
+    } else if (rawOptions.save) {
+      processedOptions.database = path.resolve(rawOptions.save);
+    }
+  }
+
+  return processedOptions;
+}
+
+export {
+  processOptions,
+  parsePipeSeparated,
+  normalizePaths,
+  normalizeExtensions,
+  determineMode
+};

package/utils/README.md

@@ -0,0 +1,261 @@
+# File System Utilities for @tb.p/dd
+
+This directory contains comprehensive file system utilities for the @tb.p/dd file deduplication tool. The utilities are organized into specialized modules for different file operations.
+
+## 📁 File Structure
+
+```
+utils/
+├── index.js              # Main exports and module aggregation
+├── fileSystemUtils.js    # Core file system operations
+├── fileScanner.js        # File scanning and discovery
+├── fileMover.js          # File moving and management
+├── fileHasher.js         # File hashing and content comparison
+├── examples.js           # Comprehensive usage examples
+└── README.md            # This documentation
+```
+
+## 🚀 Quick Start
+
+```javascript
+const {
+  scanDirectories,
+  findDuplicateFilesByHash,
+  moveDuplicateFiles,
+  formatFileSize
+} = require('./utils');
+
+// Scan directories for files
+const files = await scanDirectories({
+  targets: ['./src', './docs'],
+  extensions: ['js', 'md'],
+  maxSize: 10 * 1024 * 1024 // 10MB
+});
+
+// Find duplicate files
+const duplicates = await findDuplicateFilesByHash(files, {
+  algorithm: 'sha256'
+});
+
+// Move duplicates
+await moveDuplicateFiles(duplicates, './duplicates');
+```
+
+## 📚 Module Documentation
+
+### fileSystemUtils.js
+Core file system operations and utilities.
+
+**Key Functions:**
+- `pathExists(filePath)` - Check if path exists (async)
+- `getFileStats(filePath)` - Get file statistics
+- `createDirectory(dirPath)` - Create directory recursively
+- `moveFile(src, dest, options)` - Move file with options
+- `copyFile(src, dest, options)` - Copy file with options
+- `formatFileSize(bytes)` - Format bytes to human readable size
+
+**Example:**
+```javascript
+const { pathExists, createDirectory, moveFile } = require('./fileSystemUtils');
+
+if (await pathExists('./source.txt')) {
+  await createDirectory('./dest');
+  await moveFile('./source.txt', './dest/source.txt', { overwrite: true });
+}
+```
+
+### fileScanner.js
+Advanced file scanning and discovery operations.
+
+**Key Functions:**
+- `scanDirectory(dirPath, options)` - Scan single directory
+- `scanDirectories(options)` - Scan multiple directories
+- `findFilesByExtension(dirPaths, extensions, options)` - Find files by extension
+- `findFilesByPattern(dirPaths, pattern, options)` - Find files by pattern
+- `findDuplicatesBySize(files)` - Find duplicates by file size
+- `getScanStatistics(files)` - Get comprehensive scan statistics
+
+**Example:**
+```javascript
+const { scanDirectories, findFilesByExtension, getScanStatistics } = require('./fileScanner');
+
+// Scan with options
+const files = await scanDirectories({
+  targets: ['./src'],
+  extensions: ['js', 'ts'],
+  minSize: 1000,
+  maxSize: 1024 * 1024,
+  excludeDirs: ['node_modules'],
+});
+
+// Get statistics
+const stats = getScanStatistics(files);
+console.log(`Found ${stats.totalFiles} files, ${formatFileSize(stats.totalSize)} total`);
+```
+
+### fileMover.js
+File moving, copying, and management operations.
+
+**Key Functions:**
+- `moveSingleFile(source, destination, options)` - Move single file
+- `moveMultipleFiles(filePairs, options)` - Move multiple files
+- `moveFilesToDirectory(filePaths, destDir, options)` - Move files to directory
+- `moveDuplicateFiles(duplicates, destDir, options)` - Move duplicate files
+- `batchMoveFiles(filePairs, options)` - Batch move with progress tracking
+- `createHardLink(source, destination, options)` - Create hard links
+
+**Example:**
+```javascript
+const { moveDuplicateFiles, batchMoveFiles } = require('./fileMover');
+
+// Move duplicates
+const duplicates = [
+  { path: './file1.txt', size: 1024 },
+  { path: './file2.txt', size: 1024 }
+];
+
+await moveDuplicateFiles(duplicates, './duplicates', {
+  overwrite: false,
+  createBackup: true,
+  dryRun: false
+});
+
+// Batch move with progress
+const filePairs = [
+  { source: './src/file1.js', destination: './dest/file1.js' },
+  { source: './src/file2.js', destination: './dest/file2.js' }
+];
+
+const results = await batchMoveFiles(filePairs, {
+  onProgress: (progress) => console.log(`Moving: ${progress.source}`)
+});
+```
+
+### fileHasher.js
+File hashing and content comparison operations.
+
+**Key Functions:**
+- `calculateFileHash(filePath, options)` - Calculate file hash
+- `calculateMultipleHashes(filePath, algorithms, options)` - Multiple algorithms
+- `findDuplicateFilesByHash(files, options)` - Find duplicates by hash
+- `compareFilesByHash(file1, file2, options)` - Compare two files
+- `calculateHashesInParallel(filePaths, options)` - Parallel hashing
+- `verifyFileIntegrity(filePath, expectedHash, options)` - Verify file integrity
+
+**Example:**
+```javascript
+const { calculateFileHash, findDuplicateFilesByHash, compareFilesByHash } = require('./fileHasher');
+
+// Calculate hash
+const hashResult = await calculateFileHash('./file.txt', {
+  algorithm: 'sha256',
+  onProgress: (progress) => console.log(`${progress.percentage}%`)
+});
+
+// Find duplicates
+const duplicates = await findDuplicateFilesByHash(files, {
+  algorithm: 'sha256',
+  maxConcurrency: 5
+});
+
+// Compare files
+const areIdentical = await compareFilesByHash('./file1.txt', './file2.txt');
+```
+
+## ⚙️ Configuration Options
+
+### Scan Options
+```javascript
+const scanOptions = {
+  extensions: ['js', 'md'],        // Filter by extensions
+  minSize: 1000,                   // Minimum file size (bytes)
+  maxSize: 10 * 1024 * 1024,      // Maximum file size (bytes)
+,            // Include hidden files
+  excludeDirs: ['node_modules'],   // Directories to exclude
+  excludeFiles: ['*.log'],         // File patterns to exclude
+  followSymlinks: false,           // Follow symbolic links
+  maxDepth: Infinity               // Maximum directory depth
+};
+```
+
+### Move Options
+```javascript
+const moveOptions = {
+  overwrite: false,                // Overwrite existing files
+  createBackup: true,              // Create backup before moving
+  backupSuffix: '.backup',         // Backup file suffix
+  dryRun: false,                   // Simulate without actually moving
+  onProgress: (progress) => {},    // Progress callback
+  onError: (error, result) => {}   // Error callback
+};
+```
+
+### Hash Options
+```javascript
+const hashOptions = {
+  algorithm: 'sha256',             // Hash algorithm
+  chunkSize: 64 * 1024,           // Chunk size for streaming
+  includeMetadata: false,          // Include file metadata
+  onProgress: (progress) => {},    // Progress callback
+  maxFileSize: Infinity,           // Maximum file size to hash
+  maxConcurrency: 5                // Max parallel operations
+};
+```
+
+## 🔧 Available Hash Algorithms
+
+- `md5` - MD5 (fast, less secure)
+- `sha1` - SHA-1 (fast, moderate security)
+- `sha256` - SHA-256 (recommended, good security)
+- `sha512` - SHA-512 (slower, high security)
+- `blake2b512` - BLAKE2b-512 (fast, high security)
+- `blake2s256` - BLAKE2s-256 (fast, good security)
+
+## 📊 Performance Tips
+
+1. **Use size-based filtering first** - Filter by file size before hashing
+2. **Limit file size** - Set `maxFileSize` to avoid hashing huge files
+3. **Use parallel operations** - Set `maxConcurrency` for batch operations
+4. **Stream large files** - Use appropriate `chunkSize` for streaming
+5. **Exclude unnecessary directories** - Use `excludeDirs` to skip irrelevant paths
+
+## 🚨 Error Handling
+
+All utilities include comprehensive error handling:
+
+```javascript
+try {
+  const files = await scanDirectories({ ...options, targets: ['./src'] });
+} catch (error) {
+  console.error('Scan failed:', error.message);
+  // Handle specific error types
+  if (error.code === 'ENOENT') {
+    console.error('Directory not found');
+  }
+}
+```
+
+## 📝 Examples
+
+See `examples.js` for comprehensive usage examples including:
+- Basic file operations
+- File scanning workflows
+- Duplicate detection
+- File moving strategies
+- Complete deduplication workflow
+
+## 🔗 Integration
+
+The utilities are designed to work together seamlessly:
+
+```javascript
+// Complete deduplication workflow
+const files = await scanDirectories({ ...scanOptions, targets });
+const sizeDuplicates = findDuplicatesBySize(files);
+const hashDuplicates = await findDuplicateFilesByHash(sizeDuplicates);
+await moveDuplicateFiles(hashDuplicates, './duplicates');
+```
+
+## 📄 License
+
+MIT License - See main project license for details.