@tb.p/dd 1.0.0

package/utils/candidateDetection.js

@@ -0,0 +1,541 @@
+import { calculateContentHash } from './fileHasher.js';
+import { pathExists, getFileSize } from './fileSystemUtils.js';
+import { createConnection } from '../database/dbConnection.js';
+import { createOperations } from '../database/dbOperations.js';
+
+/**
+ * Candidate Detection for @tb.p/dd
+ * Groups files by size and hashes candidates for duplicate detection
+ */
+class CandidateDetection {
+  constructor(dbOperations, options = {}) {
+    this.db = dbOperations;
+    this.options = {
+      hashAlgorithm: 'blake3',
+      batchSize: 100,
+      maxConcurrency: 5,
+      verbose: false,
+      onProgress: null,
+      ...options
+    };
+    this.stats = {
+      totalFiles: 0,
+      uniqueFiles: 0,
+      candidatesFound: 0,
+      hashedFiles: 0,
+      skippedFiles: 0,
+      errors: 0,
+      startTime: null,
+      endTime: null
+    };
+  }
+
+  /**
+   * Detect candidates and process them
+   */
+  async detectAndProcessCandidates(options = {}) {
+    const config = { ...this.options, ...options };
+    
+    if (config.verbose) {
+      console.log('🔍 Starting candidate detection and processing...');
+    }
+
+    this.stats.startTime = Date.now();
+
+    try {
+      // Find candidates
+      const detectionResult = await this.findCandidates();
+      
+      if (!detectionResult.success) {
+        return {
+          success: false,
+          error: `Candidate detection failed: ${detectionResult.error}`,
+          stats: this.stats
+        };
+      }
+
+      if (config.verbose) {
+        console.log(`📊 Found ${detectionResult.totalCandidates} candidates in ${detectionResult.candidateGroups} size groups`);
+      }
+
+      // Process candidates if any found
+      if (detectionResult.totalCandidates > 0) {
+        const processResult = await this.processCandidates();
+        
+        if (!processResult.success) {
+          return {
+            success: false,
+            error: `Candidate processing failed: ${processResult.error}`,
+            stats: this.stats
+          };
+        }
+
+        if (config.verbose) {
+          console.log(`✅ Processed ${processResult.hashedFiles} files successfully`);
+          if (processResult.skippedFiles > 0) {
+            console.log(`⏭️  ${processResult.skippedFiles} files already had hashes, skipped`);
+          }
+          if (processResult.errors > 0) {
+            console.log(`⚠️  ${processResult.errors} files had errors during processing`);
+          }
+        }
+
+        return {
+          success: true,
+          detection: detectionResult,
+          processing: processResult,
+          stats: this.stats
+        };
+      } else {
+        if (config.verbose) {
+          console.log('ℹ️  No candidates found for processing');
+        }
+
+        return {
+          success: true,
+          detection: detectionResult,
+          processing: { hashedFiles: 0, errors: 0 },
+          stats: this.stats
+        };
+      }
+
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message,
+        stats: this.stats
+      };
+    }
+  }
+
+  /**
+   * Find all files with matching sizes (candidates for hashing)
+   */
+  async findCandidates() {
+    this.stats.totalFiles = 0;
+    this.stats.uniqueFiles = 0;
+    this.stats.candidatesFound = 0;
+
+    try {
+      // First check if there are any unhashed active files at all
+      const totalUnhashed = await this.db.db.queryOne('SELECT COUNT(*) as total FROM copies WHERE file_hash IS NULL AND active = 1');
+      
+      if (totalUnhashed.total === 0) {
+        console.log('ℹ️  No unhashed active files found - all active files already processed');
+        this.stats.endTime = Date.now();
+        return {
+          success: true,
+          stats: this.stats,
+          candidateGroups: 0,
+          totalCandidates: 0,
+          message: 'No unhashed active files found'
+        };
+      }
+
+      // Get all files grouped by size
+      const sizeGroups = await this.getFilesBySize();
+      
+      // Filter groups that have more than one file (candidates)
+      const candidateGroups = sizeGroups.filter(group => group.count > 1);
+      
+      this.stats.candidatesFound = candidateGroups.reduce((total, group) => total + group.count, 0);
+      this.stats.totalFiles = sizeGroups.reduce((total, group) => total + group.count, 0);
+      this.stats.uniqueFiles = sizeGroups.length;
+
+      if (this.options.onProgress) {
+        this.options.onProgress({
+          phase: 'candidate_detection',
+          candidatesFound: this.stats.candidatesFound,
+          totalFiles: this.stats.totalFiles,
+          candidateGroups: candidateGroups.length
+        });
+      }
+
+      return {
+        success: true,
+        stats: this.stats,
+        candidateGroups: candidateGroups.length,
+        totalCandidates: this.stats.candidatesFound
+      };
+
+    } catch (error) {
+      this.stats.endTime = Date.now();
+      return {
+        success: false,
+        error: error.message,
+        stats: this.stats
+      };
+    }
+  }
+
+  /**
+   * Process candidates by calculating hashes and updating database
+   */
+  async processCandidates() {
+    try {
+      // First check if there are any unhashed files at all
+      const totalUnhashed = await this.db.db.queryOne('SELECT COUNT(*) as total FROM copies WHERE file_hash IS NULL AND active = 1');
+      
+      if (totalUnhashed.total === 0) {
+        console.log('ℹ️  No unhashed active files found - all active files already processed');
+        this.stats.endTime = Date.now();
+        return {
+          success: true,
+          message: 'No unhashed active files found - all active files already processed',
+          stats: this.stats,
+          hashedFiles: 0,
+          errors: 0
+        };
+      }
+
+      // Get all files that need hashing (candidates with matching sizes)
+      const candidates = await this.getCandidatesForHashing();
+      
+      if (candidates.length === 0) {
+        return {
+          success: true,
+          message: 'No candidates found for hashing',
+          stats: this.stats,
+          hashedFiles: 0,
+          errors: 0
+        };
+      }
+
+      // Process candidates in batches
+      const results = await this.processCandidatesInBatches(candidates);
+      
+      this.stats.endTime = Date.now();
+      this.stats.hashedFiles = results.hashedFiles;
+      this.stats.skippedFiles = results.skippedFiles;
+      this.stats.errors = results.errors;
+
+        return {
+          success: true,
+          stats: this.stats,
+          hashedFiles: results.hashedFiles,
+          skippedFiles: results.skippedFiles,
+          errors: results.errors,
+          processingTime: this.stats.endTime - this.stats.startTime
+        };
+
+    } catch (error) {
+      this.stats.endTime = Date.now();
+      return {
+        success: false,
+        error: error.message,
+        stats: this.stats
+      };
+    }
+  }
+
+  /**
+   * Get files grouped by size from database
+   */
+  async getFilesBySize() {
+    const sql = `
+      SELECT 
+        file_size,
+        COUNT(id) as count,
+        GROUP_CONCAT(id, ',') as file_ids
+      FROM copies 
+      WHERE file_hash IS NULL AND active = 1
+      GROUP BY file_size
+      ORDER BY file_size DESC
+    `;
+
+    const results = await this.db.db.query(sql);
+    
+    return results.map(row => {
+      const count = parseInt(row.count) || 0;
+      
+      // Debug logging for negative counts
+      if (count < 0) {
+        console.error('Negative count detected in getFilesBySize:');
+        console.error('  Row data:', row);
+        console.error('  Parsed count:', count);
+        console.error('  Raw count value:', row.count);
+        console.error('  Type of count:', typeof row.count);
+      }
+      
+      return {
+        size: row.file_size,
+        count: Math.max(0, count), // Ensure count is never negative
+        fileIds: row.file_ids ? row.file_ids.split(',').map(id => parseInt(id)) : []
+      };
+    });
+  }
+
+  /**
+   * Get candidate files that need hashing (files with matching sizes)
+   */
+  async getCandidatesForHashing() {
+    const sql = `
+      SELECT c1.*
+      FROM copies c1
+      WHERE c1.file_hash IS NULL
+      AND EXISTS (
+        SELECT 1 
+        FROM copies c2 
+        WHERE c2.file_size = c1.file_size 
+        AND c2.id != c1.id
+        AND c2.file_hash IS NULL
+      )
+      ORDER BY c1.file_size DESC, c1.file_path
+    `;
+
+    return await this.db.db.query(sql);
+  }
+
+  /**
+   * Process candidates in batches with controlled concurrency
+   */
+  async processCandidatesInBatches(candidates) {
+    const results = {
+      hashedFiles: 0,
+      skippedFiles: 0,
+      errors: 0
+    };
+
+    // Process in batches
+    for (let i = 0; i < candidates.length; i += this.options.batchSize) {
+      const batch = candidates.slice(i, i + this.options.batchSize);
+      
+      // Process batch with controlled concurrency
+      const batchResults = await this.processBatch(batch);
+      
+      results.hashedFiles += batchResults.hashedFiles;
+      results.skippedFiles += batchResults.skippedFiles;
+      results.errors += batchResults.errors;
+
+      // Report progress
+      if (this.options.onProgress) {
+        this.options.onProgress({
+          phase: 'hashing',
+          processed: i + batch.length,
+          total: candidates.length,
+          hashedFiles: results.hashedFiles,
+          errors: results.errors,
+          percentage: ((i + batch.length) / candidates.length) * 100
+        });
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Process a batch of candidates with controlled concurrency
+   */
+  async processBatch(batch) {
+    const results = {
+      hashedFiles: 0,
+      skippedFiles: 0,
+      errors: 0
+    };
+
+    // Process with controlled concurrency
+    const chunks = [];
+    for (let i = 0; i < batch.length; i += this.options.maxConcurrency) {
+      chunks.push(batch.slice(i, i + this.options.maxConcurrency));
+    }
+
+    for (const chunk of chunks) {
+      const chunkPromises = chunk.map(candidate => this.processCandidate(candidate));
+      const chunkResults = await Promise.all(chunkPromises);
+      
+      chunkResults.forEach(result => {
+        if (result.success) {
+          if (result.skipped) {
+            results.skippedFiles++;
+          } else {
+            results.hashedFiles++;
+          }
+        } else {
+          results.errors++;
+        }
+      });
+    }
+
+    return results;
+  }
+
+  /**
+   * Process a single candidate file
+   */
+  async processCandidate(candidate) {
+    try {
+      // Report current file being hashed
+      if (this.options.onProgress) {
+        this.options.onProgress({
+          phase: 'hashing',
+          currentFile: candidate.file_path
+        });
+      }
+
+      // Skip if file already has a hash
+      if (candidate.file_hash !== null && candidate.file_hash !== undefined) {
+        return {
+          success: true,
+          candidateId: candidate.id,
+          filePath: candidate.file_path,
+          message: 'File already hashed, skipping',
+          skipped: true
+        };
+      }
+
+      // Verify file still exists
+      if (!(await pathExists(candidate.file_path))) {
+        return {
+          success: false,
+          error: `File not found: ${candidate.file_path}`,
+          candidateId: candidate.id
+        };
+      }
+
+      // Verify file size hasn't changed
+      const currentSize = await getFileSize(candidate.file_path);
+      if (currentSize !== candidate.file_size) {
+        return {
+          success: false,
+          error: `File size changed: ${candidate.file_path} (expected: ${candidate.file_size}, actual: ${currentSize})`,
+          candidateId: candidate.id
+        };
+      }
+
+      // Calculate hash
+      const hash = await calculateContentHash(candidate.file_path, this.options.hashAlgorithm);
+      
+      // Update database
+      const updateResult = await this.db.updateFileHash(candidate.id, hash);
+      
+      if (!updateResult.success) {
+        return {
+          success: false,
+          error: `Database update failed: ${updateResult.error}`,
+          candidateId: candidate.id
+        };
+      }
+
+      return {
+        success: true,
+        candidateId: candidate.id,
+        filePath: candidate.file_path,
+        hash: hash,
+        size: candidate.file_size
+      };
+
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message,
+        candidateId: candidate.id
+      };
+    }
+  }
+
+  /**
+   * Get detailed candidate information
+   */
+  async getCandidateDetails() {
+    try {
+      const sizeGroups = await this.getFilesBySize();
+      const candidateGroups = sizeGroups.filter(group => group.count > 1);
+      
+      const details = {
+        totalSizeGroups: sizeGroups.length,
+        candidateGroups: candidateGroups.length,
+        totalCandidates: candidateGroups.reduce((total, group) => total + group.count, 0),
+        sizeDistribution: candidateGroups.map(group => ({
+          size: group.size,
+          count: group.count,
+          sizeFormatted: this.formatFileSize(group.size)
+        })).sort((a, b) => b.size - a.size)
+      };
+
+      return {
+        success: true,
+        details
+      };
+
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message
+      };
+    }
+  }
+
+  /**
+   * Format file size in human readable format
+   */
+  formatFileSize(bytes) {
+    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    let size = bytes;
+    let unitIndex = 0;
+
+    while (size >= 1024 && unitIndex < units.length - 1) {
+      size /= 1024;
+      unitIndex++;
+    }
+
+    return `${size.toFixed(2)} ${units[unitIndex]}`;
+  }
+}
+
+/**
+ * Run candidate detection with database connection
+ */
+export async function runCandidateDetection(options) {
+  const databasePath = options.saveDb || options.resumeDb || options.database;
+
+  if (!databasePath) {
+    console.log('ℹ️  No database path available for candidate detection');
+    return;
+  }
+
+  let db;
+  try {
+    console.log('\n🔍 Starting candidate detection...');
+
+    db = createConnection(databasePath);
+    await db.connect();
+
+    const dbOps = createOperations(db);
+
+    const detector = new CandidateDetection(dbOps, {
+      hashAlgorithm: options.hashAlgorithm || 'blake3',
+      batchSize: parseInt(options.batchSize) || 100,
+      maxConcurrency: parseInt(options.maxConcurrency) || 5,
+      verbose: options.verbose || false
+    });
+
+    const result = await detector.detectAndProcessCandidates();
+
+    if (result.success) {
+      console.log('\n✅ Candidate detection completed successfully!');
+      console.log(`📊 Statistics:`);
+      console.log(`   - Total files: ${result.stats.totalFiles}`);
+      console.log(`   - Unique files: ${result.stats.uniqueFiles}`);
+      console.log(`   - Candidates found: ${result.stats.candidatesFound}`);
+      console.log(`   - Files hashed: ${result.stats.hashedFiles}`);
+      console.log(`   - Files skipped: ${result.stats.skippedFiles}`);
+      console.log(`   - Errors: ${result.stats.errors}`);
+      if (result.stats.processingTime) {
+        console.log(`   - Processing time: ${(result.stats.processingTime / 1000).toFixed(2)}s`);
+      }
+    } else {
+      console.error(`❌ Candidate detection failed: ${result.error}`);
+      console.log('Continuing without candidate detection...');
+    }
+
+    await db.close();
+  } catch (error) {
+    console.error(`❌ Candidate detection error: ${error.message}`);
+    console.log('Continuing without candidate detection...');
+    if (db) {
+      await db.close();
+    }
+  }
+}
+
+export { CandidateDetection };

package/utils/duplicateMover.js

@@ -0,0 +1,140 @@
+import path from 'path';
+import { createDatabaseManager } from '../database/index.js';
+import { moveDuplicateFiles, moveDuplicateFilesWithPathPreservation } from './fileMover.js';
+import { createDirectory } from './fileSystemUtils.js';
+
+/**
+ * Move duplicated files to a duplicates directory
+ * @param {Object} options - Processed options containing database path and preservePaths setting
+ * @returns {Promise<void>}
+ */
+async function moveDuplicates(options) {
+  const preservePaths = options.preservePaths !== false;
+  // Determine database path
+  const dbPath = options.saveDb || options.resumeDb;
+  if (!dbPath) {
+    console.error('Error: No database path available for move operation');
+    return;
+  }
+
+  console.log('Starting duplicate file move operation...');
+  console.log(`Path preservation: ${preservePaths ? 'enabled' : 'disabled'}`);
+
+  let dbManager;
+  try {
+    // Initialize database connection
+    dbManager = createDatabaseManager(dbPath);
+    await dbManager.initialize();
+
+    // Get duplicate groups from database
+    const duplicateGroups = await dbManager.getOperations().getDuplicateGroups();
+    
+    if (duplicateGroups.length === 0) {
+      console.log('No duplicate files found to move.');
+      return;
+    }
+
+    console.log(`Found ${duplicateGroups.length} duplicate groups`);
+
+    // Create duplicates directories in each target
+    const duplicatesDirs = {};
+    for (const target of options.targets) {
+      const duplicatesDir = path.join(target, '!@duplicates');
+      await createDirectory(duplicatesDir);
+      duplicatesDirs[target] = duplicatesDir;
+    }
+
+    let totalMoved = 0;
+    let totalErrors = 0;
+
+    // Process each duplicate group
+    for (const group of duplicateGroups) {
+      const filePaths = group.file_paths.split(';');
+      const duplicateFiles = filePaths.map(filePath => ({
+        path: filePath,
+        size: group.file_size,
+        hash: group.file_hash
+      }));
+
+      // Sort by priority (keep the first file, move the rest)
+      // The first file in the group should be kept, others moved
+      const filesToMove = duplicateFiles.slice(1); // Skip the first file (keep it)
+
+      if (filesToMove.length > 0) {
+        // Group files by their target directory
+        const filesByTarget = {};
+        for (const file of filesToMove) {
+          const fileDir = path.dirname(file.path);
+          // Find which target directory this file belongs to
+          let targetDir = null;
+          for (const target of options.targets) {
+            // Resolve both paths to absolute for proper comparison
+            const absoluteTarget = path.resolve(target);
+            if (fileDir.startsWith(absoluteTarget)) {
+              targetDir = target;
+              break;
+            }
+          }
+
+          if (!targetDir) {
+            // If file doesn't belong to any target, use the first target
+            targetDir = options.targets[0];
+          }
+          
+          if (!filesByTarget[targetDir]) {
+            filesByTarget[targetDir] = [];
+          }
+          filesByTarget[targetDir].push(file);
+        }
+
+        // Move files to their respective target duplicates directories
+        for (const [targetDir, files] of Object.entries(filesByTarget)) {
+          try {
+            const moveOptions = {
+              dryRun: false,
+              overwrite: false,
+              onProgress: (progress) => {
+                if (options.verbose) {
+                  console.log(`Moving: ${progress.source} -> ${progress.destination}`);
+                }
+              },
+              onError: (error, result) => {
+                console.error(`Error moving file ${result.sourcePath}: ${error.message}`);
+                totalErrors++;
+              }
+            };
+
+            const moveResults = preservePaths 
+              ? await moveDuplicateFilesWithPathPreservation(files, duplicatesDirs[targetDir], targetDir, moveOptions)
+              : await moveDuplicateFiles(files, duplicatesDirs[targetDir], moveOptions);
+
+            const successfulMoves = moveResults.filter(result => result.success);
+            totalMoved += successfulMoves.length;
+            totalErrors += moveResults.length - successfulMoves.length;
+
+          } catch (error) {
+            console.error(`Error processing duplicate group for target ${targetDir}: ${error.message}`);
+            totalErrors += files.length;
+          }
+        }
+      }
+    }
+
+    console.log(`Move operation completed:`);
+    console.log(`  - Files moved: ${totalMoved}`);
+    console.log(`  - Errors: ${totalErrors}`);
+    console.log(`  - Duplicates directories created:`);
+    for (const [target, duplicatesDir] of Object.entries(duplicatesDirs)) {
+      console.log(`    - ${target}/!@duplicates`);
+    }
+
+  } catch (error) {
+    console.error(`Error during move operation: ${error.message}`);
+  } finally {
+    if (dbManager) {
+      await dbManager.close();
+    }
+  }
+}
+
+export { moveDuplicates };