@tb.p/dd 1.0.0

package/utils/duplicateReporter.js

@@ -0,0 +1,91 @@
+import { createConnection } from '../database/dbConnection.js';
+
+/**
+ * Display duplicate files to the user
+ * @param {Object} options - Options containing database path
+ */
+export async function displayDuplicates(options) {
+  const databasePath = options.saveDb || options.resumeDb || options.database;
+
+  if (!databasePath) {
+    console.log('ℹ️  No database path available for duplicate reporting');
+    return;
+  }
+
+  let db;
+  try {
+    console.log('\n📋 Generating duplicate file report...');
+
+    db = createConnection(databasePath);
+    await db.connect();
+
+    const operations = await import('../database/dbOperations.js');
+    const dbOps = operations.createOperations(db);
+
+    const duplicateGroups = await dbOps.getDuplicateGroups();
+
+    if (!duplicateGroups || duplicateGroups.length === 0) {
+      console.log('✅ No duplicate files found!');
+      return;
+    }
+
+    console.log(`\n🔍 Found ${duplicateGroups.length} duplicate groups:\n`);
+
+    let totalDuplicates = 0;
+    let totalWastedSpace = 0;
+
+    duplicateGroups.forEach((group, index) => {
+      const filePaths = group.file_paths.split(';');
+      const duplicateCount = group.duplicate_count;
+      const fileSize = group.file_size;
+      const wastedSpace = (duplicateCount - 1) * fileSize;
+
+      totalDuplicates += duplicateCount;
+      totalWastedSpace += wastedSpace;
+
+      console.log(`📁 Duplicate Group ${index + 1} (${duplicateCount} files, ${formatFileSize(fileSize)} each):`);
+      console.log(`   Hash: ${group.file_hash}`);
+      console.log(`   Wasted space: ${formatFileSize(wastedSpace)}`);
+      console.log(`   Files:`);
+
+      filePaths.forEach((filePath, fileIndex) => {
+        const icon = fileIndex === 0 ? '🟢' : '🔴';
+        const label = fileIndex === 0 ? '[ORIGINAL]' : '[DUPLICATE]';
+        console.log(`     ${icon} ${label} ${filePath}`);
+      });
+
+      console.log('');
+    });
+
+    console.log(`📊 Summary:`);
+    console.log(`   - Duplicate groups: ${duplicateGroups.length}`);
+    console.log(`   - Total duplicate files: ${totalDuplicates - duplicateGroups.length}`);
+    console.log(`   - Total wasted space: ${formatFileSize(totalWastedSpace)}`);
+
+    await db.close();
+
+  } catch (error) {
+    console.error(`❌ Error generating duplicate report: ${error.message}`);
+    if (db) {
+      await db.close();
+    }
+  }
+}
+
+/**
+ * Format file size in human readable format
+ * @param {number} bytes - Size in bytes
+ * @returns {string} - Formatted size
+ */
+function formatFileSize(bytes) {
+  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+  let size = bytes;
+  let unitIndex = 0;
+
+  while (size >= 1024 && unitIndex < units.length - 1) {
+    size /= 1024;
+    unitIndex++;
+  }
+
+  return `${size.toFixed(2)} ${units[unitIndex]}`;
+}

package/utils/fileHasher.js

@@ -0,0 +1,195 @@
+import fs from 'fs';
+import crypto from 'crypto';
+import { blake3 } from 'hash-wasm';
+import { pathExists, getFileSize } from './fileSystemUtils.js';
+
+/**
+ * File Hasher Utilities for @tb.p/dd
+ * Advanced file hashing and content comparison operations
+ */
+
+/**
+ * Hash algorithm types
+ */
+const HASH_ALGORITHMS = {
+  MD5: 'md5',
+  SHA1: 'sha1',
+  SHA256: 'sha256',
+  SHA512: 'sha512',
+  BLAKE3: 'blake3'
+};
+
+/**
+ * Hash options
+ * @typedef {Object} HashOptions
+ * @property {string} [algorithm] - Hash algorithm to use
+ * @property {number} [chunkSize] - Chunk size for streaming (bytes)
+ * @property {boolean} [includeMetadata] - Include file metadata in hash
+ * @property {Function} [onProgress] - Progress callback function
+ * @property {number} [maxFileSize] - Maximum file size to hash (bytes)
+ */
+
+/**
+ * Hash result
+ * @typedef {Object} HashResult
+ * @property {string} hash - Calculated hash
+ * @property {string} algorithm - Algorithm used
+ * @property {number} size - File size in bytes
+ * @property {number} [timeMs] - Time taken in milliseconds
+ * @property {string} [error] - Error message if failed
+ */
+
+/**
+ * Calculate file hash using streaming
+ * @param {string} filePath - Path to file
+ * @param {HashOptions} options - Hash options
+ * @returns {Promise<HashResult>} - Hash result
+ */
+async function calculateFileHash(filePath, options = {}) {
+  const {
+    algorithm = HASH_ALGORITHMS.BLAKE3,
+    chunkSize = 64 * 1024, // 64KB chunks
+    includeMetadata = false,
+    onProgress = null,
+    maxFileSize = Infinity
+  } = options;
+
+  const startTime = Date.now();
+  const result = {
+    hash: null,
+    algorithm,
+    size: 0,
+    timeMs: 0,
+    error: null
+  };
+
+  try {
+    // Check if file exists
+    if (!await pathExists(filePath)) {
+      throw new Error(`File does not exist: ${filePath}`);
+    }
+
+    // Get file size
+    result.size = await getFileSize(filePath);
+
+    // Check file size limit
+    if (result.size > maxFileSize) {
+      throw new Error(`File too large: ${result.size} bytes (max: ${maxFileSize})`);
+    }
+
+    // Handle BLAKE3 vs other algorithms
+    if (algorithm === HASH_ALGORITHMS.BLAKE3) {
+      return await calculateBlake3Hash(filePath, result, includeMetadata, onProgress, startTime);
+    } else {
+      // Create hash instance for crypto algorithms
+      const hash = crypto.createHash(algorithm);
+
+      // Add metadata to hash if requested
+      if (includeMetadata) {
+        const stats = fs.statSync(filePath);
+        hash.update(JSON.stringify({
+          size: stats.size,
+          mtime: stats.mtime.getTime(),
+          ctime: stats.ctime.getTime(),
+          mode: stats.mode
+        }));
+      }
+
+      // Stream file and calculate hash
+      const stream = fs.createReadStream(filePath, { highWaterMark: chunkSize });
+      let bytesProcessed = 0;
+
+      return new Promise((resolve, reject) => {
+        stream.on('data', (chunk) => {
+          hash.update(chunk);
+          bytesProcessed += chunk.length;
+
+          if (onProgress) {
+            onProgress({
+              bytesProcessed,
+              totalBytes: result.size,
+              percentage: (bytesProcessed / result.size) * 100
+            });
+          }
+        });
+
+        stream.on('end', () => {
+          result.hash = hash.digest('hex');
+          result.timeMs = Date.now() - startTime;
+          resolve(result);
+        });
+
+        stream.on('error', (error) => {
+          result.error = error.message;
+          reject(error);
+        });
+      });
+    }
+
+  } catch (error) {
+    result.error = error.message;
+    result.timeMs = Date.now() - startTime;
+    throw error;
+  }
+}
+
+/**
+ * Calculate BLAKE3 hash for a file
+ * @param {string} filePath - Path to file
+ * @param {Object} result - Result object to update
+ * @param {boolean} includeMetadata - Include file metadata in hash
+ * @param {Function} onProgress - Progress callback function
+ * @param {number} startTime - Start time for timing
+ * @returns {Promise<Object>} - Hash result
+ */
+async function calculateBlake3Hash(filePath, result, includeMetadata, onProgress, startTime) {
+  try {
+    // Read file as buffer for BLAKE3
+    const fileBuffer = await fs.promises.readFile(filePath);
+
+    // Add metadata to hash if requested
+    if (includeMetadata) {
+      const stats = fs.statSync(filePath);
+      const metadata = JSON.stringify({
+        size: stats.size,
+        mtime: stats.mtime.getTime(),
+        ctime: stats.ctime.getTime(),
+        mode: stats.mode
+      });
+
+      // Combine metadata and file content
+      const combinedBuffer = Buffer.concat([
+        Buffer.from(metadata),
+        fileBuffer
+      ]);
+
+      result.hash = await blake3(combinedBuffer);
+    } else {
+      result.hash = await blake3(fileBuffer);
+    }
+
+    result.timeMs = Date.now() - startTime;
+    return result;
+  } catch (error) {
+    result.error = error.message;
+    result.timeMs = Date.now() - startTime;
+    throw error;
+  }
+}
+
+/**
+ * Calculate hash for file content only (no metadata)
+ * @param {string} filePath - Path to file
+ * @param {string} algorithm - Hash algorithm
+ * @returns {Promise<string>} - File content hash
+ */
+async function calculateContentHash(filePath, algorithm = HASH_ALGORITHMS.BLAKE3) {
+  const result = await calculateFileHash(filePath, { algorithm, includeMetadata: false });
+  return result.hash;
+}
+
+export {
+  HASH_ALGORITHMS,
+  calculateFileHash,
+  calculateContentHash
+};

package/utils/fileMover.js

@@ -0,0 +1,180 @@
+import path from 'path';
+import {
+  pathExists,
+  createDirectory,
+  moveFile,
+  getFileStats,
+  joinPaths
+} from './fileSystemUtils.js';
+
+/**
+ * Move operation result
+ * @typedef {Object} MoveResult
+ * @property {boolean} success - Whether operation was successful
+ * @property {string} sourcePath - Original source path
+ * @property {string} destinationPath - Final destination path
+ * @property {string} [error] - Error message if failed
+ * @property {number} [size] - File size in bytes
+ */
+
+/**
+ * Move a single file
+ * @param {string} sourcePath - Source file path
+ * @param {string} destinationPath - Destination file path
+ * @param {Object} options - Move options
+ * @returns {Promise<MoveResult>} - Move result
+ */
+async function moveSingleFile(sourcePath, destinationPath, options = {}) {
+  const {
+    overwrite = false,
+    dryRun = false,
+    onProgress = null,
+    onError = null
+  } = options;
+
+  const result = {
+    success: false,
+    sourcePath,
+    destinationPath,
+    error: null,
+    size: 0
+  };
+
+  try {
+    // Check if source exists
+    if (!await pathExists(sourcePath)) {
+      throw new Error(`Source file does not exist: ${sourcePath}`);
+    }
+
+    // Get file size
+    const stats = await getFileStats(sourcePath);
+    result.size = stats.size;
+
+    // Check if destination exists
+    if (!overwrite && await pathExists(destinationPath)) {
+      throw new Error(`Destination file already exists: ${destinationPath}`);
+    }
+
+    if (dryRun) {
+      console.log(`[DRY RUN] Would move: ${sourcePath} -> ${destinationPath}`);
+      result.success = true;
+      return result;
+    }
+
+    // Ensure destination directory exists
+    const destinationDir = path.dirname(destinationPath);
+    await createDirectory(destinationDir);
+
+    // Move the file
+    await moveFile(sourcePath, destinationPath, { overwrite });
+
+    result.success = true;
+
+    if (onProgress) {
+      onProgress(result);
+    }
+
+  } catch (error) {
+    result.error = error.message;
+    result.success = false;
+
+    if (onError) {
+      onError(result);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Move multiple files
+ * @param {Array<{source: string, destination: string}>} filePairs - Array of source/destination pairs
+ * @param {Object} options - Move options
+ * @returns {Promise<MoveResult[]>} - Array of move results
+ */
+async function moveMultipleFiles(filePairs, options = {}) {
+  const results = [];
+
+  for (const pair of filePairs) {
+    const result = await moveSingleFile(pair.source, pair.destination, options);
+    results.push(result);
+  }
+
+  return results;
+}
+
+/**
+ * Move duplicate files to a duplicates directory
+ * @param {Array<{path: string, size: number, hash?: string}>} duplicateFiles - Duplicate files info
+ * @param {string} duplicatesDir - Directory for duplicates
+ * @param {Object} options - Move options
+ * @returns {Promise<MoveResult[]>} - Array of move results
+ */
+async function moveDuplicateFiles(duplicateFiles, duplicatesDir, options = {}) {
+  const filePairs = [];
+
+  for (let i = 0; i < duplicateFiles.length; i++) {
+    const file = duplicateFiles[i];
+    const fileName = path.basename(file.path);
+    const nameWithoutExt = path.parse(fileName).name;
+    const ext = path.parse(fileName).ext;
+
+    // Create unique name for duplicate
+    let duplicateName = fileName;
+    if (i > 0) {
+      duplicateName = `${nameWithoutExt}_duplicate_${i}${ext}`;
+    }
+
+    filePairs.push({
+      source: file.path,
+      destination: joinPaths(duplicatesDir, duplicateName)
+    });
+  }
+
+  return await moveMultipleFiles(filePairs, options);
+}
+
+/**
+ * Move duplicate files to a duplicates directory preserving relative path structure
+ * @param {Array<{path: string, size: number, hash?: string}>} duplicateFiles - Duplicate files info
+ * @param {string} duplicatesDir - Directory for duplicates
+ * @param {string} sourceRoot - Root directory to calculate relative paths from
+ * @param {Object} options - Move options
+ * @returns {Promise<MoveResult[]>} - Array of move results
+ */
+async function moveDuplicateFilesWithPathPreservation(duplicateFiles, duplicatesDir, sourceRoot, options = {}) {
+  const filePairs = [];
+
+  for (let i = 0; i < duplicateFiles.length; i++) {
+    const file = duplicateFiles[i];
+    const fileName = path.basename(file.path);
+    const nameWithoutExt = path.parse(fileName).name;
+    const ext = path.parse(fileName).ext;
+
+    // Calculate relative path from source root
+    const relativePath = path.relative(sourceRoot, file.path);
+    const relativeDir = path.dirname(relativePath);
+
+    // Create unique name for duplicate
+    let duplicateName = fileName;
+    if (i > 0) {
+      duplicateName = `${nameWithoutExt}_duplicate_${i}${ext}`;
+    }
+
+    // Preserve directory structure in duplicates folder
+    const destinationDir = joinPaths(duplicatesDir, relativeDir);
+    const destinationPath = joinPaths(destinationDir, duplicateName);
+
+    filePairs.push({
+      source: file.path,
+      destination: destinationPath
+    });
+  }
+
+  return await moveMultipleFiles(filePairs, options);
+}
+
+export {
+  moveDuplicateFiles,
+  moveDuplicateFilesWithPathPreservation
+};

package/utils/fileScanner.js

@@ -0,0 +1,128 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import recursive from 'recursive-readdir';
+import {
+  pathExists,
+  isDirectory,
+  getFileExtension,
+  normalizePath
+} from './fileSystemUtils.js';
+
+async function scanDirectory(dirPath, options = {}) {
+  if (!(await pathExists(dirPath))) {
+    throw new Error(`Directory does not exist: ${dirPath}`);
+  }
+
+  if (!(await isDirectory(dirPath))) {
+    throw new Error(`Path is not a directory: ${dirPath}`);
+  }
+
+  try {
+    const filePaths = await new Promise((resolve, reject) => {
+      recursive(dirPath, (err, files) => {
+        if (err) reject(err);
+        else resolve(files);
+      });
+    });
+
+    // OPTIMIZATION: Batch stat() operations for better performance
+    const files = await getBatchFileInfo(filePaths, options);
+
+    return files;
+  } catch (error) {
+    console.warn(`Warning: Error reading directory ${dirPath}: ${error.message}`);
+    return [];
+  }
+}
+
+async function getBatchFileInfo(filePaths, options = {}) {
+  // OPTIMIZATION: Batch all stat() operations for parallel I/O
+  const statPromises = filePaths.map(async (filePath) => {
+    try {
+      const normalizedPath = normalizePath(filePath);
+      const stats = await fs.stat(normalizedPath);
+      return { filePath: normalizedPath, stats, error: null };
+    } catch (error) {
+      return { filePath, stats: null, error };
+    }
+  });
+
+  const statResults = await Promise.all(statPromises);
+  const files = [];
+
+  for (const result of statResults) {
+    if (result.error) {
+      console.warn(`Warning: Could not process file ${result.filePath}: ${result.error.message}`);
+      continue;
+    }
+
+    const absolutePath = path.resolve(result.filePath);
+    const extension = getFileExtension(result.filePath);
+
+    const fileInfo = {
+      path: absolutePath,
+      name: path.basename(result.filePath),
+      extension: extension,
+      size: result.stats.size,
+      hash: null, // Will be calculated separately if needed
+      active: true, // File is active by default
+    };
+
+    if (shouldIncludeFile(fileInfo, options)) {
+      files.push(fileInfo);
+    }
+  }
+
+  return files;
+}
+
+async function getFileInfo(filePath, options = {}) {
+  const normalizedPath = normalizePath(filePath);
+  const absolutePath = path.resolve(normalizedPath);
+  const stats = await fs.stat(normalizedPath);
+  const extension = getFileExtension(normalizedPath);
+
+  return {
+    path: absolutePath,
+    name: path.basename(normalizedPath),
+    extension: extension,
+    size: stats.size,
+    hash: null, // Will be calculated separately if needed
+    active: true, // File is active by default
+  };
+}
+
+function shouldIncludeFile(fileInfo, options) {
+  const {
+    extensions = null
+  } = options;
+
+  if (extensions && extensions.length > 0) {
+    if (!extensions.includes(fileInfo.extension)) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+async function scanDirectories(options = {}) {
+  const { targets = [] } = options;
+  const allFiles = [];
+
+  for (const dirPath of targets) {
+    try {
+      const files = await scanDirectory(dirPath, options);
+      allFiles.push(...files);
+    } catch (error) {
+      console.warn(`Warning: Failed to scan directory ${dirPath}: ${error.message}`);
+    }
+  }
+
+  return allFiles;
+}
+
+export {
+  scanDirectory,
+  scanDirectories
+};