@tb.p/dd 1.0.0

package/controllers/getExtensionsController.js

@@ -0,0 +1,27 @@
+import { scanDirectories } from '../utils/index.js';
+
+async function getExtensionsController(options) {
+  try {
+    const extensions = await getExtensions(options);
+    if (extensions.length > 0) {
+      process.stdout.write(extensions.join(',') + '\n');
+    }
+  } catch (error) {
+    console.error('Error:', error.message);
+    process.exit(1);
+  }
+}
+
+async function getExtensions(options) {
+  const files = await scanDirectories(options);
+  const extensions = new Set();
+  for (const file of files) {
+    if (file.extension) {
+      extensions.add(file.extension);
+    }
+  }
+  
+  return Array.from(extensions).sort();
+}
+
+export { getExtensionsController };

package/controllers/newController.js

@@ -0,0 +1,72 @@
+import { scanDirectory } from '../utils/index.js';
+import { createDatabaseManager } from '../database/index.js';
+
+async function newController(processedOptions) {
+  const { saveDb, verbose = false } = processedOptions;
+  
+  if (verbose) {
+    console.log(`Using database: ${saveDb}`);
+  }
+  
+  const dbManager = createDatabaseManager(saveDb);
+  await dbManager.initialize();
+  
+  try {
+    await performFileOperations(processedOptions, dbManager);
+  } finally {
+    await dbManager.close();
+  }
+}
+
+async function performFileOperations(options, dbManager) {
+  const { targets = [] } = options;
+  const allFiles = [];
+  const fileMap = new Map(); // For deduplication with priority
+  
+  for (let i = 0; i < targets.length; i++) {
+    const dirPath = targets[i];
+    const priority = i; // Lower index = higher priority
+    
+    try {
+      const files = await scanDirectory(dirPath, options);
+      
+      for (const file of files) {
+        const fileKey = `${file.path}_${file.name}`; // Create unique key
+        
+        if (!fileMap.has(fileKey) || fileMap.get(fileKey).priority > priority) {
+          fileMap.set(fileKey, { 
+            ...file, 
+            priority, 
+            dirGroup: dirPath,
+            active: true
+          });
+        }
+      }
+    } catch (error) {
+      console.warn(`Warning: Failed to scan directory ${dirPath}: ${error.message}`);
+    }
+  }
+  
+  // Convert map values back to array, maintaining priority order
+  allFiles.push(...Array.from(fileMap.values()).sort((a, b) => a.priority - b.priority));
+  
+  // Store files in database
+  let successCount = 0;
+  let errorCount = 0;
+  
+  for (const file of allFiles) {
+    const result = await dbManager.getOperations().addCopy(file);
+    if (result.success) {
+      successCount++;
+    } else {
+      errorCount++;
+      console.error(`Failed to add file ${file.path}: ${result.error}`);
+    }
+  }
+  
+  console.log(`Database operations: ${successCount} successful, ${errorCount} failed`);
+  
+  console.log(`Scanned and stored ${allFiles.length} files (deduplicated with priority)`);
+}
+
+export { newController };

package/controllers/resumeController.js

@@ -0,0 +1,233 @@
+import fs from 'fs';
+import { createDatabaseManager } from '../database/index.js';
+import { scanDirectory } from '../utils/index.js';
+
+/**
+ * Resume Controller for @tb.p/dd
+ * Handles resuming from existing database - sets all rows to active=false,
+ * scans for files with matching extensions, sets existing matches to active=true,
+ * adds new files normally
+ */
+async function resumeController(processedOptions) {
+  const { resumeDb, verbose = false } = processedOptions;
+  
+  if (!fs.existsSync(resumeDb)) {
+    console.error(`Error: Database file does not exist: ${resumeDb}`);
+    process.exit(1);
+  }
+
+  if (verbose) {
+    console.log(`🔄 Resuming from database: ${resumeDb}`);
+  }
+
+  // Initialize database manager
+  const dbManager = createDatabaseManager(resumeDb);
+  await dbManager.initialize();
+
+  try {
+    // 1. Load original parameters from meta table
+    const originalParams = await loadOriginalParameters(dbManager);
+    
+    // 2. Merge with CLI overrides
+    const mergedParams = mergeParameters(originalParams, processedOptions);
+    
+    if (verbose) {
+      console.log('📋 Loaded original parameters from database');
+      console.log('🔧 Applied CLI overrides');
+    }
+    
+    // 3. Set all rows to active=false
+    await setAllRowsInactive(dbManager, verbose);
+    
+    // 4. Scan directory for files with matching extensions
+    await scanAndSetActiveFiles(dbManager, mergedParams, verbose);
+    
+    // 5. Show final statistics using the new two-step approach
+    if (verbose) {
+      const stats = await dbManager.getOperations().getResumeStatistics();
+      console.log(`📊 Database statistics after resume:`);
+      console.log(`   - Total files: ${stats.totalFiles}`);
+      console.log(`   - Active files: ${stats.activeFiles}`);
+      console.log(`   - Unhashed active files: ${stats.unhashedActiveFiles}`);
+      console.log(`   - Size duplicate groups: ${stats.sizeDuplicateGroups}`);
+      console.log(`   - Hash duplicate groups: ${stats.hashDuplicateGroups}`);
+    }
+    
+    // 6. Add any new files if targets changed
+    if (hasNewTargets(originalParams, mergedParams)) {
+      await addNewFiles(dbManager, mergedParams, verbose);
+    }
+    
+    if (verbose) {
+      console.log('✅ Resume completed successfully');
+    }
+    
+  } catch (error) {
+    console.error(`❌ Resume failed: ${error.message}`);
+    throw error;
+  } finally {
+    await dbManager.close();
+  }
+}
+
+/**
+ * Load original parameters from database meta table
+ * @param {Object} dbManager - Database manager instance
+ * @returns {Promise<Object>} Original parameters
+ */
+async function loadOriginalParameters(dbManager) {
+  const meta = await dbManager.getOperations().getAllMeta();
+  
+  // Convert stored strings back to appropriate types
+  return {
+    targets: meta.targets ? meta.targets.split('|') : [],
+    extensions: meta.extensions ? meta.extensions.split(',') : [],
+    minSize: parseInt(meta.min_size) || 0,
+    maxSize: parseInt(meta.max_size) || 0,
+    keepStrategy: meta.keep_strategy || 'priority',
+    hashAlgorithm: meta.hash_algorithm || 'blake3',
+    recursive: meta.recursive === 'true',
+    excludeSystem: meta.exclude_system === 'true',
+    excludeHidden: meta.exclude_hidden === 'true',
+    batchSize: parseInt(meta.batch_size) || 100,
+    maxConcurrency: parseInt(meta.max_concurrency) || 5,
+    move: meta.move === 'true',
+    preservePaths: meta.preserve_paths !== 'false'
+  };
+}
+
+/**
+ * Merge original parameters with CLI overrides
+ * @param {Object} original - Original parameters from database
+ * @param {Object} cli - CLI parameters
+ * @returns {Object} Merged parameters
+ */
+function mergeParameters(original, cli) {
+  return {
+    ...original,
+    ...(cli.targets && { targets: cli.targets }),
+    ...(cli.extensions && { extensions: cli.extensions }),
+    ...(cli.minSize !== undefined && { minSize: cli.minSize }),
+    ...(cli.maxSize !== undefined && { maxSize: cli.maxSize }),
+    ...(cli.keepStrategy && { keepStrategy: cli.keepStrategy }),
+    ...(cli.hashAlgorithm && { hashAlgorithm: cli.hashAlgorithm }),
+    ...(cli.batchSize && { batchSize: cli.batchSize }),
+    ...(cli.maxConcurrency && { maxConcurrency: cli.maxConcurrency }),
+    ...(cli.move !== undefined && { move: cli.move }),
+    ...(cli.preservePaths !== undefined && { preservePaths: cli.preservePaths }),
+    ...(cli.verbose !== undefined && { verbose: cli.verbose })
+  };
+}
+
+/**
+ * Set all rows to active=false
+ * @param {Object} dbManager - Database manager instance
+ * @param {boolean} verbose - Whether to show verbose output
+ */
+async function setAllRowsInactive(dbManager, verbose) {
+  if (verbose) {
+    console.log('🔄 Setting all rows to inactive...');
+  }
+  
+  const result = await dbManager.getOperations().setAllActive(false);
+  if (!result.success) {
+    throw new Error(`Failed to set all rows inactive: ${result.error}`);
+  }
+  
+  if (verbose) {
+    console.log(`✅ Set ${result.changes} rows to inactive`);
+  }
+}
+
+/**
+ * Scan directory for files with matching extensions and set existing matches to active=true
+ * @param {Object} dbManager - Database manager instance
+ * @param {Object} params - Processing parameters
+ * @param {boolean} verbose - Whether to show verbose output
+ */
+async function scanAndSetActiveFiles(dbManager, params, verbose) {
+  if (verbose) {
+    console.log('🔍 Scanning directory for files with matching extensions...');
+  }
+
+  let activatedCount = 0;
+  let newFileCount = 0;
+
+  // OPTIMIZATION: Load all existing file paths once (fixes N+1 query problem)
+  const existingFilePaths = await dbManager.getOperations().getExistingFilePaths();
+
+  for (const target of params.targets) {
+    try {
+      const files = await scanDirectory(target, params);
+
+      for (const file of files) {
+        // Check if file already exists using Set lookup (O(1) vs O(n))
+        if (existingFilePaths.has(file.path)) {
+          // Set existing file to active=true
+          const result = await dbManager.getOperations().setActiveByPath(file.path, true);
+          if (result.success) {
+            activatedCount++;
+          }
+        } else {
+          // Add new file normally
+          const result = await dbManager.getOperations().addCopy({
+            ...file,
+            dirGroup: target,
+            priority: params.targets.indexOf(target)
+          });
+
+          if (result.success) {
+            newFileCount++;
+          }
+        }
+      }
+    } catch (error) {
+      if (verbose) {
+        console.warn(`⚠️  Warning: Failed to scan directory ${target}: ${error.message}`);
+      }
+    }
+  }
+
+  if (verbose) {
+    console.log(`✅ Activated ${activatedCount} existing files`);
+    console.log(`📄 Added ${newFileCount} new files`);
+  }
+}
+
+/**
+ * Check if there are new targets to scan
+ * @param {Object} original - Original parameters
+ * @param {Object} merged - Merged parameters
+ * @returns {boolean} Whether there are new targets
+ */
+function hasNewTargets(original, merged) {
+  if (!original.targets || !merged.targets) return false;
+  
+  // Check if any targets are different
+  const originalSet = new Set(original.targets);
+  const mergedSet = new Set(merged.targets);
+  
+  return originalSet.size !== mergedSet.size || 
+         [...originalSet].some(target => !mergedSet.has(target));
+}
+
+/**
+ * Add new files from changed targets (simplified version)
+ * @param {Object} dbManager - Database manager instance
+ * @param {Object} params - Processing parameters
+ * @param {boolean} verbose - Whether to show verbose output
+ */
+async function addNewFiles(dbManager, params, verbose) {
+  if (verbose) {
+    console.log('📁 Adding new files from changed targets...');
+  }
+  
+  // This is now handled in scanAndSetActiveFiles, but keeping for compatibility
+  if (verbose) {
+    console.log('ℹ️  New files are handled during directory scanning');
+  }
+}
+
+
+
+export { resumeController };

package/database/README.md

@@ -0,0 +1,262 @@
+# Database Module
+
+This module provides comprehensive database functionality for the file deduplication tool, including initialization, validation, CRUD operations, and utilities.
+
+## Overview
+
+The database module uses SQLite to store deduplication data, enabling resume functionality and persistent tracking of duplicate files across multiple runs.
+
+## Files
+
+- `dbConnection.js` - Database connection and initialization
+- `dbValidator.js` - Database validation and integrity checks
+- `dbOperations.js` - CRUD operations for all tables
+- `dbUtils.js` - High-level utilities and helper functions
+- `index.js` - Unified module exports
+- `example.js` - Usage examples
+
+## Quick Start
+
+```javascript
+const { createDatabaseManager } = require('./database');
+
+// Create and initialize database
+const db = createDatabaseManager('./dedupe.sqlite');
+await db.initialize();
+
+// Use database operations
+await db.getOperations().setMeta('targets', '/path/to/files');
+await db.getOperations().addCopy({
+  path: '/path/to/files/image.jpg',
+  name: 'image.jpg',
+  extension: 'jpg',
+  size: 1024000,
+  dirGroup: '/path/to/files'
+});
+
+// Close when done
+await db.close();
+```
+
+## Components
+
+### DatabaseConnection
+
+Handles SQLite database connection, table creation, and basic operations.
+
+```javascript
+const { createConnection } = require('./database');
+const db = createConnection('./dedupe.sqlite');
+await db.connect();
+```
+
+### DatabaseValidator
+
+Validates database structure, data integrity, and file system consistency.
+
+```javascript
+const { createValidator } = require('./database');
+const validator = createValidator(db);
+const validation = await validator.validateAll();
+```
+
+### DatabaseOperations
+
+Provides CRUD operations for meta, data, and copies tables.
+
+```javascript
+const { createOperations } = require('./database');
+const ops = createOperations(db);
+
+// Meta operations
+await ops.setMeta('key', 'value');
+const value = await ops.getMeta('key');
+
+// Data operations
+const dataResult = await ops.createData('hash123', 1024);
+const data = await ops.getDataByHash('hash123');
+
+// Copies operations
+await ops.addCopy(fileInfo);
+const copies = await ops.getUnhashedCopies();
+```
+
+### DatabaseUtils
+
+High-level utilities for database management, export/import, and cleanup.
+
+```javascript
+const { createUtils } = require('./database');
+const utils = createUtils('./dedupe.sqlite');
+await utils.initialize();
+
+// Get database info
+const info = await utils.getDatabaseInfo();
+
+// Export to CSV
+await utils.exportToCSV('./export.csv');
+
+// Cleanup database
+await utils.cleanupDatabase();
+```
+
+## Database Schema
+
+### Meta Table
+Stores configuration parameters from deduplication runs.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| key | TEXT PRIMARY KEY | Configuration parameter name |
+| value | TEXT | Configuration parameter value |
+
+### Copies Table
+Stores file paths and their metadata for deduplication tracking.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| id | INTEGER PRIMARY KEY AUTOINCREMENT | Unique incrementing identifier |
+| dir_group | TEXT | Directory group identifier (source directory from --targets) |
+| file_path | TEXT NOT NULL | Full path to the file |
+| file_name | TEXT NOT NULL | Just the filename |
+| file_extension | TEXT | File extension (e.g., 'jpg', 'png', 'pdf') |
+| file_size | INTEGER NOT NULL | File size in bytes |
+| file_hash | TEXT | Content hash (BLAKE3, SHA-256, MD5, etc.) |
+| active | BOOLEAN | Whether the file is active in processing |
+| priority | INTEGER | Processing priority (higher numbers = higher priority) |
+
+## Usage Examples
+
+### Initialize New Database
+
+```javascript
+const { createDatabaseManager } = require('./database');
+
+const db = createDatabaseManager('./new-db.sqlite');
+await db.initialize();
+
+// Set configuration
+await db.getOperations().setMeta('targets', '/path/to/files');
+await db.getOperations().setMeta('extensions', 'jpg,png,gif');
+await db.getOperations().setMeta('hash_algorithm', 'blake3');
+
+await db.close();
+```
+
+### Resume From Existing Database
+
+```javascript
+const { createDatabaseManager } = require('./database');
+
+const db = createDatabaseManager('./existing-db.sqlite');
+await db.initialize();
+
+// Validate database
+const validation = await db.getValidator().validateAll();
+if (!validation.valid) {
+  console.error('Database validation failed:', validation.errors);
+  process.exit(1);
+}
+
+// Get existing configuration
+const config = await db.getOperations().getAllMeta();
+console.log('Configuration:', config);
+
+// Continue processing...
+await db.close();
+```
+
+### Process Files
+
+```javascript
+const { createDatabaseManager } = require('./database');
+
+const db = createDatabaseManager('./dedupe.sqlite');
+await db.initialize();
+
+// Add files to database
+const files = [
+  { path: '/path/file1.jpg', name: 'file1.jpg', extension: 'jpg', size: 1024, dirGroup: '/path' },
+  { path: '/path/file2.jpg', name: 'file2.jpg', extension: 'jpg', size: 1024, dirGroup: '/path' }
+];
+
+for (const file of files) {
+  await db.getOperations().addCopy(file);
+}
+
+// Calculate hashes and update files
+const unhashedFiles = await db.getOperations().getUnhashedCopies();
+for (const file of unhashedFiles) {
+  const hash = calculateHash(file.file_path); // Your hash function
+  await db.getOperations().updateFileHash(file.id, hash);
+}
+
+// Find duplicates
+const duplicates = await db.getOperations().getDuplicateGroups();
+console.log('Found', duplicates.length, 'duplicate groups');
+
+await db.close();
+```
+
+### Export and Cleanup
+
+```javascript
+const { createDatabaseManager } = require('./database');
+
+const db = createDatabaseManager('./dedupe.sqlite');
+await db.initialize();
+
+// Export to CSV
+await db.getUtils().exportToCSV('./duplicates.csv');
+
+// Get statistics
+const stats = await db.getOperations().getStatistics();
+console.log('Statistics:', stats);
+
+// Cleanup orphaned data
+await db.getUtils().cleanupDatabase();
+
+await db.close();
+```
+
+## Error Handling
+
+All database operations return results with success/error information:
+
+```javascript
+const result = await db.getOperations().setMeta('key', 'value');
+if (!result.success) {
+  console.error('Error:', result.error);
+}
+```
+
+## Transactions
+
+Use transactions for atomic operations:
+
+```javascript
+await db.getConnection().beginTransaction();
+try {
+  // Multiple operations
+  await db.getOperations().setMeta('key1', 'value1');
+  await db.getOperations().setMeta('key2', 'value2');
+  await db.getConnection().commit();
+} catch (error) {
+  await db.getConnection().rollback();
+  throw error;
+}
+```
+
+## Performance Considerations
+
+- Use indexes for better query performance
+- Batch operations when possible
+- Use transactions for multiple related operations
+- Clean up orphaned data regularly
+- Consider database size limits for very large file sets
+
+## Dependencies
+
+- `sqlite3` - SQLite database driver
+- `fs` - File system operations
+- `path` - Path utilities