@tb.p/dd 1.0.0

package/database/dbUtils.js

@@ -0,0 +1,125 @@
+import fs from 'fs';
+import path from 'path';
+import { createConnection } from './dbConnection.js';
+import { createValidator } from './dbValidator.js';
+import { createOperations } from './dbOperations.js';
+
+class DatabaseUtils {
+  constructor(dbPath) {
+    this.dbPath = dbPath;
+    this.connection = null;
+    this.validator = null;
+    this.operations = null;
+  }
+
+  async initialize() {
+    this.connection = createConnection(this.dbPath);
+    await this.connection.connect();
+    this.validator = createValidator(this.connection);
+    this.operations = createOperations(this.connection);
+  }
+
+  async close() {
+    if (this.connection) {
+      await this.connection.close();
+      this.connection = null;
+      this.validator = null;
+      this.operations = null;
+    }
+  }
+
+
+
+  /**
+   * Get database operations instance
+   * @returns {DatabaseOperations}
+   */
+  getOperations() {
+    if (!this.operations) {
+      throw new Error('Database not initialized. Call initialize() first.');
+    }
+    return this.operations;
+  }
+
+  /**
+   * Initialize a new database with configuration from processed options
+   * @param {Object} processedOptions - Processed CLI options
+   * @returns {Promise<Object>} Initialization result
+   */
+  async initializeNewDatabase(processedOptions) {
+    try {
+      await this.initialize();
+
+      // Extract configuration from processed options
+      const targets = processedOptions.targets;
+      const config = {
+        targets: targets.join('|'),
+        extensions: processedOptions.extensions ? processedOptions.extensions.join(',') : '',
+        min_size: processedOptions.minSize || '0',
+        max_size: processedOptions.maxSize || '0',
+        keep_strategy: processedOptions.keepStrategy || 'priority',
+        hash_algorithm: processedOptions.hashAlgorithm || 'blake3',
+        recursive: processedOptions.recursive !== false ? 'true' : 'false',
+        exclude_system: processedOptions.excludeSystem !== false ? 'true' : 'false',
+        exclude_hidden: processedOptions.excludeHidden !== false ? 'true' : 'false',
+        batch_size: processedOptions.batchSize || '100',
+        max_concurrency: processedOptions.maxConcurrency || '5',
+        move: processedOptions.move ? 'true' : 'false',
+        preserve_paths: processedOptions.preservePaths !== false ? 'true' : 'false',
+        created_at: new Date().toISOString(),
+        version: '1.0.0'
+      };
+
+      // Store configuration in meta table
+      for (const [key, value] of Object.entries(config)) {
+        await this.operations.setMeta(key, value);
+      }
+
+      // Validate database structure
+      const validation = await this.validator.validateAll();
+      if (!validation.valid) {
+        return {
+          success: false,
+          error: 'Database validation failed',
+          validation
+        };
+      }
+
+      // Get database info
+      const dbInfo = await this.connection.getInfo();
+
+      return {
+        success: true,
+        message: 'Database initialized successfully',
+        config,
+        dbInfo,
+        validation
+      };
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message
+      };
+    }
+  }
+
+
+
+
+
+
+
+
+
+}
+
+/**
+ * Create a database utils instance
+ * @param {string} dbPath - Path to database file
+ * @returns {DatabaseUtils} Database utils instance
+ */
+function createUtils(dbPath) {
+  return new DatabaseUtils(dbPath);
+}
+
+export { DatabaseUtils, createUtils };

package/database/dbValidator.js

@@ -0,0 +1,325 @@
+import fs from 'fs';
+
+class DatabaseValidator {
+  constructor(dbConnection) {
+    this.db = dbConnection;
+  }
+
+  validateDatabaseFile() {
+    const result = {
+      valid: true,
+      errors: [],
+      warnings: []
+    };
+
+    if (!this.db.exists()) {
+      result.valid = false;
+      result.errors.push('Database file does not exist');
+      return result;
+    }
+
+    try {
+      fs.accessSync(this.db.dbPath, fs.constants.R_OK);
+    } catch (error) {
+      result.valid = false;
+      result.errors.push('Database file is not readable');
+      return result;
+    }
+
+    try {
+      fs.accessSync(this.db.dbPath, fs.constants.W_OK);
+    } catch (error) {
+      result.warnings.push('Database file is not writable (read-only mode)');
+    }
+
+    const fileSize = this.db.getFileSize();
+    if (fileSize === 0) {
+      result.warnings.push('Database file is empty');
+    }
+
+    return result;
+  }
+
+  async validateSchema() {
+    const result = {
+      valid: true,
+      errors: [],
+      warnings: []
+    };
+
+    try {
+      const tables = await this.db.query(`
+        SELECT name FROM sqlite_master 
+        WHERE type='table' AND name IN ('meta', 'copies')
+        ORDER BY name
+      `);
+
+      const tableNames = tables.map(t => t.name);
+      const expectedTables = ['meta', 'copies'];
+
+      for (const expectedTable of expectedTables) {
+        if (!tableNames.includes(expectedTable)) {
+          result.valid = false;
+          result.errors.push(`Missing table: ${expectedTable}`);
+        }
+      }
+
+      for (const table of tables) {
+        const tableValidation = await this.validateTableStructure(table.name);
+        if (!tableValidation.valid) {
+          result.valid = false;
+          result.errors.push(...tableValidation.errors);
+        }
+        result.warnings.push(...tableValidation.warnings);
+      }
+
+      const indexes = await this.db.query(`
+        SELECT name FROM sqlite_master 
+        WHERE type='index' AND name LIKE 'idx_%'
+      `);
+
+      const expectedIndexes = [
+        'idx_copies_file_hash',
+        'idx_copies_file_path',
+        'idx_copies_file_name',
+        'idx_copies_dir_group',
+        'idx_copies_extension'
+      ];
+
+      for (const expectedIndex of expectedIndexes) {
+        if (!indexes.some(idx => idx.name === expectedIndex)) {
+          result.warnings.push(`Missing index: ${expectedIndex}`);
+        }
+      }
+
+    } catch (error) {
+      result.valid = false;
+      result.errors.push(`Schema validation failed: ${error.message}`);
+    }
+
+    return result;
+  }
+
+  async validateTableStructure(tableName) {
+    const result = {
+      valid: true,
+      errors: [],
+      warnings: []
+    };
+
+    try {
+      const columns = await this.db.query(`PRAGMA table_info(${tableName})`);
+      
+      if (columns.length === 0) {
+        result.valid = false;
+        result.errors.push(`Table ${tableName} has no columns`);
+        return result;
+      }
+
+      switch (tableName) {
+        case 'meta':
+          this.validateMetaTable(columns, result);
+          break;
+        case 'copies':
+          this.validateCopiesTable(columns, result);
+          break;
+        default:
+          result.warnings.push(`Unknown table: ${tableName}`);
+      }
+
+    } catch (error) {
+      result.valid = false;
+      result.errors.push(`Failed to validate table ${tableName}: ${error.message}`);
+    }
+
+    return result;
+  }
+
+  validateMetaTable(columns, result) {
+    const expectedColumns = [
+      { name: 'key', type: 'TEXT', pk: true },
+      { name: 'value', type: 'TEXT', pk: false }
+    ];
+
+    this.validateColumns(columns, expectedColumns, 'meta', result);
+  }
+
+
+  validateCopiesTable(columns, result) {
+    const expectedColumns = [
+      { name: 'id', type: 'INTEGER', pk: true },
+      { name: 'dir_group', type: 'TEXT', pk: false },
+      { name: 'file_path', type: 'TEXT', pk: false },
+      { name: 'file_name', type: 'TEXT', pk: false },
+      { name: 'file_extension', type: 'TEXT', pk: false },
+      { name: 'file_size', type: 'INTEGER', pk: false },
+      { name: 'file_hash', type: 'TEXT', pk: false },
+      { name: 'active', type: 'BOOLEAN', pk: false },
+      { name: 'priority', type: 'INTEGER', pk: false }
+    ];
+
+    this.validateColumns(columns, expectedColumns, 'copies', result);
+  }
+
+  validateColumns(actualColumns, expectedColumns, tableName, result) {
+    const actualColumnNames = actualColumns.map(c => c.name);
+
+    for (const expected of expectedColumns) {
+      const actual = actualColumns.find(c => c.name === expected.name);
+      
+      if (!actual) {
+        result.valid = false;
+        result.errors.push(`Table ${tableName} missing column: ${expected.name}`);
+        continue;
+      }
+
+      if (actual.type.toUpperCase() !== expected.type.toUpperCase()) {
+        result.warnings.push(`Table ${tableName} column ${expected.name} has type ${actual.type}, expected ${expected.type}`);
+      }
+
+      if (actual.pk !== expected.pk) {
+        result.warnings.push(`Table ${tableName} column ${expected.name} primary key mismatch`);
+      }
+    }
+  }
+
+  async validateDataIntegrity() {
+    const result = {
+      valid: true,
+      errors: [],
+      warnings: []
+    };
+
+    try {
+      const negativeSizes = await this.db.query(`
+        SELECT id, file_path, file_size
+        FROM copies
+        WHERE file_size < 0
+      `);
+
+      if (negativeSizes.length > 0) {
+        result.warnings.push(`Found ${negativeSizes.length} files with negative sizes`);
+      }
+
+      const emptyPaths = await this.db.query(`
+        SELECT id, file_path
+        FROM copies
+        WHERE file_path = '' OR file_path IS NULL
+      `);
+
+      if (emptyPaths.length > 0) {
+        result.valid = false;
+        result.errors.push(`Found ${emptyPaths.length} files with empty paths`);
+      }
+
+      const emptyNames = await this.db.query(`
+        SELECT id, file_name
+        FROM copies
+        WHERE file_name = '' OR file_name IS NULL
+      `);
+
+      if (emptyNames.length > 0) {
+        result.valid = false;
+        result.errors.push(`Found ${emptyNames.length} files with empty names`);
+      }
+
+    } catch (error) {
+      result.valid = false;
+      result.errors.push(`Data integrity validation failed: ${error.message}`);
+    }
+
+    return result;
+  }
+
+  async validateFileSystemConsistency() {
+    const result = {
+      valid: true,
+      errors: [],
+      warnings: []
+    };
+
+    try {
+      const allFiles = await this.db.query('SELECT id, file_path FROM copies');
+      let missingFiles = 0;
+      let accessibleFiles = 0;
+
+      for (const file of allFiles) {
+        try {
+          if (fs.existsSync(file.file_path)) {
+            accessibleFiles++;
+          } else {
+            missingFiles++;
+            result.warnings.push(`File not found: ${file.file_path}`);
+          }
+        } catch (error) {
+          result.warnings.push(`Error checking file ${file.file_path}: ${error.message}`);
+        }
+      }
+
+      if (missingFiles > 0) {
+        result.warnings.push(`Found ${missingFiles} missing files out of ${allFiles.length} total files`);
+      }
+
+      result.stats = {
+        totalFiles: allFiles.length,
+        accessibleFiles,
+        missingFiles
+      };
+
+    } catch (error) {
+      result.valid = false;
+      result.errors.push(`File system validation failed: ${error.message}`);
+    }
+
+    return result;
+  }
+
+  async validateAll() {
+    const result = {
+      valid: true,
+      errors: [],
+      warnings: [],
+      details: {}
+    };
+
+    const fileValidation = this.validateDatabaseFile();
+    result.details.file = fileValidation;
+    if (!fileValidation.valid) {
+      result.valid = false;
+    }
+    result.errors.push(...fileValidation.errors);
+    result.warnings.push(...fileValidation.warnings);
+
+    const schemaValidation = await this.validateSchema();
+    result.details.schema = schemaValidation;
+    if (!schemaValidation.valid) {
+      result.valid = false;
+    }
+    result.errors.push(...schemaValidation.errors);
+    result.warnings.push(...schemaValidation.warnings);
+
+    const integrityValidation = await this.validateDataIntegrity();
+    result.details.integrity = integrityValidation;
+    if (!integrityValidation.valid) {
+      result.valid = false;
+    }
+    result.errors.push(...integrityValidation.errors);
+    result.warnings.push(...integrityValidation.warnings);
+
+    const fsValidation = await this.validateFileSystemConsistency();
+    result.details.fileSystem = fsValidation;
+    if (!fsValidation.valid) {
+      result.valid = false;
+    }
+    result.errors.push(...fsValidation.errors);
+    result.warnings.push(...fsValidation.warnings);
+
+    return result;
+  }
+}
+
+function createValidator(dbConnection) {
+  return new DatabaseValidator(dbConnection);
+}
+
+export { DatabaseValidator, createValidator };

package/database/index.js

@@ -0,0 +1,102 @@
+/**
+ * Database module for file deduplication tool
+ * Provides unified access to all database functionality
+ */
+
+import { createConnection } from './dbConnection.js';
+import { createValidator } from './dbValidator.js';
+import { createOperations } from './dbOperations.js';
+import { createUtils } from './dbUtils.js';
+
+/**
+ * Create a complete database manager instance
+ * @param {string} dbPath - Path to database file
+ * @returns {Object} Database manager with all components
+ */
+function createDatabaseManager(dbPath) {
+  return {
+    // Core components
+    connection: null,
+    validator: null,
+    operations: null,
+    utils: null,
+
+    // Initialize all components
+    async initialize() {
+      this.connection = createConnection(dbPath);
+      await this.connection.connect();
+      this.validator = createValidator(this.connection);
+      this.operations = createOperations(this.connection);
+      this.utils = createUtils(dbPath);
+      await this.utils.initialize();
+    },
+
+    // Close all connections
+    async close() {
+      if (this.connection) {
+        await this.connection.close();
+      }
+      if (this.utils) {
+        await this.utils.close();
+      }
+      this.connection = null;
+      this.validator = null;
+      this.operations = null;
+      this.utils = null;
+    },
+
+    // Get individual components
+    getConnection() {
+      if (!this.connection) {
+        throw new Error('Database not initialized. Call initialize() first.');
+      }
+      return this.connection;
+    },
+
+    getValidator() {
+      if (!this.validator) {
+        throw new Error('Database not initialized. Call initialize() first.');
+      }
+      return this.validator;
+    },
+
+    getOperations() {
+      if (!this.operations) {
+        throw new Error('Database not initialized. Call initialize() first.');
+      }
+      return this.operations;
+    },
+
+    getUtils() {
+      if (!this.utils) {
+        throw new Error('Database not initialized. Call initialize() first.');
+      }
+      return this.utils;
+    }
+  };
+}
+
+/**
+ * Initialize a new database with processed options
+ * @param {string} dbPath - Path to database file
+ * @param {Object} processedOptions - Processed CLI options
+ * @returns {Promise<Object>} Initialization result
+ */
+async function initializeNewDatabase(dbPath, processedOptions) {
+  const utils = createUtils(dbPath);
+  try {
+    const result = await utils.initializeNewDatabase(processedOptions);
+    return result;
+  } finally {
+    await utils.close();
+  }
+}
+
+export {
+  createConnection,
+  createValidator,
+  createOperations,
+  createUtils,
+  createDatabaseManager,
+  initializeNewDatabase
+};

package/index.js

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander';
+import { getExtensionsController } from './controllers/getExtensionsController.js';
+import { newController } from './controllers/newController.js';
+import { resumeController } from './controllers/resumeController.js';
+import { runCandidateDetection } from './utils/candidateDetection.js';
+import { validateOptions } from './validators/optionValidator.js';
+import { processOptions } from './processors/optionETL.js';
+import { moveDuplicates } from './utils/duplicateMover.js';
+import { displayDuplicates } from './utils/duplicateReporter.js';
+
+const program = new Command();
+
+program
+  .name('@tb.p/dd')
+  .description('File Deduplication Tool')
+  .version('1.0.0')
+  .option('-t, --targets <string>', 'Pipe-separated paths to directories', process.cwd())
+  .option('-e, --extensions <string>', 'Pipe-separated file extensions')
+  .option('-g, --get-extensions', 'Get file extensions in target directories')
+  .option('-r, --resume <string>', 'Path to database file to resume from')
+  .option('-s, --save <string>', 'Path to database file to save to')
+  .option('-m, --move', 'Move duplicated files (only available with --save or --resume)')
+  .option('--preserve-paths', 'Preserve directory structure when moving duplicates (default: true)')
+  .option('--flat-duplicates', 'Move duplicates to flat structure (overrides --preserve-paths)')
+  .option('--hash-algorithm <string>', 'Hash algorithm to use (blake3, md5, sha1, sha256)', 'blake3')
+  .option('-b, --batch-size <number>', 'Batch size for processing candidates', '100')
+  .option('-c, --max-concurrency <number>', 'Maximum concurrent hash operations', '5')
+  .option('-v, --verbose', 'Show detailed progress information');
+
+program.parse();
+
+const rawOptions = program.opts();
+
+const processedOptions = processOptions(rawOptions);
+
+validateOptions(processedOptions);
+
+async function runController() {
+  try {
+    switch (processedOptions.mode) {
+      case 'get-extensions':
+        getExtensionsController(processedOptions);
+        break;
+      case 'save':
+
+        await newController(processedOptions);
+        await runCandidateDetection(processedOptions);
+        await displayDuplicates(processedOptions);
+        if (processedOptions.move) {
+          await moveDuplicates(processedOptions);
+        }
+        break;
+      case 'resume':
+        await resumeController(processedOptions);
+        await runCandidateDetection(processedOptions);
+        await displayDuplicates(processedOptions);
+        if (processedOptions.move) {
+          await moveDuplicates(processedOptions);
+        }
+        break;
+
+      default:
+        console.error('Error: You must specify a mode: --get-extensions, --resume <db>, or --save <db>');
+        process.exit(1);
+    }
+  } catch (error) {
+    console.error('Error:', error.message);
+    process.exit(1);
+  }
+}
+
+
+runController();

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@tb.p/dd",
+  "version": "1.0.0",
+  "description": "A comprehensive command-line tool for finding and removing duplicate files using content-based hashing",
+  "type": "module",
+  "main": "index.js",
+  "bin": {
+    "@tb.p/dd": "./index.js"
+  },
+  "scripts": {
+    "start": "node index.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [
+    "duplicate",
+    "files",
+    "deduplication",
+    "hash",
+    "cli",
+    "file-management"
+  ],
+  "author": "@tb.p",
+  "license": "MIT",
+  "dependencies": {
+    "commander": "^11.1.0",
+    "hash-wasm": "^4.12.0",
+    "sqlite3": "^5.1.6"
+  },
+  "engines": {
+    "node": ">=14.0.0"
+  }
+}