@tb.p/dd 1.0.0

package/database/dbConnection.js

@@ -0,0 +1,314 @@
+import sqlite3 from 'sqlite3';
+import path from 'path';
+import fs from 'fs';
+
+/**
+ * Database connection manager for the file deduplication tool
+ * Handles SQLite database initialization, connection, and cleanup
+ */
+class DatabaseConnection {
+  constructor(dbPath) {
+    this.dbPath = dbPath;
+    this.db = null;
+    this.isConnected = false;
+  }
+
+  /**
+   * Initialize database connection and create tables if needed
+   * @returns {Promise<void>}
+   */
+  async connect() {
+    return new Promise((resolve, reject) => {
+      try {
+        // Ensure directory exists
+        const dir = path.dirname(this.dbPath);
+        if (!fs.existsSync(dir)) {
+          fs.mkdirSync(dir, { recursive: true });
+        }
+
+        this.db = new sqlite3.Database(this.dbPath, (err) => {
+          if (err) {
+            reject(new Error(`Failed to connect to database: ${err.message}`));
+            return;
+          }
+          
+          this.isConnected = true;
+          resolve();
+        });
+
+        // Enable foreign key constraints
+        this.db.run('PRAGMA foreign_keys = ON');
+        
+        // Create tables if they don't exist
+        this.createTables()
+          .then(() => resolve())
+          .catch(reject);
+
+      } catch (error) {
+        reject(error);
+      }
+    });
+  }
+
+  /**
+   * Create database tables if they don't exist
+   * @returns {Promise<void>}
+   */
+  async createTables() {
+    return new Promise((resolve, reject) => {
+      const createTablesSQL = `
+        -- Meta table for storing configuration parameters
+        CREATE TABLE IF NOT EXISTS meta (
+          key TEXT PRIMARY KEY,
+          value TEXT NOT NULL
+        );
+
+        -- Copies table for storing file paths and their metadata
+        CREATE TABLE IF NOT EXISTS copies (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          dir_group TEXT,
+          file_path TEXT NOT NULL,
+          file_name TEXT NOT NULL,
+          file_extension TEXT,
+          file_size INTEGER NOT NULL,
+          file_hash TEXT,
+          active BOOLEAN DEFAULT 1,
+          priority INTEGER DEFAULT 0
+        );
+      `;
+
+      this.db.exec(createTablesSQL, (err) => {
+        if (err) {
+          reject(new Error(`Failed to create tables: ${err.message}`));
+          return;
+        }
+
+        // Create indexes for better performance
+        this.createIndexes()
+          .then(() => resolve())
+          .catch(reject);
+      });
+    });
+  }
+
+  /**
+   * Create database indexes for better performance
+   * @returns {Promise<void>}
+   */
+  async createIndexes() {
+    return new Promise((resolve, reject) => {
+      const createIndexesSQL = `
+        CREATE INDEX IF NOT EXISTS idx_copies_file_hash ON copies(file_hash);
+        CREATE INDEX IF NOT EXISTS idx_copies_file_path ON copies(file_path);
+        CREATE INDEX IF NOT EXISTS idx_copies_file_name ON copies(file_name);
+        CREATE INDEX IF NOT EXISTS idx_copies_dir_group ON copies(dir_group);
+        CREATE INDEX IF NOT EXISTS idx_copies_extension ON copies(file_extension);
+      `;
+
+      this.db.exec(createIndexesSQL, (err) => {
+        if (err) {
+          reject(new Error(`Failed to create indexes: ${err.message}`));
+          return;
+        }
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Get the database instance
+   * @returns {sqlite3.Database|null}
+   */
+  getDatabase() {
+    if (!this.isConnected) {
+      throw new Error('Database not connected. Call connect() first.');
+    }
+    return this.db;
+  }
+
+  /**
+   * Execute a query with parameters
+   * @param {string} sql - SQL query
+   * @param {Array} params - Query parameters
+   * @returns {Promise<Array>} Query results
+   */
+  async query(sql, params = []) {
+    return new Promise((resolve, reject) => {
+      this.db.all(sql, params, (err, rows) => {
+        if (err) {
+          reject(new Error(`Query failed: ${err.message}`));
+          return;
+        }
+        resolve(rows);
+      });
+    });
+  }
+
+  /**
+   * Execute a single query with parameters
+   * @param {string} sql - SQL query
+   * @param {Array} params - Query parameters
+   * @returns {Promise<Object>} Single query result
+   */
+  async queryOne(sql, params = []) {
+    return new Promise((resolve, reject) => {
+      this.db.get(sql, params, (err, row) => {
+        if (err) {
+          reject(new Error(`Query failed: ${err.message}`));
+          return;
+        }
+        resolve(row);
+      });
+    });
+  }
+
+  /**
+   * Execute a statement (INSERT, UPDATE, DELETE)
+   * @param {string} sql - SQL statement
+   * @param {Array} params - Statement parameters
+   * @returns {Promise<Object>} Statement result with lastID and changes
+   */
+  async run(sql, params = []) {
+    return new Promise((resolve, reject) => {
+      this.db.run(sql, params, function(err) {
+        if (err) {
+          reject(new Error(`Statement failed: ${err.message}`));
+          return;
+        }
+        resolve({
+          lastID: this.lastID,
+          changes: this.changes
+        });
+      });
+    });
+  }
+
+  /**
+   * Begin a transaction
+   * @returns {Promise<void>}
+   */
+  async beginTransaction() {
+    return new Promise((resolve, reject) => {
+      this.db.run('BEGIN TRANSACTION', (err) => {
+        if (err) {
+          reject(new Error(`Failed to begin transaction: ${err.message}`));
+          return;
+        }
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Commit a transaction
+   * @returns {Promise<void>}
+   */
+  async commit() {
+    return new Promise((resolve, reject) => {
+      this.db.run('COMMIT', (err) => {
+        if (err) {
+          reject(new Error(`Failed to commit transaction: ${err.message}`));
+          return;
+        }
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Rollback a transaction
+   * @returns {Promise<void>}
+   */
+  async rollback() {
+    return new Promise((resolve, reject) => {
+      this.db.run('ROLLBACK', (err) => {
+        if (err) {
+          reject(new Error(`Failed to rollback transaction: ${err.message}`));
+          return;
+        }
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Close database connection
+   * @returns {Promise<void>}
+   */
+  async close() {
+    return new Promise((resolve, reject) => {
+      if (!this.db) {
+        resolve();
+        return;
+      }
+
+      this.db.close((err) => {
+        if (err) {
+          reject(new Error(`Failed to close database: ${err.message}`));
+          return;
+        }
+        
+        this.isConnected = false;
+        this.db = null;
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Check if database file exists
+   * @returns {boolean}
+   */
+  exists() {
+    return fs.existsSync(this.dbPath);
+  }
+
+  /**
+   * Get database file size
+   * @returns {number} File size in bytes
+   */
+  getFileSize() {
+    if (!this.exists()) {
+      return 0;
+    }
+    const stats = fs.statSync(this.dbPath);
+    return stats.size;
+  }
+
+  /**
+   * Get database information
+   * @returns {Promise<Object>} Database info
+   */
+  async getInfo() {
+    if (!this.isConnected) {
+      throw new Error('Database not connected');
+    }
+
+    const [metaCount, copiesCount] = await Promise.all([
+      this.queryOne('SELECT COUNT(*) as count FROM meta'),
+      this.queryOne('SELECT COUNT(*) as count FROM copies')
+    ]);
+
+    return {
+      path: this.dbPath,
+      fileSize: this.getFileSize(),
+      exists: this.exists(),
+      connected: this.isConnected,
+      tables: {
+        meta: metaCount.count,
+        copies: copiesCount.count
+      }
+    };
+  }
+}
+
+/**
+ * Create a new database connection
+ * @param {string} dbPath - Path to database file
+ * @returns {DatabaseConnection} Database connection instance
+ */
+function createConnection(dbPath) {
+  return new DatabaseConnection(dbPath);
+}
+
+export { DatabaseConnection, createConnection };

package/database/dbOperations.js

@@ -0,0 +1,332 @@
+import path from 'path';
+
+class DatabaseOperations {
+  constructor(dbConnection) {
+    this.db = dbConnection;
+  }
+
+  async setMeta(key, value) {
+    try {
+      const result = await this.db.run(
+        'INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)',
+        [key, value]
+      );
+      return { success: true, changes: result.changes };
+    } catch (error) {
+      return { success: false, error: error.message };
+    }
+  }
+
+
+  async getAllMeta() {
+    try {
+      const results = await this.db.query('SELECT key, value FROM meta');
+      const meta = {};
+      results.forEach(row => {
+        meta[row.key] = row.value;
+      });
+      return meta;
+    } catch (error) {
+      throw new Error(`Failed to get all meta values: ${error.message}`);
+    }
+  }
+
+
+
+
+
+
+
+  async addCopy(fileInfo) {
+    try {
+      const { 
+        path, 
+        name, 
+        extension, 
+        size, 
+        hash = null, 
+        active = true, 
+        priority = 0, 
+        dirGroup = null 
+      } = fileInfo;
+      
+      const result = await this.db.run(
+        'INSERT INTO copies (dir_group, file_path, file_name, file_extension, file_size, file_hash, active, priority) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+        [dirGroup, path, name, extension, size, hash, active, priority]
+      );
+      return { success: true, id: result.lastID, changes: result.changes };
+    } catch (error) {
+      return { success: false, error: error.message };
+    }
+  }
+
+
+
+
+  async getUnhashedCopies(options = {}) {
+    try {
+      const { limit, offset } = options;
+      let sql = 'SELECT * FROM copies WHERE file_hash IS NULL ORDER BY file_path';
+      
+      if (limit) {
+        sql += ` LIMIT ${limit}`;
+        if (offset) {
+          sql += ` OFFSET ${offset}`;
+        }
+      }
+
+      const results = await this.db.query(sql);
+      return results;
+    } catch (error) {
+      throw new Error(`Failed to get unhashed copies: ${error.message}`);
+    }
+  }
+
+  async updateFileHash(copyId, hash) {
+    try {
+      const result = await this.db.run(
+        'UPDATE copies SET file_hash = ? WHERE id = ?',
+        [hash, copyId]
+      );
+      return { success: true, changes: result.changes };
+    } catch (error) {
+      return { success: false, error: error.message };
+    }
+  }
+
+
+
+
+  async getDuplicateGroups(options = {}) {
+    try {
+      const sql = `
+        SELECT
+          file_hash,
+          file_size,
+          COUNT(id) as duplicate_count,
+          GROUP_CONCAT(file_path, ';') as file_paths
+        FROM copies
+        WHERE file_hash IS NOT NULL AND active = 1
+        GROUP BY file_hash
+        HAVING COUNT(id) > 1
+        ORDER BY COUNT(id) DESC, file_size DESC
+      `;
+
+      const results = await this.db.query(sql);
+      return results;
+    } catch (error) {
+      throw new Error(`Failed to get duplicate groups: ${error.message}`);
+    }
+  }
+
+
+  async getStatistics() {
+    try {
+      const [metaCount, copiesCount, unhashedCount, duplicateGroups, uniqueFiles] = await Promise.all([
+        this.db.queryOne('SELECT COUNT(*) as count FROM meta'),
+        this.db.queryOne('SELECT COUNT(*) as count FROM copies'),
+        this.db.queryOne('SELECT COUNT(*) as count FROM copies WHERE file_hash IS NULL'),
+        this.db.queryOne(`
+          SELECT COUNT(*) as count FROM (
+            SELECT file_hash FROM copies 
+            WHERE file_hash IS NOT NULL 
+            GROUP BY file_hash 
+            HAVING COUNT(*) > 1
+          )
+        `),
+        this.db.queryOne('SELECT COUNT(DISTINCT file_size) as count FROM copies WHERE file_hash IS NULL')
+      ]);
+
+      const totalSize = await this.db.queryOne(`
+        SELECT SUM(file_size) as total_size FROM copies
+      `);
+
+      const duplicateSize = await this.db.queryOne(`
+        SELECT SUM(file_size * (count - 1)) as duplicate_size FROM (
+          SELECT file_hash, file_size, COUNT(*) as count FROM copies 
+          WHERE file_hash IS NOT NULL 
+          GROUP BY file_hash 
+          HAVING COUNT(*) > 1
+        )
+      `);
+
+      return {
+        meta: metaCount.count,
+        copies: copiesCount.count,
+        unhashed: unhashedCount.count,
+        uniqueFiles: uniqueFiles.count,
+        duplicateGroups: duplicateGroups.count,
+        totalSize: totalSize.total_size || 0,
+        duplicateSize: duplicateSize.duplicate_size || 0,
+        spaceSavings: duplicateSize.duplicate_size || 0
+      };
+    } catch (error) {
+      throw new Error(`Failed to get statistics: ${error.message}`);
+    }
+  }
+
+  async getAllCopies() {
+    try {
+      const results = await this.db.query('SELECT * FROM copies ORDER BY file_path');
+      return results;
+    } catch (error) {
+      throw new Error(`Failed to get all copies: ${error.message}`);
+    }
+  }
+
+  async getExistingFilePaths() {
+    try {
+      const results = await this.db.query('SELECT file_path FROM copies');
+      return new Set(results.map(row => row.file_path));
+    } catch (error) {
+      throw new Error(`Failed to get existing file paths: ${error.message}`);
+    }
+  }
+
+
+  async setAllActive(active) {
+    try {
+      const result = await this.db.run(
+        'UPDATE copies SET active = ?',
+        [active]
+      );
+      return { success: true, changes: result.changes };
+    } catch (error) {
+      return { success: false, error: error.message };
+    }
+  }
+
+  async setActiveByPath(filePath, active) {
+    try {
+      const result = await this.db.run(
+        'UPDATE copies SET active = ? WHERE file_path = ?',
+        [active, filePath]
+      );
+      return { success: true, changes: result.changes };
+    } catch (error) {
+      return { success: false, error: error.message };
+    }
+  }
+
+
+  async getActiveFiles() {
+    try {
+      const results = await this.db.query('SELECT * FROM copies WHERE active = 1 ORDER BY file_path');
+      return results;
+    } catch (error) {
+      throw new Error(`Failed to get active files: ${error.message}`);
+    }
+  }
+
+  /**
+   * Step 1: Get files with duplicates based on file size, excluding inactive and already-hashed files
+   * @returns {Promise<Array>} Array of file size groups with potential duplicates
+   */
+  async getFilesWithSizeDuplicates() {
+    try {
+      const sql = `
+        SELECT 
+          file_size,
+          COUNT(id) as count,
+          GROUP_CONCAT(id, ',') as file_ids,
+          GROUP_CONCAT(file_path, '|') as file_paths
+        FROM copies 
+        WHERE active = 1 AND file_hash IS NULL
+        GROUP BY file_size
+        HAVING COUNT(id) > 1
+        ORDER BY file_size DESC
+      `;
+      
+      const results = await this.db.query(sql);
+      return results.map(row => ({
+        size: row.file_size,
+        count: parseInt(row.count),
+        fileIds: row.file_ids ? row.file_ids.split(',').map(id => parseInt(id)) : [],
+        filePaths: row.file_paths ? row.file_paths.split('|') : []
+      }));
+    } catch (error) {
+      throw new Error(`Failed to get files with size duplicates: ${error.message}`);
+    }
+  }
+
+  /**
+   * Step 2: Get active files grouped by hash to find actual duplicates
+   * @returns {Promise<Array>} Array of hash groups with duplicates
+   */
+  async getActiveFilesGroupedByHash() {
+    try {
+      const sql = `
+        SELECT 
+          file_hash,
+          file_size,
+          COUNT(id) as count,
+          GROUP_CONCAT(id, ',') as file_ids,
+          GROUP_CONCAT(file_path, '|') as file_paths
+        FROM copies 
+        WHERE active = 1 AND file_hash IS NOT NULL
+        GROUP BY file_hash
+        HAVING COUNT(id) > 1
+        ORDER BY file_size DESC, COUNT(id) DESC
+      `;
+      
+      const results = await this.db.query(sql);
+      return results.map(row => ({
+        hash: row.file_hash,
+        size: row.file_size,
+        count: parseInt(row.count),
+        fileIds: row.file_ids ? row.file_ids.split(',').map(id => parseInt(id)) : [],
+        filePaths: row.file_paths ? row.file_paths.split('|') : []
+      }));
+    } catch (error) {
+      throw new Error(`Failed to get active files grouped by hash: ${error.message}`);
+    }
+  }
+
+  /**
+   * Get statistics for resume process
+   * @returns {Promise<Object>} Resume statistics
+   */
+  async getResumeStatistics() {
+    try {
+      const [totalFiles, activeFiles, unhashedActiveFiles, sizeDuplicates, hashDuplicates] = await Promise.all([
+        this.db.queryOne('SELECT COUNT(*) as count FROM copies'),
+        this.db.queryOne('SELECT COUNT(*) as count FROM copies WHERE active = 1'),
+        this.db.queryOne('SELECT COUNT(*) as count FROM copies WHERE active = 1 AND file_hash IS NULL'),
+        this.db.queryOne(`
+          SELECT COUNT(*) as count FROM (
+            SELECT file_size FROM copies 
+            WHERE active = 1 AND file_hash IS NULL 
+            GROUP BY file_size 
+            HAVING COUNT(*) > 1
+          )
+        `),
+        this.db.queryOne(`
+          SELECT COUNT(*) as count FROM (
+            SELECT file_hash FROM copies 
+            WHERE active = 1 AND file_hash IS NOT NULL 
+            GROUP BY file_hash 
+            HAVING COUNT(*) > 1
+          )
+        `)
+      ]);
+
+      return {
+        totalFiles: totalFiles.count,
+        activeFiles: activeFiles.count,
+        unhashedActiveFiles: unhashedActiveFiles.count,
+        sizeDuplicateGroups: sizeDuplicates.count,
+        hashDuplicateGroups: hashDuplicates.count
+      };
+    } catch (error) {
+      throw new Error(`Failed to get resume statistics: ${error.message}`);
+    }
+  }
+
+
+}
+
+function createOperations(dbConnection) {
+  return new DatabaseOperations(dbConnection);
+}
+
+export { DatabaseOperations, createOperations };