@tb.p/dd 1.1.3 → 1.2.0

package/controllers/newController.js

@@ -48,7 +48,11 @@ async function performFileOperations(options, dbManager) {
   }
   
   // Convert map values back to array, maintaining priority order
-  allFiles.push(...Array.from(fileMap.values()).sort((a, b) => a.priority - b.priority));
+  // Use a loop instead of apply/spread to avoid stack overflow with very large arrays
+  const sortedFiles = Array.from(fileMap.values()).sort((a, b) => a.priority - b.priority);
+  for (const file of sortedFiles) {
+    allFiles.push(file);
+  }
   
   // Store files in database
   let successCount = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tb.p/dd",
-  "version": "1.1.3",
+  "version": "1.2.0",
   "description": "A comprehensive command-line tool for finding and removing duplicate files using content-based hashing",
   "type": "module",
   "main": "index.js",

package/utils/candidateDetection.js

@@ -18,6 +18,15 @@ class CandidateDetection {
       onProgress: null,
       ...options
     };
+    
+    // Enforce minimum batch size to prevent stack overflow
+    // Batch sizes below 10 can cause excessive nested promises
+    if (this.options.batchSize < 10) {
+      if (this.options.verbose) {
+        console.log(`⚠️  Batch size ${this.options.batchSize} is too small, using minimum of 10 to prevent stack overflow`);
+      }
+      this.options.batchSize = 10;
+    }
     this.stats = {
       totalFiles: 0,
       uniqueFiles: 0,
@@ -514,9 +523,13 @@ export async function runCandidateDetection(options) {
       maxConcurrency: parseInt(options.maxConcurrency) || 5,
       verbose: options.verbose || false,
       onProgress: options.verbose ? (progress) => {
-        if (progress.phase === 'hashing') {
-          const percentage = (progress.percentage || 0).toFixed(1);
-          console.log(`📊 Hashing progress: ${progress.processed}/${progress.total} files (${percentage}%) - ${progress.hashedFiles} hashed, ${progress.errors} errors`);
+        if (progress && progress.phase === 'hashing') {
+          const processed = progress.processed || 0;
+          const total = progress.total || 0;
+          const hashedFiles = progress.hashedFiles || 0;
+          const errors = progress.errors || 0;
+          const percentage = total > 0 ? ((processed / total) * 100).toFixed(1) : '0.0';
+          console.log(`📊 Hashing progress: ${processed}/${total} files (${percentage}%) - ${hashedFiles} hashed, ${errors} errors`);
         }
       } : null
     });

package/utils/fileScanner.js

@@ -19,7 +19,13 @@ async function scanDirectory(dirPath, options = {}) {
 
   try {
     const filePaths = await new Promise((resolve, reject) => {
+      // Add timeout to prevent hanging on very large directories
+      const timeout = setTimeout(() => {
+        reject(new Error('Directory scan timeout - directory may be too large or contain circular references'));
+      }, 300000); // 5 minute timeout
+      
       recursive(dirPath, (err, files) => {
+        clearTimeout(timeout);
         if (err) reject(err);
         else resolve(files);
       });
@@ -36,18 +42,35 @@ async function scanDirectory(dirPath, options = {}) {
 }
 
 async function getBatchFileInfo(filePaths, options = {}) {
-  // OPTIMIZATION: Batch all stat() operations for parallel I/O
-  const statPromises = filePaths.map(async (filePath) => {
-    try {
-      const normalizedPath = normalizePath(filePath);
-      const stats = await fs.stat(normalizedPath);
-      return { filePath: normalizedPath, stats, error: null };
-    } catch (error) {
-      return { filePath, stats: null, error };
-    }
-  });
+  // OPTIMIZATION: Batch stat() operations with controlled concurrency to prevent stack overflow
+  const BATCH_SIZE = 100; // Process files in chunks to prevent excessive Promise.all
+  
+  // Safety check: prevent processing extremely large file lists that could cause stack overflow
+  if (filePaths.length > 100000) {
+    console.warn(`Warning: Very large file list detected (${filePaths.length} files). This may cause performance issues.`);
+    console.warn('Consider processing smaller directories or using file filters to reduce the scope.');
+  }
+  
+  const statResults = [];
+
+  for (let i = 0; i < filePaths.length; i += BATCH_SIZE) {
+    const batch = filePaths.slice(i, i + BATCH_SIZE);
+    const statPromises = batch.map(async (filePath) => {
+      try {
+        const normalizedPath = normalizePath(filePath);
+        const stats = await fs.stat(normalizedPath);
+        return { filePath: normalizedPath, stats, error: null };
+      } catch (error) {
+        return { filePath, stats: null, error };
+      }
+    });
 
-  const statResults = await Promise.all(statPromises);
+    const batchResults = await Promise.all(statPromises);
+    // Use a loop instead of apply/spread to avoid stack overflow with very large arrays
+    for (const result of batchResults) {
+      statResults.push(result);
+    }
+  }
   const files = [];
 
   for (const result of statResults) {
@@ -113,7 +136,10 @@ async function scanDirectories(options = {}) {
   for (const dirPath of targets) {
     try {
       const files = await scanDirectory(dirPath, options);
-      allFiles.push(...files);
+      // Use a loop instead of apply/spread to avoid stack overflow with very large arrays
+      for (const file of files) {
+        allFiles.push(file);
+      }
     } catch (error) {
       console.warn(`Warning: Failed to scan directory ${dirPath}: ${error.message}`);
     }

package/validators/optionValidator.js

@@ -94,6 +94,30 @@ function validateOptions(options) {
       process.exit(1);
     }
   }
+
+  // Validate batch size to prevent stack overflow
+  if (options.batchSize) {
+    const batchSize = parseInt(options.batchSize);
+    if (isNaN(batchSize) || batchSize < 1) {
+      console.error('Error: Batch size must be a positive number');
+      process.exit(1);
+    } else if (batchSize < 10) {
+      console.warn(`Warning: Batch size ${batchSize} is very small and may cause performance issues or stack overflow. Recommended minimum: 10`);
+    } else if (batchSize > 1000) {
+      console.warn(`Warning: Batch size ${batchSize} is very large and may cause memory issues. Recommended maximum: 1000`);
+    }
+  }
+
+  // Validate max concurrency
+  if (options.maxConcurrency) {
+    const maxConcurrency = parseInt(options.maxConcurrency);
+    if (isNaN(maxConcurrency) || maxConcurrency < 1) {
+      console.error('Error: Max concurrency must be a positive number');
+      process.exit(1);
+    } else if (maxConcurrency > 50) {
+      console.warn(`Warning: Max concurrency ${maxConcurrency} is very high and may cause system instability. Recommended maximum: 50`);
+    }
+  }
 }
 
 export {