@tb.p/dd 1.1.4 → 1.2.0

package/controllers/newController.js

@@ -48,7 +48,11 @@ async function performFileOperations(options, dbManager) {
   }
   
   // Convert map values back to array, maintaining priority order
-  allFiles.push(...Array.from(fileMap.values()).sort((a, b) => a.priority - b.priority));
+  // Use a loop instead of apply/spread to avoid stack overflow with very large arrays
+  const sortedFiles = Array.from(fileMap.values()).sort((a, b) => a.priority - b.priority);
+  for (const file of sortedFiles) {
+    allFiles.push(file);
+  }
   
   // Store files in database
   let successCount = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tb.p/dd",
-  "version": "1.1.4",
+  "version": "1.2.0",
   "description": "A comprehensive command-line tool for finding and removing duplicate files using content-based hashing",
   "type": "module",
   "main": "index.js",

package/utils/candidateDetection.js

@@ -18,6 +18,15 @@ class CandidateDetection {
       onProgress: null,
       ...options
     };
+    
+    // Enforce minimum batch size to prevent stack overflow
+    // Batch sizes below 10 can cause excessive nested promises
+    if (this.options.batchSize < 10) {
+      if (this.options.verbose) {
+        console.log(`⚠️  Batch size ${this.options.batchSize} is too small, using minimum of 10 to prevent stack overflow`);
+      }
+      this.options.batchSize = 10;
+    }
     this.stats = {
       totalFiles: 0,
       uniqueFiles: 0,

package/utils/fileScanner.js

@@ -19,7 +19,13 @@ async function scanDirectory(dirPath, options = {}) {
 
   try {
     const filePaths = await new Promise((resolve, reject) => {
+      // Add timeout to prevent hanging on very large directories
+      const timeout = setTimeout(() => {
+        reject(new Error('Directory scan timeout - directory may be too large or contain circular references'));
+      }, 300000); // 5 minute timeout
+      
       recursive(dirPath, (err, files) => {
+        clearTimeout(timeout);
         if (err) reject(err);
         else resolve(files);
       });
@@ -38,6 +44,13 @@ async function scanDirectory(dirPath, options = {}) {
 async function getBatchFileInfo(filePaths, options = {}) {
   // OPTIMIZATION: Batch stat() operations with controlled concurrency to prevent stack overflow
   const BATCH_SIZE = 100; // Process files in chunks to prevent excessive Promise.all
+  
+  // Safety check: prevent processing extremely large file lists that could cause stack overflow
+  if (filePaths.length > 100000) {
+    console.warn(`Warning: Very large file list detected (${filePaths.length} files). This may cause performance issues.`);
+    console.warn('Consider processing smaller directories or using file filters to reduce the scope.');
+  }
+  
   const statResults = [];
 
   for (let i = 0; i < filePaths.length; i += BATCH_SIZE) {
@@ -53,7 +66,10 @@ async function getBatchFileInfo(filePaths, options = {}) {
     });
 
     const batchResults = await Promise.all(statPromises);
-    statResults.push(...batchResults);
+    // Use a loop instead of apply/spread to avoid stack overflow with very large arrays
+    for (const result of batchResults) {
+      statResults.push(result);
+    }
   }
   const files = [];
 
@@ -120,7 +136,10 @@ async function scanDirectories(options = {}) {
   for (const dirPath of targets) {
     try {
       const files = await scanDirectory(dirPath, options);
-      allFiles.push(...files);
+      // Use a loop instead of apply/spread to avoid stack overflow with very large arrays
+      for (const file of files) {
+        allFiles.push(file);
+      }
     } catch (error) {
       console.warn(`Warning: Failed to scan directory ${dirPath}: ${error.message}`);
     }

package/validators/optionValidator.js

@@ -94,6 +94,30 @@ function validateOptions(options) {
       process.exit(1);
     }
   }
+
+  // Validate batch size to prevent stack overflow
+  if (options.batchSize) {
+    const batchSize = parseInt(options.batchSize);
+    if (isNaN(batchSize) || batchSize < 1) {
+      console.error('Error: Batch size must be a positive number');
+      process.exit(1);
+    } else if (batchSize < 10) {
+      console.warn(`Warning: Batch size ${batchSize} is very small and may cause performance issues or stack overflow. Recommended minimum: 10`);
+    } else if (batchSize > 1000) {
+      console.warn(`Warning: Batch size ${batchSize} is very large and may cause memory issues. Recommended maximum: 1000`);
+    }
+  }
+
+  // Validate max concurrency
+  if (options.maxConcurrency) {
+    const maxConcurrency = parseInt(options.maxConcurrency);
+    if (isNaN(maxConcurrency) || maxConcurrency < 1) {
+      console.error('Error: Max concurrency must be a positive number');
+      process.exit(1);
+    } else if (maxConcurrency > 50) {
+      console.warn(`Warning: Max concurrency ${maxConcurrency} is very high and may cause system instability. Recommended maximum: 50`);
+    }
+  }
 }
 
 export {