@arela/uploader 1.0.8 → 1.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "1.0.8",
+  "version": "1.0.9",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"

package/src/commands/ScanCommand.js

@@ -1,5 +1,5 @@
 import cliProgress from 'cli-progress';
-import { globbyStream } from 'globby';
+import { globby, globbyStream } from 'globby';
 import path from 'path';
 import { Transform } from 'stream';
 import { pipeline } from 'stream/promises';
@@ -27,6 +27,7 @@ export class ScanCommand {
    * @param {Object} options - Command options
    * @param {boolean} options.countFirst - Count files first for percentage-based progress
    * @param {string} options.api - API target: 'default', 'agencia', or 'cliente'
+   * @param {boolean} options.stream - Use streaming file discovery (default: true, use --no-stream to disable)
    */
   async execute(options = {}) {
     const startTime = Date.now();
@@ -54,6 +55,9 @@ export class ScanCommand {
       logger.info(`🖥️  Server: ${scanConfig.serverId}`);
       logger.info(`📂 Base Path: ${basePath}`);
       logger.info(`📊 Directory Level: ${scanConfig.directoryLevel}`);
+      logger.info(
+        `🔄 File Discovery: ${options.stream !== false ? 'Streaming (globbyStream)' : 'Synchronous (globby)'}`,
+      );
 
       // Step 1: Discover directories at specified level
       logger.info('\n🔍 Discovering directories...');
@@ -94,7 +98,11 @@ export class ScanCommand {
       let totalFiles = null;
       if (options.countFirst) {
         logger.info('\n🔢 Counting files...');
-        totalFiles = await this.#countFiles(basePath, scanConfig);
+        totalFiles = await this.#countFiles(
+          basePath,
+          scanConfig,
+          options.stream !== false,
+        );
         logger.info(`📊 Found ${totalFiles.toLocaleString()} files to scan`);
       }
 
@@ -114,6 +122,7 @@ export class ScanCommand {
           scanConfig,
           reg.tableName,
           null, // Don't use percentage for individual directories
+          options.stream !== false, // Use streaming by default, --no-stream sets stream=false
         );
 
         // Step 4: Complete scan for this directory
@@ -281,22 +290,37 @@ export class ScanCommand {
   /**
    * Count files for percentage-based progress
    * @private
+   * @param {string} basePath - Base path to count from
+   * @param {Object} scanConfig - Scan configuration
+   * @param {boolean} useStream - Use streaming (globbyStream) or sync (globby) approach
    */
-  async #countFiles(basePath, scanConfig) {
+  async #countFiles(basePath, scanConfig, useStream = true) {
     const sources = appConfig.getUploadSources();
     let totalCount = 0;
 
     for (const source of sources) {
       const sourcePath = path.resolve(basePath, source);
-      const files = await globbyStream('**/*', {
-        cwd: sourcePath,
-        onlyFiles: true,
-        absolute: true,
-      });
 
-      for await (const file of files) {
-        if (!this.#shouldExcludeFile(file, scanConfig.excludePatterns)) {
-          totalCount++;
+      if (useStream) {
+        // Streaming approach
+        const files = await globbyStream('**/*', {
+          cwd: sourcePath,
+          onlyFiles: true,
+          absolute: true,
+        });
+
+        for await (const file of files) {
+          if (!this.#shouldExcludeFile(file, scanConfig.excludePatterns)) {
+            totalCount++;
+          }
+        }
+      } else {
+        // Synchronous approach (original method)
+        const files = await globby([`${sourcePath}/**/*`], { onlyFiles: true });
+        for (const file of files) {
+          if (!this.#shouldExcludeFile(file, scanConfig.excludePatterns)) {
+            totalCount++;
+          }
         }
       }
     }
@@ -307,12 +331,18 @@ export class ScanCommand {
   /**
    * Stream files from a single directory and upload stats in batches
    * @private
+   * @param {string} dirPath - Directory path to scan
+   * @param {Object} scanConfig - Scan configuration
+   * @param {string} tableName - Target table name
+   * @param {number|null} totalFiles - Total files count for progress (optional)
+   * @param {boolean} useStream - Use streaming (globbyStream) or sync (globby) approach
    */
   async #streamScanDirectory(
     dirPath,
     scanConfig,
     tableName,
     totalFiles = null,
+    useStream = true,
   ) {
     // For directory-level scanning, we scan the directory directly
     const batchSize = scanConfig.batchSize || 2000;
@@ -328,61 +358,67 @@ export class ScanCommand {
     const progressBar = this.#createProgressBar(totalFiles);
 
     try {
-      // Create stream with stats option
-      const fileStream = globbyStream('**/*', {
-        cwd: dirPath,
-        onlyFiles: true,
-        absolute: true,
-        stats: true, // Get file stats during discovery
-      });
-
-      // Process each file from stream
-      for await (const entry of fileStream) {
-        // globby with stats:true returns {path, stats} objects
-        const filePath = typeof entry === 'string' ? entry : entry.path;
-        const stats = typeof entry === 'object' ? entry.stats : null;
-
-        // Check if file should be excluded
-        if (this.#shouldExcludeFile(filePath, scanConfig.excludePatterns)) {
-          filesSkipped++;
-          continue;
-        }
-
-        // Get file stats (use from globby or fetch manually)
-        const fileStats = stats || FileOperations.getFileStats(filePath);
-        if (!fileStats) {
-          logger.debug(`⚠️ Could not read stats: ${filePath}`);
-          filesSkipped++;
-          continue;
-        }
-
-        // Normalize file record
-        const record = this.#normalizeFileRecord(
-          filePath,
-          fileStats,
-          dirPath,
-          scanTimestamp,
-        );
+      if (useStream) {
+        // Streaming approach: use globbyStream with stats option
+        const fileStream = globbyStream('**/*', {
+          cwd: dirPath,
+          onlyFiles: true,
+          absolute: true,
+          stats: true, // Get file stats during discovery
+        });
 
-        currentBatch.push(record);
-        filesScanned++;
-        totalSize += record.sizeBytes;
+        // Process each file from stream
+        for await (const entry of fileStream) {
+          // globby with stats:true returns {path, stats} objects
+          const filePath = typeof entry === 'string' ? entry : entry.path;
+          const stats = typeof entry === 'object' ? entry.stats : null;
+
+          const result = await this.#processFileEntry(
+            filePath,
+            stats,
+            dirPath,
+            scanConfig,
+            scanTimestamp,
+            tableName,
+            currentBatch,
+            batchSize,
+            progressBar,
+            totalFiles,
+            { filesScanned, filesInserted, filesSkipped, totalSize },
+          );
 
-        // Update progress
-        if (totalFiles) {
-          progressBar.update(filesScanned);
-        } else {
-          // Show throughput instead of percentage
-          const elapsed = (Date.now() - progressBar.startTime) / 1000;
-          const rate = (filesScanned / elapsed).toFixed(1);
-          progressBar.update(filesScanned, { rate });
+          filesScanned = result.filesScanned;
+          filesInserted = result.filesInserted;
+          filesSkipped = result.filesSkipped;
+          totalSize = result.totalSize;
+          currentBatch = result.currentBatch;
         }
+      } else {
+        // Synchronous approach: use globby (original method from UploadCommand)
+        logger.debug('Using synchronous file discovery (globby)...');
+        const files = await globby([`${dirPath}/**/*`], { onlyFiles: true });
+        logger.debug(`Found ${files.length} files to process`);
+
+        for (const filePath of files) {
+          const result = await this.#processFileEntry(
+            filePath,
+            null, // Stats will be fetched manually
+            dirPath,
+            scanConfig,
+            scanTimestamp,
+            tableName,
+            currentBatch,
+            batchSize,
+            progressBar,
+            totalFiles,
+            { filesScanned, filesInserted, filesSkipped, totalSize },
+          );
 
-        // Upload batch when full
-        if (currentBatch.length >= batchSize) {
-          const inserted = await this.#uploadBatch(tableName, currentBatch);
-          filesInserted += inserted;
-          currentBatch = [];
+          filesScanned = result.filesScanned;
+          filesInserted = result.filesInserted;
+          filesSkipped = result.filesSkipped;
+          totalSize = result.totalSize;
+          currentBatch = result.currentBatch;
         }
       }
     } catch (error) {
@@ -405,6 +441,89 @@ export class ScanCommand {
     };
   }
 
+  /**
+   * Process a single file entry (used by both streaming and sync approaches)
+   * @private
+   */
+  async #processFileEntry(
+    filePath,
+    stats,
+    dirPath,
+    scanConfig,
+    scanTimestamp,
+    tableName,
+    currentBatch,
+    batchSize,
+    progressBar,
+    totalFiles,
+    counters,
+  ) {
+    let { filesScanned, filesInserted, filesSkipped, totalSize } = counters;
+
+    // Check if file should be excluded
+    if (this.#shouldExcludeFile(filePath, scanConfig.excludePatterns)) {
+      filesSkipped++;
+      return {
+        filesScanned,
+        filesInserted,
+        filesSkipped,
+        totalSize,
+        currentBatch,
+      };
+    }
+
+    // Get file stats (use from globby or fetch manually)
+    const fileStats = stats || FileOperations.getFileStats(filePath);
+    if (!fileStats) {
+      logger.debug(`⚠️ Could not read stats: ${filePath}`);
+      filesSkipped++;
+      return {
+        filesScanned,
+        filesInserted,
+        filesSkipped,
+        totalSize,
+        currentBatch,
+      };
+    }
+
+    // Normalize file record
+    const record = this.#normalizeFileRecord(
+      filePath,
+      fileStats,
+      dirPath,
+      scanTimestamp,
+    );
+
+    currentBatch.push(record);
+    filesScanned++;
+    totalSize += record.sizeBytes;
+
+    // Update progress
+    if (totalFiles) {
+      progressBar.update(filesScanned);
+    } else {
+      // Show throughput instead of percentage
+      const elapsed = (Date.now() - progressBar.startTime) / 1000;
+      const rate = (filesScanned / elapsed).toFixed(1);
+      progressBar.update(filesScanned, { rate });
+    }
+
+    // Upload batch when full
+    if (currentBatch.length >= batchSize) {
+      const inserted = await this.#uploadBatch(tableName, currentBatch);
+      filesInserted += inserted;
+      currentBatch = [];
+    }
+
+    return {
+      filesScanned,
+      filesInserted,
+      filesSkipped,
+      totalSize,
+      currentBatch,
+    };
+  }
+
   /**
    * Upload a batch of file records
    * @private

package/src/config/config.js

@@ -34,10 +34,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '1.0.8';
+      return packageJson.version || '1.0.9';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '1.0.8';
+      return '1.0.9';
     }
   }
 

package/src/index.js

@@ -185,6 +185,10 @@ class ArelaUploaderCLI {
         '--count-first',
         'Count files first for percentage-based progress (slower start)',
       )
+      .option(
+        '--no-stream',
+        'Use synchronous file discovery instead of streaming (original approach)',
+      )
       .action(async (options) => {
         try {
           // Set API target if specified