@arela/uploader 1.0.2 → 1.0.3

package/src/commands/ScanCommand.js

@@ -0,0 +1,516 @@
+import cliProgress from 'cli-progress';
+import { globbyStream } from 'globby';
+import path from 'path';
+import { Transform } from 'stream';
+import { pipeline } from 'stream/promises';
+
+import logger from '../services/LoggingService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+import { ConfigurationError } from '../errors/ErrorTypes.js';
+import FileOperations from '../utils/FileOperations.js';
+
+/**
+ * Scan Command Handler
+ * Handles the optimized arela scan command with streaming support
+ */
+export class ScanCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+    this.scanApiService = null; // Will be initialized in execute
+  }
+
+  /**
+   * Execute the scan command
+   * @param {Object} options - Command options
+   * @param {boolean} options.countFirst - Count files first for percentage-based progress
+   */
+  async execute(options = {}) {
+    const startTime = Date.now();
+
+    try {
+      // Validate scan configuration
+      appConfig.validateScanConfig();
+
+      // Import ScanApiService dynamically
+      const { default: ScanApiService } = await import(
+        '../services/ScanApiService.js'
+      );
+      this.scanApiService = new ScanApiService();
+
+      const scanConfig = appConfig.getScanConfig();
+      const basePath = appConfig.getBasePath();
+
+      logger.info('🔍 Starting arela scan command');
+      logger.info(`📦 Company: ${scanConfig.companySlug}`);
+      logger.info(`🖥️  Server: ${scanConfig.serverId}`);
+      logger.info(`📂 Base Path: ${basePath}`);
+      logger.info(`🏷️  Label: ${scanConfig.basePathLabel}`);
+      logger.info(`📊 Directory Level: ${scanConfig.directoryLevel}`);
+
+      // Step 1: Discover directories at specified level
+      logger.info('\n🔍 Discovering directories...');
+      const directories = await this.#discoverDirectories(
+        basePath,
+        scanConfig.directoryLevel,
+      );
+      logger.info(
+        `📁 Found ${directories.length} director${directories.length === 1 ? 'y' : 'ies'} to scan`,
+      );
+
+      // Step 2: Register instances for each directory
+      logger.info('\n📝 Registering scan instances...');
+      const registrations = [];
+      for (const dir of directories) {
+        const dirLabel = dir.label
+          ? `${scanConfig.basePathLabel}_${dir.label.replace(/[^a-zA-Z0-9_-]/g, '_')}`
+          : scanConfig.basePathLabel;
+
+        const registration = await this.scanApiService.registerInstance({
+          companySlug: scanConfig.companySlug,
+          serverId: scanConfig.serverId,
+          basePathLabel: dirLabel,
+          basePathFull: dir.path,
+        });
+
+        registrations.push({ ...registration, directory: dir });
+
+        if (registration.existed) {
+          logger.info(`  ✓ ${dir.label || 'root'}: ${registration.tableName}`);
+        } else {
+          logger.success(
+            `  ✓ ${dir.label || 'root'}: ${registration.tableName} (new)`,
+          );
+        }
+      }
+
+      // Optional: Count files first for percentage-based progress
+      let totalFiles = null;
+      if (options.countFirst) {
+        logger.info('\n🔢 Counting files...');
+        totalFiles = await this.#countFiles(basePath, scanConfig);
+        logger.info(`📊 Found ${totalFiles.toLocaleString()} files to scan`);
+      }
+
+      // Step 3: Stream files and upload stats for each directory
+      logger.info('\n🚀 Starting file scan...');
+      let totalStats = {
+        filesScanned: 0,
+        filesInserted: 0,
+        filesSkipped: 0,
+        totalSize: 0,
+      };
+
+      for (const reg of registrations) {
+        logger.info(`\n📂 Scanning: ${reg.directory.label || 'root'}`);
+        const stats = await this.#streamScanDirectory(
+          reg.directory.path,
+          scanConfig,
+          reg.tableName,
+          null, // Don't use percentage for individual directories
+        );
+
+        // Step 4: Complete scan for this directory
+        await this.scanApiService.completeScan({
+          tableName: reg.tableName,
+          totalFiles: stats.filesScanned,
+          totalSizeBytes: stats.totalSize,
+        });
+
+        totalStats.filesScanned += stats.filesScanned;
+        totalStats.filesInserted += stats.filesInserted;
+        totalStats.filesSkipped += stats.filesSkipped;
+        totalStats.totalSize += stats.totalSize;
+      }
+
+      const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+      const filesPerSec = (totalStats.filesScanned / duration).toFixed(2);
+
+      logger.success('\n✅ Scan completed successfully!');
+      logger.info(`\n📊 Scan Statistics:`);
+      logger.info(`   Directories scanned: ${registrations.length}`);
+      logger.info(
+        `   Files scanned: ${totalStats.filesScanned.toLocaleString()}`,
+      );
+      logger.info(
+        `   Files inserted: ${totalStats.filesInserted.toLocaleString()}`,
+      );
+      logger.info(
+        `   Files skipped: ${totalStats.filesSkipped.toLocaleString()} (excluded patterns)`,
+      );
+      logger.info(`   Total size: ${this.#formatBytes(totalStats.totalSize)}`);
+      logger.info(`   Duration: ${duration}s`);
+      logger.info(`   Throughput: ${filesPerSec} files/sec`);
+      logger.info(`\n📋 Tables created:`);
+      for (const reg of registrations) {
+        logger.info(`   - ${reg.tableName}`);
+      }
+
+      return {
+        success: true,
+        tables: registrations.map((r) => r.tableName),
+        stats: totalStats,
+      };
+    } catch (error) {
+      this.errorHandler.handleError(error, 'scan');
+      return {
+        success: false,
+        error: error.message,
+        stats: {
+          filesScanned: 0,
+          filesInserted: 0,
+          filesSkipped: 0,
+          totalSize: 0,
+        },
+      };
+    }
+  }
+
+  /**
+   * Discover directories at specified level
+   * @private
+   */
+  async #discoverDirectories(basePath, level) {
+    // Get sources, defaults to ['.'] if not configured
+    const sources = appConfig.getUploadSources();
+    const isDefaultSource = sources.length === 1 && sources[0] === '.';
+
+    if (level === 0) {
+      // Level 0: Create one entry per source
+      return sources.map((source) => {
+        const sourcePath =
+          source === '.' ? basePath : path.resolve(basePath, source);
+        const label =
+          source === '.' ? '' : source.replace(/[^a-zA-Z0-9_-]/g, '_');
+        return { path: sourcePath, label };
+      });
+    }
+
+    // For level > 0: First discover directories at the base path, then combine with sources
+    const fs = await import('fs/promises');
+    const directories = [];
+
+    try {
+      const fs = await import('fs/promises');
+
+      // Step 1: Discover directories at the specified level from base path
+      const levelDirs = await this.#getDirectoriesAtLevel(basePath, level, '');
+
+      // Step 2: For each discovered directory, create entries for each source
+      for (const levelDir of levelDirs) {
+        for (const source of sources) {
+          if (source === '.') {
+            // Source is current directory, use discovered path as-is
+            directories.push(levelDir);
+          } else {
+            // Append source to both path and label
+            const combinedPath = path.resolve(levelDir.path, source);
+            const sourceLabel = source.replace(/[^a-zA-Z0-9_-]/g, '_');
+            const combinedLabel = levelDir.label
+              ? `${levelDir.label}_${sourceLabel}`
+              : sourceLabel;
+
+            // Only add if the combined path actually exists
+            try {
+              const stats = await fs.stat(combinedPath);
+              if (stats.isDirectory()) {
+                directories.push({
+                  path: combinedPath,
+                  label: combinedLabel,
+                });
+              } else {
+                logger.debug(`⏭️ Skipping ${combinedPath} (not a directory)`);
+              }
+            } catch (error) {
+              logger.debug(`⏭️ Skipping ${combinedPath} (does not exist)`);
+            }
+          }
+        }
+      }
+    } catch (error) {
+      logger.warn(`⚠️ Could not discover directories: ${error.message}`);
+    }
+
+    return directories;
+  }
+
+  /**
+   * Recursively get directories at specified level
+   * @private
+   */
+  async #getDirectoriesAtLevel(
+    basePath,
+    targetLevel,
+    currentPath,
+    currentLevel = 0,
+  ) {
+    const fs = await import('fs/promises');
+    const fullPath = path.join(basePath, currentPath);
+
+    if (currentLevel === targetLevel) {
+      const label = currentPath.replace(/\\/g, '/').replace(/^\//g, '');
+      return [{ path: fullPath, label: label || 'root' }];
+    }
+
+    const directories = [];
+    const entries = await fs.readdir(fullPath, { withFileTypes: true });
+
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        const subPath = path.join(currentPath, entry.name);
+        const subDirs = await this.#getDirectoriesAtLevel(
+          basePath,
+          targetLevel,
+          subPath,
+          currentLevel + 1,
+        );
+        directories.push(...subDirs);
+      }
+    }
+
+    return directories;
+  }
+
+  /**
+   * Count files for percentage-based progress
+   * @private
+   */
+  async #countFiles(basePath, scanConfig) {
+    const sources = appConfig.getUploadSources();
+    let totalCount = 0;
+
+    for (const source of sources) {
+      const sourcePath = path.resolve(basePath, source);
+      const files = await globbyStream('**/*', {
+        cwd: sourcePath,
+        onlyFiles: true,
+        absolute: true,
+      });
+
+      for await (const file of files) {
+        if (!this.#shouldExcludeFile(file, scanConfig.excludePatterns)) {
+          totalCount++;
+        }
+      }
+    }
+
+    return totalCount;
+  }
+
+  /**
+   * Stream files from a single directory and upload stats in batches
+   * @private
+   */
+  async #streamScanDirectory(
+    dirPath,
+    scanConfig,
+    tableName,
+    totalFiles = null,
+  ) {
+    // For directory-level scanning, we scan the directory directly
+    const batchSize = scanConfig.batchSize || 2000;
+    const scanTimestamp = new Date().toISOString();
+
+    let filesScanned = 0;
+    let filesInserted = 0;
+    let filesSkipped = 0;
+    let totalSize = 0;
+    let currentBatch = [];
+
+    // Create progress bar
+    const progressBar = this.#createProgressBar(totalFiles);
+
+    try {
+      // Create stream with stats option
+      const fileStream = globbyStream('**/*', {
+        cwd: dirPath,
+        onlyFiles: true,
+        absolute: true,
+        stats: true, // Get file stats during discovery
+      });
+
+      // Process each file from stream
+      for await (const entry of fileStream) {
+        // globby with stats:true returns {path, stats} objects
+        const filePath = typeof entry === 'string' ? entry : entry.path;
+        const stats = typeof entry === 'object' ? entry.stats : null;
+
+        // Check if file should be excluded
+        if (this.#shouldExcludeFile(filePath, scanConfig.excludePatterns)) {
+          filesSkipped++;
+          continue;
+        }
+
+        // Get file stats (use from globby or fetch manually)
+        const fileStats = stats || FileOperations.getFileStats(filePath);
+        if (!fileStats) {
+          logger.debug(`⚠️ Could not read stats: ${filePath}`);
+          filesSkipped++;
+          continue;
+        }
+
+        // Normalize file record
+        const record = this.#normalizeFileRecord(
+          filePath,
+          fileStats,
+          dirPath,
+          scanTimestamp,
+        );
+
+        currentBatch.push(record);
+        filesScanned++;
+        totalSize += record.sizeBytes;
+
+        // Update progress
+        if (totalFiles) {
+          progressBar.update(filesScanned);
+        } else {
+          // Show throughput instead of percentage
+          const elapsed = (Date.now() - progressBar.startTime) / 1000;
+          const rate = (filesScanned / elapsed).toFixed(1);
+          progressBar.update(filesScanned, { rate });
+        }
+
+        // Upload batch when full
+        if (currentBatch.length >= batchSize) {
+          const inserted = await this.#uploadBatch(tableName, currentBatch);
+          filesInserted += inserted;
+          currentBatch = [];
+        }
+      }
+    } catch (error) {
+      logger.error(`❌ Error scanning directory: ${error.message}`);
+    }
+
+    // Upload remaining files
+    if (currentBatch.length > 0) {
+      const inserted = await this.#uploadBatch(tableName, currentBatch);
+      filesInserted += inserted;
+    }
+
+    progressBar.stop();
+
+    return {
+      filesScanned,
+      filesInserted,
+      filesSkipped,
+      totalSize,
+    };
+  }
+
+  /**
+   * Upload a batch of file records
+   * @private
+   */
+  async #uploadBatch(tableName, records) {
+    try {
+      const result = await this.scanApiService.batchInsertStats(
+        tableName,
+        records,
+      );
+      return result.inserted;
+    } catch (error) {
+      logger.error(`❌ Failed to upload batch: ${error.message}`);
+      return 0;
+    }
+  }
+
+  /**
+   * Normalize file record for database insertion
+   * @private
+   */
+  #normalizeFileRecord(filePath, fileStats, basePath, scanTimestamp) {
+    const fileName = path.basename(filePath);
+    const fileExtension = path.extname(filePath).toLowerCase().replace('.', '');
+    const directoryPath = path.dirname(filePath);
+    const relativePath = path.relative(basePath, filePath);
+
+    return {
+      fileName,
+      fileExtension,
+      directoryPath,
+      relativePath,
+      absolutePath: filePath,
+      sizeBytes: Number(fileStats.size),
+      modifiedAt: fileStats.mtime.toISOString(),
+      scanTimestamp,
+    };
+  }
+
+  /**
+   * Check if file should be excluded based on patterns
+   * @private
+   */
+  #shouldExcludeFile(filePath, excludePatterns) {
+    const fileName = path.basename(filePath);
+
+    for (const pattern of excludePatterns) {
+      // Convert glob pattern to regex
+      const regexPattern = pattern
+        .replace(/\./g, '\\.') // Escape dots
+        .replace(/\*/g, '.*') // * to .*
+        .replace(/\?/g, '.'); // ? to .
+
+      const regex = new RegExp(`^${regexPattern}$`, 'i');
+      if (regex.test(fileName)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * Create progress bar
+   * @private
+   */
+  #createProgressBar(totalFiles) {
+    if (totalFiles) {
+      // Percentage-based progress
+      const bar = new cliProgress.SingleBar(
+        {
+          format:
+            '📊 Scanning |{bar}| {percentage}% | {value}/{total} files | {rate} files/sec',
+          barCompleteChar: '\u2588',
+          barIncompleteChar: '\u2591',
+          hideCursor: true,
+        },
+        cliProgress.Presets.shades_classic,
+      );
+      bar.start(totalFiles, 0, { rate: '0.0' });
+      bar.startTime = Date.now();
+      return bar;
+    } else {
+      // Throughput-based progress
+      const bar = new cliProgress.SingleBar(
+        {
+          format: '📊 Scanning | {value} files | {rate} files/sec',
+          hideCursor: true,
+          clearOnComplete: false,
+          stopOnComplete: false,
+        },
+        cliProgress.Presets.legacy,
+      );
+      bar.start(0, 0, { rate: '0.0' });
+      bar.startTime = Date.now();
+      return bar;
+    }
+  }
+
+  /**
+   * Format bytes to human-readable size
+   * @private
+   */
+  #formatBytes(bytes) {
+    if (bytes === 0) return '0 Bytes';
+
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+  }
+}
+
+export default new ScanCommand();