@arela/uploader 1.1.3 → 1.2.0

package/.claude/worktrees/agent-multi-profile/src/commands/ScanCommand.js

@@ -0,0 +1,722 @@
+import cliProgress from 'cli-progress';
+import { globby, globbyStream } from 'globby';
+import path from 'path';
+import { Transform } from 'stream';
+import { pipeline } from 'stream/promises';
+
+import logger from '../services/LoggingService.js';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+import { ConfigurationError } from '../errors/ErrorTypes.js';
+import FileOperations from '../utils/FileOperations.js';
+import PathNormalizer from '../utils/PathNormalizer.js';
+
+/**
+ * Scan Command Handler
+ * Handles the optimized arela scan command with streaming support
+ */
+export class ScanCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+    this.scanApiService = null; // Will be initialized in execute
+  }
+
+  /**
+   * Execute the scan command
+   * @param {Object} options - Command options
+   * @param {boolean} options.countFirst - Count files first for percentage-based progress
+   * @param {string} options.api - API target: 'default', 'agencia', or 'cliente'
+   * @param {boolean} options.stream - Use streaming file discovery (default: true, use --no-stream to disable)
+   * @param {Function} options.onProgress - Optional callback for progress updates (percent, message)
+   */
+  async execute(options = {}) {
+    const startTime = Date.now();
+    this.onProgress = options.onProgress || null;
+
+    try {
+      // Validate scan configuration
+      appConfig.validateScanConfig();
+
+      // Determine API target
+      const apiTarget = options.api || 'default';
+
+      // Import ScanApiService dynamically and initialize with target
+      const { default: ScanApiService } = await import(
+        '../services/ScanApiService.js'
+      );
+      this.scanApiService = new ScanApiService(apiTarget);
+
+      const scanConfig = appConfig.getScanConfig();
+      // Ensure basePath is absolute for scan operations
+      // '*' is a wildcard sentinel for cross-drive pipelines — no real basePath
+      const rawBasePath = appConfig.getBasePath();
+      const basePath =
+        rawBasePath === '*' ? '*' : PathNormalizer.toAbsolutePath(rawBasePath);
+
+      logger.info('🔍 Starting arela scan command');
+      logger.info(`🎯 API Target: ${apiTarget}`);
+      logger.info(`📦 Company: ${scanConfig.companySlug}`);
+      logger.info(`🖥️  Server: ${scanConfig.serverId}`);
+      logger.info(`📂 Base Path: ${basePath}`);
+      logger.info(`📊 Directory Level: ${scanConfig.directoryLevel}`);
+      logger.info(
+        `🔄 File Discovery: ${options.stream !== false ? 'Streaming (globbyStream)' : 'Synchronous (globby)'}`,
+      );
+
+      // Step 1: Discover directories at specified level
+      logger.info('\n🔍 Discovering directories...');
+      const directories = await this.#discoverDirectories(
+        basePath,
+        scanConfig.directoryLevel,
+      );
+      logger.info(
+        `📁 Found ${directories.length} director${directories.length === 1 ? 'y' : 'ies'} to scan`,
+      );
+
+      // Step 2: Register instances for each directory
+      logger.info('\n📝 Registering scan instances...');
+      const registrations = [];
+      for (const dir of directories) {
+        // dir.path is already absolute from #discoverDirectories
+        // Use the absolute path as basePathLabel (simplifies everything!)
+        const absolutePath = dir.path;
+
+        const registration = await this.scanApiService.registerInstance({
+          companySlug: scanConfig.companySlug,
+          serverId: scanConfig.serverId,
+          basePathFull: absolutePath,
+        });
+
+        registrations.push({ ...registration, directory: dir });
+
+        if (registration.existed) {
+          logger.info(`  ✓ ${dir.label || 'root'}: ${registration.tableName}`);
+        } else {
+          logger.success(
+            `  ✓ ${dir.label || 'root'}: ${registration.tableName} (new)`,
+          );
+        }
+      }
+
+      // Optional: Count files first for percentage-based progress
+      let totalFiles = null;
+      if (options.countFirst) {
+        logger.info('\n🔢 Counting files...');
+        totalFiles = await this.#countFiles(
+          basePath,
+          scanConfig,
+          options.stream !== false,
+        );
+        logger.info(`📊 Found ${totalFiles.toLocaleString()} files to scan`);
+      }
+
+      // Step 3: Stream files and upload stats for each directory
+      logger.info('\n🚀 Starting file scan...');
+      let totalStats = {
+        filesScanned: 0,
+        filesInserted: 0,
+        filesSkipped: 0,
+        totalSize: 0,
+      };
+
+      // Report initial progress
+      this.#reportProgress(
+        0,
+        `Starting scan of ${registrations.length} directories`,
+      );
+
+      for (let i = 0; i < registrations.length; i++) {
+        const reg = registrations[i];
+        logger.info(`\n📂 Scanning: ${reg.directory.label || 'root'}`);
+        this.#reportProgress(
+          Math.round((i / registrations.length) * 100),
+          `Scanning directory ${i + 1}/${registrations.length}: ${reg.directory.label || 'root'}`,
+        );
+
+        const stats = await this.#streamScanDirectory(
+          reg.directory.path,
+          scanConfig,
+          reg.tableName,
+          null, // Don't use percentage for individual directories
+          options.stream !== false, // Use streaming by default, --no-stream sets stream=false
+        );
+
+        // Step 4: Complete scan for this directory
+        await this.scanApiService.completeScan({
+          tableName: reg.tableName,
+          totalFiles: stats.filesScanned,
+          totalSizeBytes: stats.totalSize,
+        });
+
+        totalStats.filesScanned += stats.filesScanned;
+        totalStats.filesInserted += stats.filesInserted;
+        totalStats.filesSkipped += stats.filesSkipped;
+        totalStats.totalSize += stats.totalSize;
+      }
+
+      const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+      const filesPerSec = (totalStats.filesScanned / duration).toFixed(2);
+
+      // Report completion
+      this.#reportProgress(
+        100,
+        `Scan completed: ${totalStats.filesScanned} files in ${duration}s`,
+      );
+
+      logger.success('\n✅ Scan completed successfully!');
+      logger.info(`\n📊 Scan Statistics:`);
+      logger.info(`   Directories scanned: ${registrations.length}`);
+      logger.info(
+        `   Files scanned: ${totalStats.filesScanned.toLocaleString()}`,
+      );
+      logger.info(
+        `   Files inserted: ${totalStats.filesInserted.toLocaleString()}`,
+      );
+      logger.info(
+        `   Files skipped: ${totalStats.filesSkipped.toLocaleString()} (excluded patterns)`,
+      );
+      logger.info(`   Total size: ${this.#formatBytes(totalStats.totalSize)}`);
+      logger.info(`   Duration: ${duration}s`);
+      logger.info(`   Throughput: ${filesPerSec} files/sec`);
+      logger.info(`\n📋 Tables created:`);
+      for (const reg of registrations) {
+        logger.info(`   - ${reg.tableName}`);
+      }
+
+      return {
+        success: true,
+        tables: registrations.map((r) => r.tableName),
+        stats: totalStats,
+      };
+    } catch (error) {
+      this.errorHandler.handleError(error, 'scan');
+      return {
+        success: false,
+        error: error.message,
+        stats: {
+          filesScanned: 0,
+          filesInserted: 0,
+          filesSkipped: 0,
+          totalSize: 0,
+        },
+      };
+    }
+  }
+
+  /**
+   * Discover directories at specified level
+   * @private
+   */
+  async #discoverDirectories(basePath, level) {
+    // Get sources, defaults to ['.'] if not configured
+    const sources = appConfig.getUploadSources();
+    const isDefaultSource = sources.length === 1 && sources[0] === '.';
+
+    if (level === 0) {
+      // Level 0: Create one entry per source
+      return sources.map((source) => {
+        let sourcePath;
+        if (source === '.') {
+          sourcePath = basePath;
+        } else if (source.startsWith('..') || path.isAbsolute(source)) {
+          // Source is a relative-to-CWD path (e.g., from pipeline UI) or absolute
+          sourcePath = PathNormalizer.toAbsolutePath(source);
+        } else {
+          // Source is a subdirectory of basePath
+          sourcePath = path.resolve(basePath, source);
+        }
+        // Label is relative path for display purposes only
+        const label = source === '.' ? '' : source;
+        return { path: sourcePath, label };
+      });
+    }
+
+    // For level > 0: First discover directories at the base path, then combine with sources
+    const fs = await import('fs/promises');
+    const directories = [];
+
+    try {
+      const fs = await import('fs/promises');
+
+      // Step 1: Discover directories at the specified level from base path
+      const levelDirs = await this.#getDirectoriesAtLevel(basePath, level, '');
+
+      // Step 2: For each discovered directory, create entries for each source
+      for (const levelDir of levelDirs) {
+        for (const source of sources) {
+          if (source === '.') {
+            // Source is current directory, use discovered path as-is
+            directories.push(levelDir);
+          } else {
+            // Resolve source: if it starts with ".." it's relative to CWD, not levelDir
+            const combinedPath =
+              source.startsWith('..') || path.isAbsolute(source)
+                ? PathNormalizer.toAbsolutePath(source)
+                : path.resolve(levelDir.path, source);
+
+            // Only add if the combined path actually exists
+            try {
+              const stats = await fs.stat(combinedPath);
+              if (stats.isDirectory()) {
+                // Label for display
+                const label = levelDir.label
+                  ? `${levelDir.label}/${source}`
+                  : source;
+                directories.push({
+                  path: combinedPath,
+                  label,
+                });
+              } else {
+                logger.debug(`⏭️ Skipping ${combinedPath} (not a directory)`);
+              }
+            } catch (error) {
+              logger.debug(`⏭️ Skipping ${combinedPath} (does not exist)`);
+            }
+          }
+        }
+      }
+    } catch (error) {
+      logger.warn(`⚠️ Could not discover directories: ${error.message}`);
+    }
+
+    return directories;
+  }
+
+  /**
+   * Recursively get directories at specified level
+   * @private
+   */
+  async #getDirectoriesAtLevel(
+    basePath,
+    targetLevel,
+    currentPath,
+    currentLevel = 0,
+  ) {
+    const fs = await import('fs/promises');
+    const fullPath = path.join(basePath, currentPath);
+
+    if (currentLevel === targetLevel) {
+      // Label is the relative path for display
+      const label = currentPath || '';
+      return [{ path: fullPath, label }];
+    }
+
+    const directories = [];
+    const entries = await fs.readdir(fullPath, { withFileTypes: true });
+
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        const subPath = path.join(currentPath, entry.name);
+        const subDirs = await this.#getDirectoriesAtLevel(
+          basePath,
+          targetLevel,
+          subPath,
+          currentLevel + 1,
+        );
+        directories.push(...subDirs);
+      }
+    }
+
+    return directories;
+  }
+
+  /**
+   * Count files for percentage-based progress
+   * @private
+   * @param {string} basePath - Base path to count from
+   * @param {Object} scanConfig - Scan configuration
+   * @param {boolean} useStream - Use streaming (globbyStream) or sync (globby) approach
+   */
+  async #countFiles(basePath, scanConfig, useStream = true) {
+    const sources = appConfig.getUploadSources();
+    let totalCount = 0;
+
+    for (const source of sources) {
+      const sourcePath = path.resolve(basePath, source);
+
+      if (useStream) {
+        // Streaming approach
+        const files = await globbyStream('**/*', {
+          cwd: sourcePath,
+          onlyFiles: true,
+          absolute: true,
+        });
+
+        for await (const file of files) {
+          if (!this.#shouldExcludeFile(file, scanConfig.excludePatterns)) {
+            totalCount++;
+          }
+        }
+      } else {
+        // Synchronous approach (original method)
+        const files = await globby([`${sourcePath}/**/*`], { onlyFiles: true });
+        for (const file of files) {
+          if (!this.#shouldExcludeFile(file, scanConfig.excludePatterns)) {
+            totalCount++;
+          }
+        }
+      }
+    }
+
+    return totalCount;
+  }
+
+  /**
+   * Stream files from a single directory and upload stats in batches
+   * @private
+   * @param {string} dirPath - Directory path to scan
+   * @param {Object} scanConfig - Scan configuration
+   * @param {string} tableName - Target table name
+   * @param {number|null} totalFiles - Total files count for progress (optional)
+   * @param {boolean} useStream - Use streaming (globbyStream) or sync (globby) approach
+   */
+  async #streamScanDirectory(
+    dirPath,
+    scanConfig,
+    tableName,
+    totalFiles = null,
+    useStream = true,
+  ) {
+    // For directory-level scanning, we scan the directory directly
+    const batchSize = scanConfig.batchSize || 2000;
+    const scanTimestamp = new Date().toISOString();
+
+    let filesScanned = 0;
+    let filesInserted = 0;
+    let filesSkipped = 0;
+    let totalSize = 0;
+    let currentBatch = [];
+
+    // Create progress bar
+    const progressBar = this.#createProgressBar(totalFiles);
+
+    try {
+      if (useStream) {
+        // Streaming approach: use globbyStream with stats option
+        const fileStream = globbyStream('**/*', {
+          cwd: dirPath,
+          onlyFiles: true,
+          absolute: true,
+          stats: true, // Get file stats during discovery
+        });
+
+        // Process each file from stream
+        for await (const entry of fileStream) {
+          // globby with stats:true returns {path, stats} objects
+          const filePath = typeof entry === 'string' ? entry : entry.path;
+          const stats = typeof entry === 'object' ? entry.stats : null;
+
+          const result = await this.#processFileEntry(
+            filePath,
+            stats,
+            dirPath,
+            scanConfig,
+            scanTimestamp,
+            tableName,
+            currentBatch,
+            batchSize,
+            progressBar,
+            totalFiles,
+            { filesScanned, filesInserted, filesSkipped, totalSize },
+          );
+
+          filesScanned = result.filesScanned;
+          filesInserted = result.filesInserted;
+          filesSkipped = result.filesSkipped;
+          totalSize = result.totalSize;
+          currentBatch = result.currentBatch;
+        }
+      } else {
+        // Synchronous approach: use globby (original method from UploadCommand)
+        logger.debug('Using synchronous file discovery (globby)...');
+        const files = await globby([`${dirPath}/**/*`], { onlyFiles: true });
+        logger.debug(`Found ${files.length} files to process`);
+
+        for (const filePath of files) {
+          const result = await this.#processFileEntry(
+            filePath,
+            null, // Stats will be fetched manually
+            dirPath,
+            scanConfig,
+            scanTimestamp,
+            tableName,
+            currentBatch,
+            batchSize,
+            progressBar,
+            totalFiles,
+            { filesScanned, filesInserted, filesSkipped, totalSize },
+          );
+
+          filesScanned = result.filesScanned;
+          filesInserted = result.filesInserted;
+          filesSkipped = result.filesSkipped;
+          totalSize = result.totalSize;
+          currentBatch = result.currentBatch;
+        }
+      }
+    } catch (error) {
+      logger.error(`❌ Error scanning directory: ${error.message}`);
+    }
+
+    // Upload remaining files
+    if (currentBatch.length > 0) {
+      const inserted = await this.#uploadBatch(tableName, currentBatch);
+      filesInserted += inserted;
+    }
+
+    progressBar.stop();
+
+    return {
+      filesScanned,
+      filesInserted,
+      filesSkipped,
+      totalSize,
+    };
+  }
+
+  /**
+   * Process a single file entry (used by both streaming and sync approaches)
+   * @private
+   */
+  async #processFileEntry(
+    filePath,
+    stats,
+    dirPath,
+    scanConfig,
+    scanTimestamp,
+    tableName,
+    currentBatch,
+    batchSize,
+    progressBar,
+    totalFiles,
+    counters,
+  ) {
+    let { filesScanned, filesInserted, filesSkipped, totalSize } = counters;
+
+    // Check if file should be excluded
+    if (this.#shouldExcludeFile(filePath, scanConfig.excludePatterns)) {
+      filesSkipped++;
+      return {
+        filesScanned,
+        filesInserted,
+        filesSkipped,
+        totalSize,
+        currentBatch,
+      };
+    }
+
+    // Get file stats (use from globby or fetch manually)
+    const fileStats = stats || FileOperations.getFileStats(filePath);
+    if (!fileStats) {
+      logger.debug(`⚠️ Could not read stats: ${filePath}`);
+      filesSkipped++;
+      return {
+        filesScanned,
+        filesInserted,
+        filesSkipped,
+        totalSize,
+        currentBatch,
+      };
+    }
+
+    // Normalize file record
+    const record = this.#normalizeFileRecord(
+      filePath,
+      fileStats,
+      dirPath,
+      scanTimestamp,
+    );
+
+    currentBatch.push(record);
+    filesScanned++;
+    totalSize += record.sizeBytes;
+
+    // Update progress
+    if (totalFiles) {
+      progressBar.update(filesScanned);
+    } else {
+      // Show throughput instead of percentage
+      const elapsed = (Date.now() - progressBar.startTime) / 1000;
+      const rate = (filesScanned / elapsed).toFixed(1);
+      progressBar.update(filesScanned, { rate });
+    }
+
+    // Upload batch when full
+    if (currentBatch.length >= batchSize) {
+      const inserted = await this.#uploadBatch(tableName, currentBatch);
+      filesInserted += inserted;
+      currentBatch = [];
+    }
+
+    return {
+      filesScanned,
+      filesInserted,
+      filesSkipped,
+      totalSize,
+      currentBatch,
+    };
+  }
+
+  /**
+   * Upload a batch of file records
+   * @private
+   */
+  async #uploadBatch(tableName, records) {
+    try {
+      const result = await this.scanApiService.batchInsertStats(
+        tableName,
+        records,
+      );
+      return result.inserted;
+    } catch (error) {
+      logger.error(`❌ Failed to upload batch: ${error.message}`);
+      return 0;
+    }
+  }
+
+  /**
+   * Normalize file record for database insertion
+   * Stores paths with forward slashes for consistency but keeps them absolute
+   * Sets likelySimplificado to true if file is a PDF and filename contains 'simp'
+   * Sets likelyInterAgencia to true if filename matches an inter-agency CFDI
+   * pattern (e.g. SICINGR*), so the API forces these XML/PDF through detection
+   * even though they lack the 'simp/pedim/covefact' heuristic.
+   * @private
+   */
+  #normalizeFileRecord(filePath, fileStats, basePath, scanTimestamp) {
+    const fileName = path.basename(filePath);
+    const fileExtension = path.extname(filePath).toLowerCase().replace('.', '');
+
+    // Normalize separators to forward slashes for consistency
+    const directoryPath = PathNormalizer.normalizeSeparators(
+      path.dirname(filePath),
+    );
+    const relativePath = PathNormalizer.getRelativePath(filePath, basePath);
+    const absolutePath = PathNormalizer.normalizeSeparators(filePath);
+
+    // Determine if this file is potentially a pedimento (simplificado, completo, or CoveFact).
+    // PDFs whose filename contains 'simp', 'pedim' or 'covefact' (case-insensitive)
+    // are flagged so the identify stage prioritizes them. The column name
+    // `likely_simplificado` is preserved for backwards compatibility; semantics
+    // are broader (any likely pedimento PDF).
+    const likelySimplificado =
+      fileExtension === 'pdf' && /(simp|pedim|covefact)/i.test(fileName);
+
+    // Flag inter-agency CFDIs by filename so detection picks them up.
+    // Patterns are configurable via SCAN_INTER_AGENCIA_PATTERNS env var
+    // (see config.js). Only meaningful for PDF and XML.
+    let likelyInterAgencia = false;
+    if (fileExtension === 'pdf' || fileExtension === 'xml') {
+      const patterns = appConfig.scan.interAgenciaPatterns;
+      if (patterns && patterns.length > 0) {
+        likelyInterAgencia = patterns.some((re) => re.test(fileName));
+      }
+    }
+
+    return {
+      fileName,
+      fileExtension,
+      directoryPath,
+      relativePath,
+      absolutePath,
+      sizeBytes: Number(fileStats.size),
+      modifiedAt: fileStats.mtime.toISOString(),
+      scanTimestamp,
+      likelySimplificado,
+      likelyInterAgencia,
+    };
+  }
+
+  /**
+   * Check if file should be excluded based on patterns
+   * @private
+   */
+  #shouldExcludeFile(filePath, excludePatterns) {
+    const fileName = path.basename(filePath);
+
+    for (const pattern of excludePatterns) {
+      // Convert glob pattern to regex
+      const regexPattern = pattern
+        .replace(/\./g, '\\.') // Escape dots
+        .replace(/\*/g, '.*') // * to .*
+        .replace(/\?/g, '.'); // ? to .
+
+      const regex = new RegExp(`^${regexPattern}$`, 'i');
+      if (regex.test(fileName)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * Create progress bar
+   * @private
+   */
+  #createProgressBar(totalFiles) {
+    if (totalFiles) {
+      // Percentage-based progress
+      const bar = new cliProgress.SingleBar(
+        {
+          format:
+            '📊 Scanning |{bar}| {percentage}% | {value}/{total} files | {rate} files/sec',
+          barCompleteChar: '\u2588',
+          barIncompleteChar: '\u2591',
+          hideCursor: true,
+        },
+        cliProgress.Presets.shades_classic,
+      );
+      bar.start(totalFiles, 0, { rate: '0.0' });
+      bar.startTime = Date.now();
+      return bar;
+    } else {
+      // Throughput-based progress
+      const bar = new cliProgress.SingleBar(
+        {
+          format: '📊 Scanning | {value} files | {rate} files/sec',
+          hideCursor: true,
+          clearOnComplete: false,
+          stopOnComplete: false,
+        },
+        cliProgress.Presets.legacy,
+      );
+      bar.start(0, 0, { rate: '0.0' });
+      bar.startTime = Date.now();
+      return bar;
+    }
+  }
+
+  /**
+   * Format bytes to human-readable size
+   * @private
+   */
+  #formatBytes(bytes) {
+    if (bytes === 0) return '0 Bytes';
+
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+  }
+
+  /**
+   * Report progress to callback if available (for worker mode)
+   * @private
+   * @param {number} percent - Progress percentage (0-100)
+   * @param {string} message - Progress message
+   */
+  #reportProgress(percent, message) {
+    if (this.onProgress && typeof this.onProgress === 'function') {
+      try {
+        this.onProgress(percent, message);
+      } catch (error) {
+        logger.debug(`Progress callback error: ${error.message}`);
+      }
+    }
+  }
+}
+
+export default new ScanCommand();