@arela/uploader 1.0.1 → 1.0.3

package/src/config/config.js

@@ -15,6 +15,8 @@ class Config {
     this.supabase = this.#loadSupabaseConfig();
     this.api = this.#loadApiConfig();
     this.upload = this.#loadUploadConfig();
+    this.scan = this.#loadScanConfig();
+    this.push = this.#loadPushConfig();
     this.performance = this.#loadPerformanceConfig();
     this.logging = this.#loadLoggingConfig();
     this.watch = this.#loadWatchConfig();
@@ -30,10 +32,10 @@ class Config {
       const __dirname = path.dirname(__filename);
       const packageJsonPath = path.resolve(__dirname, '../../package.json');
       const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-      return packageJson.version || '1.0.1';
+      return packageJson.version || '1.0.3';
     } catch (error) {
       console.warn('⚠️ Could not read package.json version, using fallback');
-      return '1.0.1';
+      return '1.0.3';
     }
   }
 
@@ -239,6 +241,177 @@ class Config {
     };
   }
 
+  /**
+   * Load scan configuration
+   * @private
+   */
+  #loadScanConfig() {
+    const companySlug = process.env.ARELA_COMPANY_SLUG;
+    const serverId = process.env.ARELA_SERVER_ID;
+    let basePathLabel = process.env.ARELA_BASE_PATH_LABEL;
+
+    // Auto-derive basePathLabel from UPLOAD_BASE_PATH if not set
+    if (!basePathLabel && process.env.UPLOAD_BASE_PATH) {
+      const basePath = process.env.UPLOAD_BASE_PATH;
+      // Get the last segment of the path
+      const segments = basePath.split(path.sep).filter(Boolean);
+      basePathLabel = segments[segments.length - 1] || 'root';
+      // Sanitize the label
+      basePathLabel = basePathLabel.replace(/[^a-zA-Z0-9_-]/g, '_');
+    }
+
+    // Parse exclude patterns
+    const defaultExcludePatterns =
+      '.DS_Store,Thumbs.db,desktop.ini,__pycache__,.pyc,.tmp,.swp,$RECYCLE.BIN,System Volume Information,~$*';
+    const excludePatterns = (
+      process.env.SCAN_EXCLUDE_PATTERNS || defaultExcludePatterns
+    )
+      .split(',')
+      .map((p) => p.trim())
+      .filter(Boolean);
+
+    // Generate table name if all components are available
+    let tableName = null;
+    if (companySlug && serverId && basePathLabel) {
+      const rawName = `${companySlug}_${serverId}_${basePathLabel}`;
+      tableName = this.#sanitizeTableName(rawName);
+    }
+
+    return {
+      companySlug,
+      serverId,
+      basePathLabel,
+      tableName,
+      excludePatterns,
+      batchSize: parseInt(process.env.SCAN_BATCH_SIZE) || 2000,
+      directoryLevel: parseInt(process.env.SCAN_DIRECTORY_LEVEL) || 0,
+    };
+  }
+
+  /**
+   * Load push configuration
+   * @private
+   */
+  #loadPushConfig() {
+    const pushRfcs = process.env.PUSH_RFCS?.split('|')
+      .map((s) => s.trim().toUpperCase())
+      .filter(Boolean);
+
+    const pushYears = process.env.PUSH_YEARS?.split('|')
+      .map((s) => parseInt(s.trim(), 10))
+      .filter((y) => !isNaN(y));
+
+    return {
+      rfcs: pushRfcs,
+      years: pushYears,
+      batchSize: parseInt(process.env.PUSH_BATCH_SIZE) || 50,
+      uploadBatchSize: parseInt(process.env.PUSH_UPLOAD_BATCH_SIZE) || 10,
+      bucket:
+        process.env.PUSH_BUCKET || process.env.SUPABASE_BUCKET || 'archivos',
+    };
+  }
+
+  /**
+   * Sanitize and generate table name
+   * @private
+   */
+  #sanitizeTableName(rawName) {
+    // Sanitize: lowercase, replace special chars with underscore
+    let sanitized = rawName
+      .toLowerCase()
+      .replace(/[^a-z0-9_]/g, '_')
+      .replace(/_+/g, '_')
+      .replace(/^_|_$/g, '');
+
+    const prefix = 'scan_';
+    let tableName = prefix + sanitized;
+
+    // PostgreSQL table name limit is 63 characters
+    if (tableName.length > 63) {
+      // Simple hash without crypto module
+      let hash = 0;
+      for (let i = 0; i < rawName.length; i++) {
+        const char = rawName.charCodeAt(i);
+        hash = (hash << 5) - hash + char;
+        hash = hash & hash; // Convert to 32bit integer
+      }
+      const hashStr = Math.abs(hash).toString(36).substring(0, 8);
+      const maxBaseLength = 63 - hashStr.length - 1;
+      tableName = tableName.substring(0, maxBaseLength) + '_' + hashStr;
+    }
+
+    return tableName;
+  }
+
+  /**
+   * Validate scan configuration
+   * @throws {Error} If required scan configuration is missing
+   */
+  validateScanConfig() {
+    const errors = [];
+
+    if (!this.scan.companySlug) {
+      errors.push(
+        'ARELA_COMPANY_SLUG is required (e.g., "acme_corp", "cliente_123")',
+      );
+    } else if (!/^[a-zA-Z0-9_-]+$/.test(this.scan.companySlug)) {
+      errors.push(
+        'ARELA_COMPANY_SLUG must contain only alphanumeric characters, dashes, and underscores',
+      );
+    }
+
+    if (!this.scan.serverId) {
+      errors.push(
+        'ARELA_SERVER_ID is required (e.g., "nas01", "server-mx", "storage-01")',
+      );
+    } else if (!/^[a-zA-Z0-9_-]+$/.test(this.scan.serverId)) {
+      errors.push(
+        'ARELA_SERVER_ID must contain only alphanumeric characters, dashes, and underscores',
+      );
+    }
+
+    if (!this.upload.basePath) {
+      errors.push(
+        'UPLOAD_BASE_PATH is required to determine the scan base path',
+      );
+    }
+
+    if (!this.scan.basePathLabel) {
+      errors.push(
+        'Could not determine base path label. Set ARELA_BASE_PATH_LABEL or UPLOAD_BASE_PATH',
+      );
+    }
+
+    if (errors.length > 0) {
+      throw new Error(
+        '⚠️ Scan configuration errors:\n   - ' + errors.join('\n   - '),
+      );
+    }
+  }
+
+  /**
+   * Get scan configuration
+   * @returns {Object} Scan configuration
+   */
+  getScanConfig() {
+    return {
+      ...this.scan,
+      basePathFull: this.upload.basePath,
+    };
+  }
+
+  /**
+   * Get push configuration
+   * @returns {Object} Push configuration
+   */
+  getPushConfig() {
+    return {
+      ...this.push,
+      rfcs: this.push.rfcs || [],
+      years: this.push.years || [],
+    };
+  }
+
   /**
    * Load performance configuration
    * @private
@@ -365,14 +538,11 @@ class Config {
 
   /**
    * Get upload sources with validation
-   * @returns {string[]} Array of upload sources
-   * @throws {Error} If sources are not configured
+   * @returns {string[]} Array of upload sources (defaults to ['.'] if not configured)
    */
   getUploadSources() {
     if (!this.upload.sources || this.upload.sources.length === 0) {
-      throw new Error(
-        '⚠️ No upload sources configured. Please set UPLOAD_SOURCES environment variable.',
-      );
+      return ['.'];
     }
     return this.upload.sources;
   }

package/src/file-detection.js

@@ -1,11 +1,9 @@
 import fs from 'fs';
-import { getTextExtractor } from 'office-text-extractor';
 import path from 'path';
+import { PDFParse } from 'pdf-parse';
 
 import { extractDocumentFields } from './document-type-shared.js';
 
-const extractor = getTextExtractor();
-
 /**
  * Compose arela_path from extracted pedimento fields
  * Format: RFC/Year/Patente/Aduana/Pedimento/
@@ -151,18 +149,19 @@ export class FileDetectionService {
   }
 
   /**
-   * Extract text from PDF file
+   * Extract text from PDF file using pdf-parse
+   * More reliable for concurrent operations than office-text-extractor
    * @param {string} filePath - Path to PDF file
    * @returns {Promise<string>} - Extracted text
    */
   async extractTextFromPDF(filePath) {
     try {
-      const buffer = fs.readFileSync(filePath);
-      const text = await extractor.extractText({
-        input: buffer,
-        type: 'file',
-      });
-      return text;
+      const dataBuffer = fs.readFileSync(filePath);
+      // Convert Buffer to Uint8Array as required by pdf-parse
+      const uint8Array = new Uint8Array(dataBuffer);
+      const pdfParse = new PDFParse(uint8Array);
+      const result = await pdfParse.getText();
+      return result.text;
     } catch (error) {
       console.error(
         `Error extracting text from PDF ${filePath}:`,

package/src/index.js

@@ -1,6 +1,10 @@
 #!/usr/bin/env node
 import { Command } from 'commander';
 
+import identifyCommand from './commands/IdentifyCommand.js';
+import PropagateCommand from './commands/PropagateCommand.js';
+import PushCommand from './commands/PushCommand.js';
+import scanCommand from './commands/ScanCommand.js';
 import UploadCommand from './commands/UploadCommand.js';
 import watchCommand from './commands/WatchCommand.js';
 import appConfig from './config/config.js';
@@ -15,6 +19,8 @@ class ArelaUploaderCLI {
   constructor() {
     this.program = new Command();
     this.errorHandler = new ErrorHandler(logger);
+    this.identifyCommand = identifyCommand;
+    this.scanCommand = scanCommand;
     this.uploadCommand = new UploadCommand();
     this.watchCommand = watchCommand;
 
@@ -164,6 +170,33 @@ class ArelaUploaderCLI {
         }
       });
 
+    // Scan command (optimized stats collection with streaming)
+    this.program
+      .command('scan')
+      .description(
+        'Scan filesystem and collect file statistics (optimized with streaming)',
+      )
+      .option(
+        '--api <target>',
+        'API target: agencia|cliente|default',
+        'default',
+      )
+      .option(
+        '--count-first',
+        'Count files first for percentage-based progress (slower start)',
+      )
+      .action(async (options) => {
+        try {
+          // Set API target if specified
+          if (options.api && options.api !== 'default') {
+            appConfig.setApiTarget(options.api);
+          }
+          await this.scanCommand.execute(options);
+        } catch (error) {
+          this.errorHandler.handleFatalError(error, { command: 'scan' });
+        }
+      });
+
     // Detection command
     this.program
       .command('detect')
@@ -277,6 +310,123 @@ class ArelaUploaderCLI {
         }
       });
 
+    // ============================================================================
+    // NEW SIMPLIFIED COMMANDS (Optimized versions with better naming)
+    // ============================================================================
+
+    // Identify command - simplified version of "detect --detect-pdfs"
+    this.program
+      .command('identify')
+      .description('🔍 Identify document types using matchers (optimized)')
+      .option(
+        '--api <target>',
+        'API target: agencia|cliente|default',
+        'default',
+      )
+      .option(
+        '-b, --batch-size <size>',
+        'Number of files to process in each batch',
+        '100',
+      )
+      .option('--show-stats', 'Show performance statistics')
+      .action(async (options) => {
+        try {
+          await this.identifyCommand.execute(options);
+        } catch (error) {
+          this.errorHandler.handleFatalError(error, { command: 'identify' });
+        }
+      });
+
+    // Propagate command - simplified version of "detect --propagate-arela-path"
+    this.program
+      .command('propagate')
+      .description(
+        '🔄 Propagate arela_path from pedimentos to related files (optimized)',
+      )
+      .option(
+        '--api <target>',
+        'API target: agencia|cliente|default',
+        'default',
+      )
+      .option(
+        '-b, --batch-size <size>',
+        'Number of pedimentos to process per batch',
+        '50',
+      )
+      .option('--show-stats', 'Show performance statistics')
+      .action(async (options) => {
+        try {
+          const propagateCommand = new PropagateCommand(options);
+          await propagateCommand.execute();
+        } catch (error) {
+          this.errorHandler.handleFatalError(error, { command: 'propagate' });
+        }
+      });
+
+    // Push command - simplified version of "upload --upload-by-rfc"
+    this.program
+      .command('push')
+      .description('📤 Upload files by RFC to Arela API (optimized)')
+      .option(
+        '--api <target>',
+        'API target for scan operations: default|agencia|cliente',
+        'default',
+      )
+      .option(
+        '--scan-api <target>',
+        'API for reading scan table: default|agencia|cliente',
+        'default',
+      )
+      .option(
+        '--push-api <target>',
+        'API for uploading files: default|agencia|cliente',
+      )
+      .option(
+        '-b, --batch-size <size>',
+        'Number of files to fetch per batch',
+        '100',
+      )
+      .option(
+        '--upload-batch-size <size>',
+        'Number of files to upload concurrently',
+        '10',
+      )
+      .option(
+        '--rfcs <rfcs>',
+        'Comma-separated RFCs to filter (overrides PUSH_RFCS env var)',
+      )
+      .option(
+        '--years <years>',
+        'Comma-separated years to filter (overrides PUSH_YEARS env var)',
+      )
+      .option('--show-stats', 'Show performance statistics')
+      .action(async (options) => {
+        try {
+          // Parse comma-separated values
+          if (options.rfcs) {
+            options.rfcs = options.rfcs
+              .split(',')
+              .map((r) => r.trim().toUpperCase())
+              .filter(Boolean);
+          }
+          if (options.years) {
+            options.years = options.years
+              .split(',')
+              .map((y) => parseInt(y.trim(), 10))
+              .filter((y) => !isNaN(y));
+          }
+
+          const pushCommand = new PushCommand();
+          await pushCommand.execute(options);
+        } catch (error) {
+          this.errorHandler.handleFatalError(error, { command: 'push' });
+        }
+      });
+
+    // ============================================================================
+    // END OF NEW SIMPLIFIED COMMANDS
+    // ============================================================================
+
     // Watch command
     this.program
       .command('watch')

package/src/services/DatabaseService.js

@@ -498,8 +498,8 @@ export class DatabaseService {
     );
 
     const processingBatchSize = parseInt(options.batchSize) || 10;
-    // Reduced query batch size to avoid timeouts
-    const queryBatchSize = 100; // Reduced from 500 to 100
+    // Query batch size for each API call
+    const queryBatchSize = 100;
 
     let totalDetected = 0;
     let totalProcessed = 0;