@arela/uploader 0.2.4 → 0.2.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "0.2.4",
+  "version": "0.2.5",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"
@@ -33,6 +33,7 @@
     "commander": "13.1.0",
     "dotenv": "16.5.0",
     "form-data": "4.0.4",
+    "formdata-node": "^6.0.3",
     "globby": "14.1.0",
     "mime-types": "3.0.1",
     "node-fetch": "3.3.2",

package/src/commands/UploadCommand.js

@@ -0,0 +1,388 @@
+import cliProgress from 'cli-progress';
+import { globby } from 'globby';
+import mime from 'mime-types';
+import path from 'path';
+
+import appConfig from '../config/config.js';
+import ErrorHandler from '../errors/ErrorHandler.js';
+import { ConfigurationError, FileOperationError } from '../errors/ErrorTypes.js';
+import databaseService from '../services/DatabaseService.js';
+import logger from '../services/LoggingService.js';
+import uploadServiceFactory from '../services/upload/UploadServiceFactory.js';
+import FileOperations from '../utils/FileOperations.js';
+import fileSanitizer from '../utils/FileSanitizer.js';
+import pathDetector from '../utils/PathDetector.js';
+
+/**
+ * Upload Command Handler
+ * Handles the main upload functionality
+ */
+export class UploadCommand {
+  constructor() {
+    this.errorHandler = new ErrorHandler(logger);
+  }
+
+  /**
+   * Execute the upload command
+   * @param {Object} options - Command options
+   */
+  async execute(options) {
+    try {
+      // Validate configuration
+      this.#validateOptions(options);
+      
+      // Initialize services
+      const uploadService = await uploadServiceFactory.getUploadService(options.forceSupabase);
+      const sources = appConfig.getUploadSources();
+      const basePath = appConfig.getBasePath();
+
+      // Log command start
+      logger.info(`Starting upload with ${uploadService.getServiceName()}`);
+      
+      if (options.clearLog) {
+        logger.clearLogFile();
+        logger.info('Log file cleared');
+      }
+
+      // Process each source
+      let globalResults = {
+        successCount: 0,
+        detectedCount: 0,
+        organizedCount: 0,
+        failureCount: 0,
+        skippedCount: 0,
+      };
+
+      for (const source of sources) {
+        const sourcePath = path.resolve(basePath, source).replace(/\\/g, '/');
+        logger.info(`Processing folder: ${sourcePath}`);
+
+        try {
+          const files = await this.#discoverFiles(sourcePath);
+          logger.info(`Found ${files.length} files to process`);
+
+          const result = await this.#processFilesInBatches(
+            files,
+            options,
+            uploadService,
+            basePath,
+            source,
+            sourcePath
+          );
+
+          this.#updateGlobalResults(globalResults, result);
+          this.#logSourceSummary(source, result, options);
+        } catch (error) {
+          this.errorHandler.handleError(error, { source, sourcePath });
+          globalResults.failureCount++;
+        }
+      }
+
+      this.#logFinalSummary(globalResults, options, uploadService);
+      
+      // Handle additional phases if requested
+      if (options.runAllPhases && options.statsOnly) {
+        await this.#runAdditionalPhases(options);
+      }
+
+    } catch (error) {
+      this.errorHandler.handleFatalError(error, { command: 'upload', options });
+    }
+  }
+
+  /**
+   * Validate command options
+   * @private
+   * @param {Object} options - Options to validate
+   */
+  #validateOptions(options) {
+    try {
+      appConfig.validateConfiguration(options.forceSupabase);
+    } catch (error) {
+      throw new ConfigurationError(error.message);
+    }
+
+    if (options.batchSize && (options.batchSize < 1 || options.batchSize > 100)) {
+      throw new ConfigurationError('Batch size must be between 1 and 100');
+    }
+  }
+
+  /**
+   * Discover files in a source path
+   * @private
+   * @param {string} sourcePath - Path to discover files in
+   * @returns {Promise<string[]>} Array of file paths
+   */
+  async #discoverFiles(sourcePath) {
+    try {
+      if (!FileOperations.fileExists(sourcePath)) {
+        throw new FileOperationError(`Source path does not exist: ${sourcePath}`);
+      }
+
+      const stats = FileOperations.getFileStats(sourcePath);
+      
+      if (stats?.isDirectory()) {
+        return await globby([`${sourcePath}/**/*`], { onlyFiles: true });
+      } else {
+        return [sourcePath];
+      }
+    } catch (error) {
+      throw new FileOperationError(`Failed to discover files in ${sourcePath}`, sourcePath, { originalError: error.message });
+    }
+  }
+
+  /**
+   * Process files in batches
+   * @private
+   * @param {string[]} files - Files to process
+   * @param {Object} options - Processing options
+   * @param {Object} uploadService - Upload service instance
+   * @param {string} basePath - Base path
+   * @param {string} source - Source name
+   * @param {string} sourcePath - Source path
+   * @returns {Promise<Object>} Processing results
+   */
+  async #processFilesInBatches(files, options, uploadService, basePath, source, sourcePath) {
+    const batchSize = parseInt(options.batchSize) || 10;
+    const results = {
+      successCount: 0,
+      detectedCount: 0,
+      organizedCount: 0,
+      failureCount: 0,
+      skippedCount: 0,
+    };
+
+    // Get processed paths if available
+    const processedPaths = options.skipProcessed ? databaseService.getProcessedPaths() : new Set();
+
+    // Create progress bar
+    const progressBar = new cliProgress.SingleBar({
+      format: `📤 ${source} |{bar}| {percentage}% | {value}/{total} | Success: {success} | Errors: {errors}`,
+      barCompleteChar: '█',
+      barIncompleteChar: '░',
+      hideCursor: true,
+    });
+
+    progressBar.start(files.length, 0, { success: 0, errors: 0 });
+
+    // Process files in batches
+    for (let i = 0; i < files.length; i += batchSize) {
+      const batch = files.slice(i, i + batchSize);
+      
+      try {
+        const batchResult = await this.#processBatch(
+          batch, 
+          options, 
+          uploadService, 
+          basePath, 
+          processedPaths
+        );
+        
+        this.#updateResults(results, batchResult);
+        
+        progressBar.update(Math.min(i + batchSize, files.length), { 
+          success: results.successCount, 
+          errors: results.failureCount 
+        });
+
+        // Delay between batches if configured
+        if (appConfig.performance.batchDelay > 0) {
+          await new Promise(resolve => setTimeout(resolve, appConfig.performance.batchDelay));
+        }
+      } catch (error) {
+        this.errorHandler.handleError(error, { batch: i / batchSize + 1, batchSize });
+        results.failureCount += batch.length;
+      }
+    }
+
+    progressBar.stop();
+    return results;
+  }
+
+  /**
+   * Process a batch of files
+   * @private
+   * @param {string[]} batch - Files in this batch
+   * @param {Object} options - Processing options
+   * @param {Object} uploadService - Upload service
+   * @param {string} basePath - Base path
+   * @param {Set} processedPaths - Already processed paths
+   * @returns {Promise<Object>} Batch results
+   */
+  async #processBatch(batch, options, uploadService, basePath, processedPaths) {
+    const batchResults = {
+      successCount: 0,
+      detectedCount: 0,
+      organizedCount: 0,
+      failureCount: 0,
+      skippedCount: 0,
+    };
+
+    if (options.statsOnly) {
+      // Stats-only mode: just record file information
+      const fileObjects = batch.map(filePath => ({
+        path: filePath,
+        originalName: path.basename(filePath),
+        stats: FileOperations.getFileStats(filePath),
+      }));
+
+      try {
+        const result = await databaseService.insertStatsOnlyToUploaderTable(fileObjects, options);
+        batchResults.successCount = result.totalInserted;
+        batchResults.skippedCount = result.totalSkipped;
+      } catch (error) {
+        throw new Error(`Failed to insert stats: ${error.message}`);
+      }
+    } else {
+      // Upload mode: process files for upload
+      for (const filePath of batch) {
+        try {
+          await this.#processFile(filePath, options, uploadService, basePath, processedPaths, batchResults);
+        } catch (error) {
+          this.errorHandler.handleError(error, { filePath });
+          batchResults.failureCount++;
+        }
+      }
+    }
+
+    return batchResults;
+  }
+
+  /**
+   * Process a single file
+   * @private
+   */
+  async #processFile(filePath, options, uploadService, basePath, processedPaths, batchResults) {
+    // Skip if already processed
+    if (processedPaths.has(filePath)) {
+      batchResults.skippedCount++;
+      return;
+    }
+
+    // Prepare file for upload
+    const sanitizedName = fileSanitizer.sanitizeFileName(path.basename(filePath));
+    const pathInfo = pathDetector.extractYearAndPedimentoFromPath(filePath, basePath);
+    
+    let uploadPath = sanitizedName;
+    if (pathInfo.detected && options.autoDetectStructure) {
+      uploadPath = `${pathInfo.year}/${pathInfo.pedimento}/${sanitizedName}`;
+    }
+
+    const fileObject = {
+      path: filePath,
+      name: sanitizedName,
+      contentType: this.#getMimeType(filePath),
+    };
+
+    // Upload based on service type
+    if (uploadService.getServiceName() === 'Arela API') {
+      const result = await uploadService.upload([fileObject], {
+        ...options,
+        uploadPath,
+      });
+      
+      batchResults.successCount++;
+      if (result.detectedCount) batchResults.detectedCount += result.detectedCount;
+      if (result.organizedCount) batchResults.organizedCount += result.organizedCount;
+    } else {
+      // Supabase direct upload
+      await uploadService.upload([fileObject], { uploadPath });
+      batchResults.successCount++;
+    }
+
+    logger.info(`SUCCESS: ${path.basename(filePath)} -> ${uploadPath}`);
+  }
+
+  /**
+   * Get MIME type for file
+   * @private
+   */
+  #getMimeType(filePath) {
+    return mime.lookup(filePath) || 'application/octet-stream';
+  }
+
+  /**
+   * Update results object
+   * @private
+   */
+  #updateResults(target, source) {
+    target.successCount += source.successCount;
+    target.detectedCount += source.detectedCount;
+    target.organizedCount += source.organizedCount;
+    target.failureCount += source.failureCount;
+    target.skippedCount += source.skippedCount;
+  }
+
+  /**
+   * Update global results
+   * @private
+   */
+  #updateGlobalResults(global, source) {
+    this.#updateResults(global, source);
+  }
+
+  /**
+   * Log source summary
+   * @private
+   */
+  #logSourceSummary(source, result, options) {
+    console.log(`\n📦 Summary for ${source}:`);
+    if (options.statsOnly) {
+      console.log(`   📊 Stats recorded: ${result.successCount}`);
+      console.log(`   ⏭️  Duplicates: ${result.skippedCount}`);
+    } else {
+      console.log(`   ✅ Uploaded: ${result.successCount}`);
+      if (result.detectedCount) console.log(`   🔍 Detected: ${result.detectedCount}`);
+      if (result.organizedCount) console.log(`   📁 Organized: ${result.organizedCount}`);
+      console.log(`   ⏭️  Skipped: ${result.skippedCount}`);
+    }
+    console.log(`   ❌ Errors: ${result.failureCount}`);
+  }
+
+  /**
+   * Log final summary
+   * @private
+   */
+  #logFinalSummary(results, options, uploadService) {
+    console.log(`\n${'='.repeat(60)}`);
+    if (options.statsOnly) {
+      console.log(`📊 STATS COLLECTION COMPLETED`);
+      console.log(`   📊 Total stats recorded: ${results.successCount}`);
+      console.log(`   ⏭️  Total duplicates: ${results.skippedCount}`);
+    } else {
+      console.log(`🎯 ${uploadService.getServiceName().toUpperCase()} UPLOAD COMPLETED`);
+      console.log(`   ✅ Total uploaded: ${results.successCount}`);
+      if (results.detectedCount) console.log(`   🔍 Total detected: ${results.detectedCount}`);
+      if (results.organizedCount) console.log(`   📁 Total organized: ${results.organizedCount}`);
+      console.log(`   ⏭️  Total skipped: ${results.skippedCount}`);
+    }
+    console.log(`   ❌ Total errors: ${results.failureCount}`);
+    console.log(`   📜 Log file: ${logger.getLogFilePath()}`);
+    console.log(`${'='.repeat(60)}\n`);
+  }
+
+  /**
+   * Run additional phases
+   * @private
+   */
+  async #runAdditionalPhases(options) {
+    try {
+      // Phase 2: PDF Detection
+      console.log('\n🔍 === PHASE 2: PDF Detection ===');
+      const detectionResult = await databaseService.detectPedimentosInDatabase({
+        batchSize: parseInt(options.batchSize) || 10,
+      });
+      console.log(
+        `✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`,
+      );
+
+      // Additional phases would be implemented here
+      console.log('\n🎉 All phases completed successfully!');
+    } catch (error) {
+      this.errorHandler.handleError(error, { phase: 'additional-phases' });
+      throw error;
+    }
+  }
+}
+
+export default UploadCommand;

package/src/config/config.js

@@ -0,0 +1,173 @@
+import { config } from 'dotenv';
+import fs from 'fs';
+import path from 'path';
+
+config();
+
+/**
+ * Configuration management for Arela Uploader
+ * Centralizes all environment variable handling and validation
+ */
+class Config {
+  constructor() {
+    this.packageVersion = this.#loadPackageVersion();
+    this.supabase = this.#loadSupabaseConfig();
+    this.api = this.#loadApiConfig();
+    this.upload = this.#loadUploadConfig();
+    this.performance = this.#loadPerformanceConfig();
+    this.logging = this.#loadLoggingConfig();
+  }
+
+  /**
+   * Load package version from package.json
+   * @private
+   */
+  #loadPackageVersion() {
+    try {
+      const __filename = new URL(import.meta.url).pathname;
+      const __dirname = path.dirname(__filename);
+      const packageJsonPath = path.resolve(__dirname, '../../package.json');
+      const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
+      return packageJson.version || '1.0.0';
+    } catch (error) {
+      console.warn('⚠️ Could not read package.json version, using fallback');
+      return '0.2.4';
+    }
+  }
+
+  /**
+   * Load Supabase configuration
+   * @private
+   */
+  #loadSupabaseConfig() {
+    return {
+      url: process.env.SUPABASE_URL,
+      key: process.env.SUPABASE_KEY,
+      bucket: process.env.SUPABASE_BUCKET,
+    };
+  }
+
+  /**
+   * Load API configuration
+   * @private
+   */
+  #loadApiConfig() {
+    return {
+      baseUrl: process.env.ARELA_API_URL,
+      token: process.env.ARELA_API_TOKEN,
+    };
+  }
+
+  /**
+   * Load upload configuration
+   * @private
+   */
+  #loadUploadConfig() {
+    const basePath = process.env.UPLOAD_BASE_PATH;
+    const sources = process.env.UPLOAD_SOURCES?.split('|')
+      .map((s) => s.trim())
+      .filter(Boolean);
+    
+    const uploadRfcs = process.env.UPLOAD_RFCS?.split('|')
+      .map((s) => s.trim())
+      .filter(Boolean);
+
+    return {
+      basePath,
+      sources,
+      rfcs: uploadRfcs,
+    };
+  }
+
+  /**
+   * Load performance configuration
+   * @private
+   */
+  #loadPerformanceConfig() {
+    return {
+      batchDelay: parseInt(process.env.BATCH_DELAY) || 100,
+      progressUpdateInterval: parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10,
+      logBufferSize: 100,
+      logFlushInterval: 5000,
+    };
+  }
+
+  /**
+   * Load logging configuration
+   * @private
+   */
+  #loadLoggingConfig() {
+    return {
+      verbose: process.env.VERBOSE_LOGGING === 'true',
+      logFilePath: path.resolve(process.cwd(), 'arela-upload.log'),
+    };
+  }
+
+  /**
+   * Check if API mode is available
+   * @returns {boolean}
+   */
+  isApiModeAvailable() {
+    return !!(this.api.baseUrl && this.api.token);
+  }
+
+  /**
+   * Check if Supabase mode is available
+   * @returns {boolean}
+   */
+  isSupabaseModeAvailable() {
+    return !!(this.supabase.url && this.supabase.key && this.supabase.bucket);
+  }
+
+  /**
+   * Validate configuration for the requested mode
+   * @param {boolean} forceSupabase - Whether to force Supabase mode
+   * @throws {Error} If required configuration is missing
+   */
+  validateConfiguration(forceSupabase = false) {
+    if (forceSupabase) {
+      if (!this.isSupabaseModeAvailable()) {
+        throw new Error(
+          '⚠️ Missing Supabase credentials. Please set SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET'
+        );
+      }
+      return;
+    }
+
+    if (!this.isApiModeAvailable() && !this.isSupabaseModeAvailable()) {
+      throw new Error(
+        '⚠️ Missing credentials. Please set either:\n' +
+        '   - ARELA_API_URL and ARELA_API_TOKEN for API mode, or\n' +
+        '   - SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET for direct mode'
+      );
+    }
+  }
+
+  /**
+   * Get upload sources with validation
+   * @returns {string[]} Array of upload sources
+   * @throws {Error} If sources are not configured
+   */
+  getUploadSources() {
+    if (!this.upload.sources || this.upload.sources.length === 0) {
+      throw new Error('⚠️ No upload sources configured. Please set UPLOAD_SOURCES environment variable.');
+    }
+    return this.upload.sources;
+  }
+
+  /**
+   * Get base path with validation
+   * @returns {string} Base path for uploads
+   * @throws {Error} If base path is not configured
+   */
+  getBasePath() {
+    if (!this.upload.basePath) {
+      throw new Error('⚠️ No base path configured. Please set UPLOAD_BASE_PATH environment variable.');
+    }
+    return this.upload.basePath;
+  }
+}
+
+// Export singleton instance
+export const appConfig = new Config();
+export default appConfig;