@arela/uploader 0.0.12 → 0.2.0

package/src/index.js

@@ -3,32 +3,55 @@ import { createClient } from '@supabase/supabase-js';
 import cliProgress from 'cli-progress';
 import { Command } from 'commander';
 import { config } from 'dotenv';
+import FormData from 'form-data';
 import fs from 'fs';
 import { globby } from 'globby';
 import mime from 'mime-types';
-import { createRequire } from 'module';
+import fetch from 'node-fetch';
 import path from 'path';
-
-const require = createRequire(import.meta.url);
-const { version } = require('../package.json');
+import { FileDetectionService } from './file-detection.js';
 
 config();
 
 const program = new Command();
 
+// Read package.json version at startup
+let packageVersion = '1.0.0'; // fallback
+try {
+  const __filename = new URL(import.meta.url).pathname;
+  const __dirname = path.dirname(__filename);
+  const packageJsonPath = path.resolve(__dirname, '../package.json');
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
+  packageVersion = packageJson.version || '1.0.0';
+} catch (error) {
+  console.warn('⚠️ Could not read package.json version, using fallback');
+}
+
+// Configuración de Supabase (original)
 const supabaseUrl = process.env.SUPABASE_URL;
 const supabaseKey = process.env.SUPABASE_KEY;
 const bucket = process.env.SUPABASE_BUCKET;
+
+// Configuración de API (nueva)
+const API_BASE_URL = process.env.ARELA_API_URL;
+const API_TOKEN = process.env.ARELA_API_TOKEN;
+
+// Configuración del uploader mejorado
 const basePath = process.env.UPLOAD_BASE_PATH;
 const sources = process.env.UPLOAD_SOURCES?.split('|')
   .map((s) => s.trim())
   .filter(Boolean);
 
-const supabase = createClient(supabaseUrl, supabaseKey);
+// Configuración de RFCs para upload
+const uploadRfcs = process.env.UPLOAD_RFCS?.split('|')
+  .map((s) => s.trim())
+  .filter(Boolean);
 
-// Pre-compiled regex patterns for better performance
+let supabase;
+let apiMode = false;
+
+// Pre-compiled regex patterns for better performance (from original complex uploader)
 const SANITIZATION_PATTERNS = [
-  // Character replacements (grouped for efficiency)
   [/[áàâäãåāăą]/gi, 'a'],
   [/[éèêëēĕėę]/gi, 'e'],
   [/[íìîïīĭį]/gi, 'i'],
@@ -37,730 +60,1425 @@ const SANITIZATION_PATTERNS = [
   [/[ñň]/gi, 'n'],
   [/[ç]/gi, 'c'],
   [/[ý]/gi, 'y'],
-  // Korean characters (compiled once)
   [/[멕]/g, 'meok'],
   [/[시]/g, 'si'],
   [/[코]/g, 'ko'],
   [/[용]/g, 'yong'],
   [/[가-힣]/g, 'kr'],
-  // Unicode diacritics (after normalize)
   [/[\u0300-\u036f]/g, ''],
-  // Problematic symbols
   [/[\\?%*:|"<>[\]~`^]/g, '-'],
   [/[{}]/g, '-'],
   [/[&]/g, 'and'],
-  [/[()]/g, ''], // Remove parentheses
-  // Cleanup patterns
-  [/\s+/g, '-'], // Replace spaces with dashes
-  [/-+/g, '-'], // Replace multiple dashes with single dash
-  [/^-+|-+$/g, ''], // Remove leading/trailing dashes
-  [/^\.+/, ''], // Remove leading dots
-  [/[^\w.-]/g, ''], // Remove any remaining non-alphanumeric chars
+  [/[()]/g, ''],
+  [/\s+/g, '-'],
+  [/-+/g, '-'],
+  [/^-+|-+$/g, ''],
+  [/^\.+/, ''],
+  [/[^\w.-]/g, ''],
 ];
 
-// Cache for sanitized filenames to avoid repeated processing
 const sanitizationCache = new Map();
 
-// Enhanced sanitization function with caching and pre-compiled regex
 const sanitizeFileName = (fileName) => {
-  // Check cache first
   if (sanitizationCache.has(fileName)) {
     return sanitizationCache.get(fileName);
   }
 
-  // Get file extension
   const ext = path.extname(fileName);
   const nameWithoutExt = path.basename(fileName, ext);
 
-  // Fast path for already clean filenames
   if (/^[a-zA-Z0-9._-]+$/.test(nameWithoutExt)) {
     const result = fileName;
     sanitizationCache.set(fileName, result);
     return result;
   }
 
-  // Normalize unicode first (more efficient to do once)
   let sanitized = nameWithoutExt.normalize('NFD');
 
-  // Apply all sanitization patterns
   for (const [pattern, replacement] of SANITIZATION_PATTERNS) {
     sanitized = sanitized.replace(pattern, replacement);
   }
 
-  // Ensure the filename is not empty
+  // Additional sanitization for problematic characters
+  sanitized = sanitized
+    .replace(/~/g, '-') // Replace tildes
+    .replace(/\s+/g, '-') // Replace spaces with dashes
+    .replace(/\.+/g, '-') // Replace multiple dots with dashes
+    .replace(/-+/g, '-') // Collapse multiple dashes
+    .replace(/^-+|-+$/g, ''); // Remove leading/trailing dashes
+
   if (!sanitized) {
     sanitized = 'unnamed_file';
   }
 
   const result = sanitized + ext;
-
-  // Cache the result for future use
   sanitizationCache.set(fileName, result);
-
   return result;
 };
 
-// Pre-compiled regex patterns for path sanitization
-const PATH_SANITIZATION_PATTERNS = [
-  [/[\\?%*:|"<>[\]~]/g, '-'],
-  [/ +/g, ' '],
-  [/^\.+/, ''],
-  [/\/+/g, '/'],
-];
+const checkCredentials = async (forceSupabase = false) => {
+  // Force Supabase mode if explicitly requested
+  if (forceSupabase) {
+    console.log('🔧 Force Supabase mode enabled - skipping API');
+    apiMode = false;
+  } else if (API_BASE_URL && API_TOKEN) {
+    console.log(
+      '🌐 API mode enabled - files will be uploaded to Arela API with automatic processing',
+    );
+    apiMode = true;
 
-// Cache for sanitized paths
-const pathSanitizationCache = new Map();
-
-// Batch logging system for performance
-class LogBatcher {
-  constructor(batchSize = 50, flushInterval = 5000) {
-    this.batch = [];
-    this.batchSize = batchSize;
-    this.flushInterval = flushInterval;
-    this.lastFlush = Date.now();
-    this.flushTimer = null;
-  }
-
-  add(logEntry) {
-    this.batch.push({
-      filename: path.basename(logEntry.file),
-      path: logEntry.uploadPath,
-      status: logEntry.status,
-      message: logEntry.message,
-    });
+    try {
+      const response = await fetch(`${API_BASE_URL}/api/health`, {
+        headers: {
+          'x-api-key': API_TOKEN,
+        },
+      });
 
-    // Auto-flush if batch is full or enough time has passed
-    if (
-      this.batch.length >= this.batchSize ||
-      Date.now() - this.lastFlush > this.flushInterval
-    ) {
-      this.flush();
+      if (!response.ok) {
+        console.warn(
+          '⚠️ API connection failed, falling back to direct Supabase upload',
+        );
+        apiMode = false;
+      } else {
+        console.log('✅ Connected to Arela API');
+        return;
+      }
+    } catch (err) {
+      console.warn(
+        '⚠️ API connection failed, falling back to direct Supabase upload',
+      );
+      apiMode = false;
     }
   }
 
-  async flush() {
-    if (this.batch.length === 0) return;
-
-    const logsToSend = [...this.batch];
-    this.batch = [];
-    this.lastFlush = Date.now();
-
-    // Clear any pending timer
-    if (this.flushTimer) {
-      clearTimeout(this.flushTimer);
-      this.flushTimer = null;
+  // Initialize Supabase client if not in API mode or if forced
+  if (!apiMode || forceSupabase) {
+    if (!supabaseUrl || !supabaseKey || !bucket) {
+      console.error(
+        '⚠️ Missing credentials. Please set either:\n' +
+          '   - ARELA_API_URL and ARELA_API_TOKEN for API mode, or\n' +
+          '   - SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET for direct mode',
+      );
+      process.exit(1);
     }
 
+    supabase = createClient(supabaseUrl, supabaseKey);
+
     try {
-      const { error } = await supabase.from('upload_logs').insert(logsToSend);
+      const { error } = await supabase.storage.from(bucket).list('');
       if (error) {
-        console.error(
-          `⚠️ Error saving batch of ${logsToSend.length} logs to Supabase: ${error.message}`,
-        );
-        // Re-add failed logs to batch for retry (optional)
-        this.batch.unshift(...logsToSend);
-      } else {
-        // Only show verbose output if requested
-        if (process.env.LOG_BATCH_VERBOSE === 'true') {
-          console.log(`📊 Flushed ${logsToSend.length} logs to Supabase`);
-        }
+        console.error('⚠️ Error connecting to Supabase:', error.message);
+        process.exit(1);
       }
+      console.log('✅ Connected to Supabase (direct mode)');
     } catch (err) {
-      console.error(`⚠️ Error during batch flush: ${err.message}`);
-      // Re-add failed logs to batch for retry (optional)
-      this.batch.unshift(...logsToSend);
+      console.error('⚠️ Error:', err.message);
+      process.exit(1);
     }
   }
+};
 
-  // Schedule auto-flush if not already scheduled
-  scheduleFlush() {
-    if (!this.flushTimer && this.batch.length > 0) {
-      this.flushTimer = setTimeout(() => {
-        this.flush();
-      }, this.flushInterval);
-    }
+const logFilePath = path.resolve(process.cwd(), 'arela-upload.log');
+const writeLog = (message) => {
+  try {
+    const timestamp = new Date().toISOString();
+    fs.appendFileSync(logFilePath, `[${timestamp}] ${message}\n`);
+  } catch (error) {
+    console.error(`❌ Error writing to log file: ${error.code} | ${error.message} | path: ${logFilePath}`);
   }
+};
 
-  // Force flush all pending logs (called at end of process)
-  async forceFlush() {
-    if (this.flushTimer) {
-      clearTimeout(this.flushTimer);
-      this.flushTimer = null;
+/**
+ * Extracts year and pedimento number from file path
+ * Supports patterns like:
+ * - /path/to/2024/4023260/file.pdf
+ * - /path/to/pedimentos/2024/4023260/file.pdf
+ * - /path/to/docs/año2024/ped4023260/file.pdf
+ */
+const extractYearAndPedimentoFromPath = (filePath, basePath) => {
+  try {
+    const relativePath = path.relative(basePath, filePath);
+    const pathParts = relativePath.split(path.sep);
+
+    let year = null;
+    let pedimento = null;
+
+    // Pattern 1: Direct year/pedimento structure (2024/4023260)
+    for (let i = 0; i < pathParts.length - 1; i++) {
+      const part = pathParts[i];
+      const nextPart = pathParts[i + 1];
+
+      // Check if current part looks like a year (2020-2030)
+      const yearMatch = part.match(/^(202[0-9])$/);
+      if (yearMatch && nextPart) {
+        year = yearMatch[1];
+
+        // Check if next part looks like a pedimento (4-8 digits)
+        const pedimentoMatch = nextPart.match(/^(\d{4,8})$/);
+        if (pedimentoMatch) {
+          pedimento = pedimentoMatch[1];
+          break;
+        }
+      }
     }
-    await this.flush();
-  }
-}
 
-// Global log batcher instance
-const logBatcher = new LogBatcher();
+    // Pattern 2: Named patterns (año2024, ped4023260)
+    if (!year || !pedimento) {
+      for (const part of pathParts) {
+        if (!year) {
+          const namedYearMatch = part.match(/(?:año|year|anio)(\d{4})/i);
+          if (namedYearMatch) {
+            year = namedYearMatch[1];
+          }
+        }
 
-// Function to manage cache size (prevent memory issues in long sessions)
-const manageCaches = () => {
-  const MAX_CACHE_SIZE = 1000;
+        if (!pedimento) {
+          const namedPedimentoMatch = part.match(
+            /(?:ped|pedimento|pedi)(\d{4,8})/i,
+          );
+          if (namedPedimentoMatch) {
+            pedimento = namedPedimentoMatch[1];
+          }
+        }
+      }
+    }
 
-  if (sanitizationCache.size > MAX_CACHE_SIZE) {
-    // Keep only the most recent 500 entries
-    const entries = Array.from(sanitizationCache.entries());
-    sanitizationCache.clear();
-    entries.slice(-500).forEach(([key, value]) => {
-      sanitizationCache.set(key, value);
-    });
-  }
+    // Pattern 3: Loose year detection in any part
+    if (!year) {
+      for (const part of pathParts) {
+        const yearMatch = part.match(/(202[0-9])/);
+        if (yearMatch) {
+          year = yearMatch[1];
+          break;
+        }
+      }
+    }
 
-  if (pathSanitizationCache.size > MAX_CACHE_SIZE) {
-    const entries = Array.from(pathSanitizationCache.entries());
-    pathSanitizationCache.clear();
-    entries.slice(-500).forEach(([key, value]) => {
-      pathSanitizationCache.set(key, value);
-    });
+    // Pattern 4: Loose pedimento detection (4-8 consecutive digits)
+    if (!pedimento) {
+      for (const part of pathParts) {
+        const pedimentoMatch = part.match(/(\d{4,8})/);
+        if (pedimentoMatch && pedimentoMatch[1].length >= 4) {
+          pedimento = pedimentoMatch[1];
+          break;
+        }
+      }
+    }
+
+    return { year, pedimento, detected: !!(year && pedimento) };
+  } catch (error) {
+    return {
+      year: null,
+      pedimento: null,
+      detected: false,
+      error: error.message,
+    };
   }
 };
 
-const sanitizePath = (inputPath) => {
-  // Check cache first
-  if (pathSanitizationCache.has(inputPath)) {
-    return pathSanitizationCache.get(inputPath);
+const getProcessedPaths = () => {
+  const processed = new Set();
+  const lines = fs.existsSync(logFilePath)
+    ? fs.readFileSync(logFilePath, 'utf-8').split('\n')
+    : [];
+
+  for (const line of lines) {
+    const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
+    if (match) {
+      const [, , path] = match;
+      if (path) {
+        processed.add(path.trim());
+      }
+    }
   }
 
-  // Fast path for already clean paths
-  if (!/[\\?%*:|"<>[\]~]|^ +|^\.+|\/\/+/.test(inputPath)) {
-    pathSanitizationCache.set(inputPath, inputPath);
-    return inputPath;
-  }
+  return processed;
+};
 
-  let sanitized = inputPath;
+/**
+ * Upload files to Arela API with automatic detection and organization
+ */
+const uploadToApi = async (files, options) => {
+  const formData = new FormData();
+
+  files.forEach((file) => {
+    const fileBuffer = fs.readFileSync(file.path);
+    formData.append('files', fileBuffer, {
+      filename: file.name,
+      contentType: file.contentType,
+    });
+  });
 
-  // Apply path sanitization patterns
-  for (const [pattern, replacement] of PATH_SANITIZATION_PATTERNS) {
-    sanitized = sanitized.replace(pattern, replacement);
+  if (bucket) formData.append('bucket', bucket);
+  if (options.prefix) formData.append('prefix', options.prefix);
+
+  // Nueva funcionalidad: estructura de carpetas personalizada
+  let combinedStructure = null;
+
+  if (
+    options.folderStructure &&
+    options.autoDetectStructure &&
+    files.length > 0
+  ) {
+    // Combine custom folder structure with auto-detection
+    const firstFile = files[0];
+    const detection = extractYearAndPedimentoFromPath(
+      firstFile.path,
+      process.cwd(),
+    );
+    if (detection.detected) {
+      const autoStructure = `${detection.year}/${detection.pedimento}`;
+      combinedStructure = `${options.folderStructure}/${autoStructure}`;
+      formData.append('folderStructure', combinedStructure);
+      console.log(
+        `📁 Combined folder structure: ${options.folderStructure} + ${autoStructure} = ${combinedStructure}`,
+      );
+    } else {
+      // Fallback to just custom structure if auto-detection fails
+      formData.append('folderStructure', options.folderStructure);
+      console.log(
+        `📁 Using custom folder structure (auto-detection failed): ${options.folderStructure}`,
+      );
+    }
+  } else if (options.folderStructure) {
+    formData.append('folderStructure', options.folderStructure);
+    console.log(`📁 Using custom folder structure: ${options.folderStructure}`);
+  } else if (options.autoDetectStructure && files.length > 0) {
+    // Try to auto-detect from the first file if no explicit structure is provided
+    const firstFile = files[0];
+    const detection = extractYearAndPedimentoFromPath(
+      firstFile.path,
+      process.cwd(),
+    );
+    if (detection.detected) {
+      const autoStructure = `${detection.year}/${detection.pedimento}`;
+      formData.append('folderStructure', autoStructure);
+    }
   }
 
-  // Cache the result
-  pathSanitizationCache.set(inputPath, sanitized);
-
-  return sanitized;
-};
-
-const sendLogToSupabase = async ({ file, uploadPath, status, message }) => {
-  // Add to batch instead of sending immediately
-  logBatcher.add({ file, uploadPath, status, message });
+  // Si se especifica clientPath para user_metadata
+  if (options.clientPath) {
+    formData.append('clientPath', options.clientPath);
+  }
 
-  // Schedule auto-flush if needed
-  logBatcher.scheduleFlush();
-};
+  formData.append('autoDetect', String(options.autoDetect ?? true));
+  formData.append('autoOrganize', String(options.autoOrganize ?? true));
+  formData.append('batchSize', String(options.batchSize || 10));
+  formData.append('clientVersion', packageVersion);
 
-// Enhanced version for immediate sending (used for critical errors)
-const sendLogToSupabaseImmediate = async ({
-  file,
-  uploadPath,
-  status,
-  message,
-}) => {
-  const { error } = await supabase.from('upload_logs').insert([
+  const response = await fetch(
+    `${API_BASE_URL}/api/storage/batch-upload-and-process`,
     {
-      filename: path.basename(file),
-      path: uploadPath,
-      status,
-      message,
+      method: 'POST',
+      headers: {
+        'x-api-key': API_TOKEN,
+      },
+      body: formData,
     },
-  ]);
-
-  if (error) {
-    console.error(
-      `⚠️ Error saving immediate log to Supabase: ${error.message}`,
-    );
-  }
-};
+  );
 
-const checkCredentials = async () => {
-  if (!supabaseUrl || !supabaseKey || !bucket) {
-    console.error(
-      '⚠️ Missing Supabase credentials. Please set SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET in your environment variables.',
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(
+      `API request failed: ${response.status} ${response.statusText} - ${errorText}`,
     );
-    writeLog('⚠️ Missing Supabase credentials.');
-    await sendLogToSupabaseImmediate({
-      file: 'Error',
-      uploadPath: 'Error',
-      status: 'error',
-      message: 'Missing Supabase credentials.',
-    });
-    process.exit(1);
   }
 
-  try {
-    const { error } = await supabase.storage.from(bucket).list('');
-    if (error) {
-      console.error('⚠️ Error connecting to Supabase:', error.message);
-      writeLog(`⚠️ Error connecting to Supabase: ${error.message}`);
-      await sendLogToSupabaseImmediate({
-        file: 'Error',
-        uploadPath: 'Error',
-        status: 'error',
-        message: error.message,
-      });
-      process.exit(1);
-    }
-  } catch (err) {
-    console.error('⚠️ Error:', err.message);
-    writeLog(`⚠️ Error: ${err.message}`);
-    await sendLogToSupabaseImmediate({
-      file: 'Error',
-      uploadPath: 'Error',
-      status: 'error',
-      message: err.message,
-    });
-    process.exit(1);
-  }
+  return response.json();
 };
 
-await checkCredentials();
+/**
+ * Upload file directly to Supabase (fallback method)
+ */
+const uploadToSupabase = async (file, uploadPath) => {
+  const content = fs.readFileSync(file);
+  const contentType = mime.lookup(file) || 'application/octet-stream';
 
-const fileExistsInBucket = async (pathInBucket) => {
-  const dir = path.dirname(pathInBucket);
-  const filename = path.basename(pathInBucket);
   const { data, error } = await supabase.storage
     .from(bucket)
-    .list(dir === '.' ? '' : dir, { limit: 1000 });
-  if (error) {
-    console.error(`⚠️ Could not verify duplicate: ${error.message}`);
-    writeLog(`⚠️ Could not verify duplicate: ${error.message}`);
-    await sendLogToSupabaseImmediate({
-      file: 'Error',
-      uploadPath: 'Error',
-      status: 'error',
-      message: error.message,
+    .upload(uploadPath.replace(/\\/g, '/'), content, {
+      upsert: true,
+      contentType,
     });
-    return false;
-  }
-  return data.some((file) => file.name === filename);
-};
 
-const logFilePath = path.resolve(process.cwd(), 'upload.log');
-const writeLog = (message) => {
-  try {
-    const timestamp = new Date().toISOString();
-    fs.appendFileSync(logFilePath, `[${timestamp}] ${message}\n`);
-  } catch (error) {
-    console.error(`❌ Error writing to log file: ${error.code} | ${error.message} | path: ${logFilePath}`);
+  if (error) {
+    throw new Error(error.message);
   }
+
+  return data;
 };
 
-// Modified to fetch from Supabase first, then fallback to local log
-const getProcessedPaths = async () => {
-  const processed = new Set();
+/**
+ * Insert file stats into uploader table with document detection
+ */
+const insertStatsToUploaderTable = async (files, options) => {
+  if (!supabase) {
+    throw new Error('Supabase client not initialized. Stats mode requires Supabase connection.');
+  }
 
-  // Try to fetch from Supabase first
-  try {
-    const { data, error } = await supabase
-      .from('upload_logs')
-      .select('path')
-      .in('status', ['success', 'skipped']);
+  const detectionService = new FileDetectionService();
+  const records = [];
+
+  for (const file of files) {
+    const stats = fs.statSync(file.path);
+    const originalPath = options.clientPath || file.path;
+    
+    // Check if record already exists
+    const { data: existingRecords, error: checkError } = await supabase
+      .from('uploader')
+      .select('id, original_path')
+      .eq('original_path', originalPath)
+      .limit(1);
+
+    if (checkError) {
+      console.error(`❌ Error checking for existing record: ${checkError.message}`);
+      continue;
+    }
 
-    if (error) {
-      console.warn(
-        `⚠️ Could not fetch processed paths from Supabase: ${error.message}. Falling back to local log.`,
-      );
-      // Fallback to local log if Supabase fetch fails
-      const lines = fs.existsSync(logFilePath)
-        ? fs.readFileSync(logFilePath, 'utf-8').split('\\n')
-        : [];
-      for (const line of lines) {
-        const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
-        if (match) {
-          const [, , path] = match;
-          processed.add(path.trim());
-        }
-      }
-    } else if (data) {
-      data.forEach((log) => {
-        if (log.path) {
-          processed.add(log.path.trim());
-        }
-      });
-      // Also read from local log to ensure any paths logged before this change or during a Supabase outage are included
-      const lines = fs.existsSync(logFilePath)
-        ? fs.readFileSync(logFilePath, 'utf-8').split('\\n')
-        : [];
-      for (const line of lines) {
-        const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
-        if (match) {
-          const [, , pathValue] = match;
-          if (pathValue) {
-            processed.add(pathValue.trim());
+    if (existingRecords && existingRecords.length > 0) {
+      console.log(`⏭️ Skipping duplicate: ${path.basename(file.path)}`);
+      continue;
+    }
+    
+    // Initialize record with basic file stats
+    const record = {
+      document_type: null,
+      size: stats.size,
+      num_pedimento: null,
+      filename: file.originalName || path.basename(file.path),
+      original_path: originalPath,
+      arela_path: null,
+      status: 'stats',
+      rfc: null,
+      message: null
+    };
+
+    // Try to detect document type for supported files
+    if (detectionService.isSupportedFileType(file.path)) {
+      try {
+        const detection = await detectionService.detectFile(file.path);
+        
+        if (detection.detectedType) {
+          record.document_type = detection.detectedType;
+          record.num_pedimento = detection.detectedPedimento;
+          record.status = 'detected';
+          
+          // Set arela_path for pedimento_simplificado documents
+          if (detection.arelaPath) {
+            record.arela_path = detection.arelaPath;
+          }
+          
+          // Extract RFC from fields if available
+          const rfcField = detection.fields.find(f => f.name === 'rfc' && f.found);
+          if (rfcField) {
+            record.rfc = rfcField.value;
+          }
+        } else {
+          record.status = 'not-detected';
+          if (detection.error) {
+            record.message = detection.error;
           }
         }
+      } catch (error) {
+        console.error(`❌ Error detecting ${record.filename}:`, error.message);
+        record.status = 'detection-error';
+        record.message = error.message;
       }
+    } else {
+      record.status = 'unsupported';
+      record.message = 'File type not supported for detection';
     }
-  } catch (e) {
-    console.warn(
-      `⚠️ Error fetching from Supabase or reading local log: ${e.message}. Proceeding with an empty set of processed paths initially.`,
-    );
-    // Ensure local log is still attempted if Supabase connection itself fails
-    const lines = fs.existsSync(logFilePath)
-      ? fs.readFileSync(logFilePath, 'utf-8').split('\\n')
-      : [];
-    for (const line of lines) {
-      const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
-      if (match) {
-        const [, , path] = match;
-        processed.add(path.trim());
-      }
-    }
+
+    records.push(record);
   }
-  return processed;
-};
 
-const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+  if (records.length === 0) {
+    console.log('📝 No new records to insert (all were duplicates or errors)');
+    return [];
+  }
 
-const uploadWithRetry = async (uploadFn, maxRetries = 5, delayMs = 2000) => {
-  let attempt = 0;
-  let lastError;
+  console.log(`💾 Inserting ${records.length} new records into uploader table...`);
+  
+  const { data, error } = await supabase
+    .from('uploader')
+    .insert(records)
+    .select();
 
-  while (attempt < maxRetries) {
-    try {
-      const result = await uploadFn();
-      if (!result.error) return result;
-      lastError = result.error;
-      attempt++;
-
-      // Exponential backoff with jitter
-      if (attempt < maxRetries) {
-        const backoffDelay =
-          delayMs * Math.pow(2, attempt - 1) + Math.random() * 1000;
-        console.log(
-          `Retry ${attempt}/${maxRetries} after ${Math.round(backoffDelay)}ms...`,
-        );
-        await delay(backoffDelay);
-      }
-    } catch (error) {
-      lastError = error;
-      attempt++;
-
-      if (attempt < maxRetries) {
-        const backoffDelay =
-          delayMs * Math.pow(2, attempt - 1) + Math.random() * 1000;
-        console.log(
-          `Retry ${attempt}/${maxRetries} after ${Math.round(backoffDelay)}ms due to exception...`,
-        );
-        await delay(backoffDelay);
-      }
-    }
+  if (error) {
+    throw new Error(`Failed to insert stats records: ${error.message}`);
   }
 
-  return {
-    error: new Error(
-      `Max retries exceeded. Last error: ${lastError?.message || 'Unknown error'}`,
-    ),
-  };
+  return data;
 };
 
-// Function to process a single file
-const processFile = async (
-  file,
+const processFilesInBatches = async (
+  files,
+  batchSize,
   options,
   basePath,
   folder,
   sourcePath,
   processedPaths,
 ) => {
-  let currentFile = file;
-  let result = {
-    success: false,
-    skipped: false,
-    error: null,
-    message: '',
-  };
+  let totalUploaded = 0;
+  let totalDetected = 0;
+  let totalOrganized = 0;
+  let totalErrors = 0;
+  let totalSkipped = 0;
 
-  try {
-    // Check if we need to rename the file
-    if (options.renameFiles) {
-      const originalName = path.basename(file);
-      const sanitizedName = sanitizeFileName(originalName);
-
-      if (originalName !== sanitizedName) {
-        const newFilePath = path.join(path.dirname(file), sanitizedName);
-
-        if (options.dryRun) {
-          result.message = `Would rename: ${originalName} → ${sanitizedName}`;
-          result.skipped = true;
-          return result;
-        } else {
-          try {
-            fs.renameSync(file, newFilePath);
-            currentFile = newFilePath;
-            writeLog(`RENAMED: ${originalName} → ${sanitizedName}`);
-            await sendLogToSupabase({
-              file: originalName,
-              uploadPath: sanitizedName,
-              status: 'renamed',
-              message: `Renamed from ${originalName}`,
-            });
-          } catch (renameError) {
-            result.error = `Failed to rename ${originalName}: ${renameError.message}`;
-            writeLog(`RENAME_ERROR: ${originalName} | ${renameError.message}`);
-            return result;
+  const messageBuffer = [];
+
+  const progressBarFormat = options.statsOnly 
+    ? '📊 Processing [{bar}] {percentage}% | {value}/{total} files | Stats: {successCount} | Errors: {failureCount} | Duplicates: {skippedCount}'
+    : '📂 Processing [{bar}] {percentage}% | {value}/{total} files | Success: {successCount} | Errors: {failureCount} | Skipped: {skippedCount}';
+
+  const progressBar = new cliProgress.SingleBar({
+    format: progressBarFormat,
+    barCompleteChar: '█',
+    barIncompleteChar: '░',
+    hideCursor: true,
+  });
+
+  progressBar.start(files.length, 0, {
+    successCount: 0,
+    failureCount: 0,
+    skippedCount: 0,
+  });
+
+  if (options.statsOnly) {
+    // Stats-only mode - Read file stats and insert to uploader table
+    console.log('📊 Processing files in stats-only mode...');
+    
+    let totalDetected = 0;
+    let totalNotDetected = 0;
+    let totalUnsupported = 0;
+    let totalDetectionErrors = 0;
+    
+    for (let i = 0; i < files.length; i += batchSize) {
+      const batch = files.slice(i, i + batchSize);
+      
+      const statsFiles = batch.map((file) => {
+        const originalFileName = path.basename(file);
+        
+        return {
+          path: file,
+          originalName: originalFileName,
+        };
+      });
+
+      try {
+        const insertedRecords = await insertStatsToUploaderTable(statsFiles, options);
+        const actualInserted = insertedRecords.length;
+        const skippedDuplicates = statsFiles.length - actualInserted;
+        
+        totalUploaded += actualInserted;
+        totalSkipped += skippedDuplicates;
+        
+        // Count detection results from inserted records
+        insertedRecords.forEach(record => {
+          switch (record.status) {
+            case 'detected':
+              totalDetected++;
+              break;
+            case 'not-detected':
+              totalNotDetected++;
+              break;
+            case 'unsupported':
+              totalUnsupported++;
+              break;
+            case 'detection-error':
+              totalDetectionErrors++;
+              break;
+          }
+        });
+        
+        statsFiles.forEach((file) => {
+          const wasInserted = insertedRecords.some(record => 
+            record.original_path === (options.clientPath || file.path)
+          );
+          if (wasInserted) {
+            writeLog(`STATS: ${file.path} -> uploader table`);
+          } else {
+            writeLog(`DUPLICATE: ${file.path} -> already exists in uploader table`);
           }
+        });
+        
+        if (actualInserted > 0) {
+          console.log(`📈 Inserted ${actualInserted} stats records`);
+        }
+        if (skippedDuplicates > 0) {
+          console.log(`⏭️ Skipped ${skippedDuplicates} duplicates`);
         }
+        if (options.detect !== false) {
+          console.log(`   🔍 Detected: ${totalDetected}, Not detected: ${totalNotDetected}, Unsupported: ${totalUnsupported}, Errors: ${totalDetectionErrors}`);
+        }
+        
+      } catch (error) {
+        totalErrors += statsFiles.length;
+        statsFiles.forEach((file) => {
+          writeLog(`ERROR: ${file.path}: ${error.message}`);
+          messageBuffer.push(`❌ ${file.originalName}: ${error.message}`);
+        });
       }
-    }
 
-    const content = fs.readFileSync(currentFile);
-    const relativePathRaw = path
-      .relative(basePath, currentFile)
-      .replace(/^[\\/]+/, '')
-      .replace(/\\/g, '/');
-
-    // Always sanitize the filename for upload path
-    const pathParts = relativePathRaw.split('/');
-    const originalFileName = pathParts[pathParts.length - 1];
-    const sanitizedFileName = sanitizeFileName(originalFileName);
-    pathParts[pathParts.length - 1] = sanitizedFileName;
-    const sanitizedRelativePath = pathParts.join('/');
-
-    const uploadPathRaw = options.prefix
-      ? path.posix.join(options.prefix, sanitizedRelativePath)
-      : sanitizedRelativePath;
-    const uploadPath = sanitizePath(uploadPathRaw);
-
-    if (
-      uploadPath !== uploadPathRaw ||
-      originalFileName !== sanitizedFileName
-    ) {
-      writeLog(`SANITIZED: ${relativePathRaw} → ${uploadPath}`);
-      await sendLogToSupabase({
-        file: currentFile,
-        uploadPath: relativePathRaw,
-        status: 'sanitized',
-        message: `Sanitized to ${uploadPath} (Arela Version: ${version})`,
+      progressBar.update(i + batch.length, {
+        successCount: totalUploaded,
+        failureCount: totalErrors,
+        skippedCount: totalSkipped,
       });
-    }
 
-    if (processedPaths.has(uploadPath)) {
-      result.skipped = true;
-      result.message = `Already processed (log): ${currentFile}`;
-      return result;
+      if (i + batchSize < files.length) {
+        await new Promise((resolve) => setTimeout(resolve, 200));
+      }
     }
+    
+    // Store detection stats for summary
+    totalDetected = totalDetected || 0;
+    totalNotDetected = totalNotDetected || 0;
+    totalUnsupported = totalUnsupported || 0;
+    totalDetectionErrors = totalDetectionErrors || 0;
+    
+  } else if (apiMode && !options.forceSupabase) {
+    // API Mode - Process in batches
+    for (let i = 0; i < files.length; i += batchSize) {
+      const batch = files.slice(i, i + batchSize);
+      let sanitizedRelativePath;
+
+      const apiFiles = batch
+        .map((file) => {
+          const relativePathRaw = path
+            .relative(basePath, file)
+            .replace(/^[\\/]+/, '')
+            .replace(/\\/g, '/');
+
+          const pathParts = relativePathRaw.split('/');
+          const originalFileName = pathParts[pathParts.length - 1];
+          const sanitizedFileName = sanitizeFileName(originalFileName);
+          pathParts[pathParts.length - 1] = sanitizedFileName;
+          sanitizedRelativePath = pathParts.join('/');
+
+          let uploadPath;
+
+          // Handle combined folder structure + auto-detection
+          if (options.folderStructure && options.autoDetectStructure) {
+            const detection = extractYearAndPedimentoFromPath(file, basePath);
+            if (detection.detected) {
+              const autoStructure = `${detection.year}/${detection.pedimento}`;
+              const combinedStructure = `${options.folderStructure}/${autoStructure}`;
+              uploadPath = path.posix.join(
+                combinedStructure,
+                sanitizedFileName,
+              );
+              console.log(
+                `📁 Combined structure: ${options.folderStructure}/${autoStructure} for ${originalFileName} -> ${uploadPath}`,
+              );
+            } else {
+              // Fallback to just custom structure if auto-detection fails
+              uploadPath = path.posix.join(
+                options.folderStructure,
+                sanitizedFileName,
+              );
+              console.log(
+                `📁 Custom structure (auto-detection failed): ${uploadPath}`,
+              );
+            }
+          } else if (options.folderStructure) {
+            // Use custom folder structure only
+            uploadPath = path.posix.join(
+              options.folderStructure,
+              sanitizedFileName,
+            );
+            console.log(`📁 Custom structure: ${uploadPath}`);
+          } else if (options.autoDetectStructure) {
+            // Auto-detect structure from path if enabled
+            const detection = extractYearAndPedimentoFromPath(file, basePath);
+            if (detection.detected) {
+              const autoStructure = `${detection.year}/${detection.pedimento}`;
+              uploadPath = path.posix.join(autoStructure, sanitizedFileName);
+              console.log(
+                `🔍 Auto-detected: ${autoStructure} for ${originalFileName} -> ${uploadPath}`,
+              );
+            } else {
+              uploadPath = options.prefix
+                ? path.posix.join(options.prefix, sanitizedRelativePath)
+                : sanitizedRelativePath;
+              console.log(`📁 Using relative path: ${uploadPath}`);
+            }
+          } else {
+            uploadPath = options.prefix
+              ? path.posix.join(options.prefix, sanitizedRelativePath)
+              : sanitizedRelativePath;
+            console.log(`📁 Using standard path: ${uploadPath}`);
+          }
 
-    const contentType = mime.lookup(currentFile) || 'application/octet-stream';
+          if (processedPaths.has(uploadPath)) {
+            totalSkipped++;
+            writeLog(`SKIPPED: ${file} -> ${uploadPath}`);
+            return null;
+          }
+
+          return {
+            path: file,
+            name: sanitizedFileName,
+            originalName: originalFileName,
+            uploadPath: uploadPath.replace(/\\/g, '/'), // Ensure forward slashes
+            contentType: mime.lookup(file) || 'application/octet-stream',
+          };
+        })
+        .filter(Boolean);
+
+      if (apiFiles.length > 0) {
+        // console.log(`🔄 Processing batch of ${apiFiles.length} files`);
+        // apiFiles.forEach(f => console.log(`   📄 ${f.name} -> ${f.uploadPath}`));
+
+        try {
+          // Use clientPath from options if specified, otherwise construct from detection or folder
+          let clientPath = options.clientPath;
+
+          if (!clientPath && apiFiles.length > 0) {
+            const firstFile = apiFiles[0];
+            const detection = extractYearAndPedimentoFromPath(
+              firstFile.path,
+              basePath,
+            );
+            if (detection.detected) {
+              // clientPath = `${detection.year}/${detection.pedimento}/`;
+              clientPath = path
+                .resolve(basePath, sanitizedRelativePath)
+                .replace(/\\/g, '/');
+            } else {
+              // Fallback to folder structure if no year/pedimento detected
+              clientPath = path.resolve(basePath, folder).replace(/\\/g, '/');
+            }
+          }
 
-    const exists = await fileExistsInBucket(uploadPath);
+          const result = await uploadToApi(apiFiles, {
+            ...options,
+            clientPath: clientPath,
+          });
+
+          totalUploaded += result.stats.uploadedCount;
+          totalDetected += result.stats.detectedCount;
+          totalOrganized += result.stats.organizedCount;
+          totalErrors += result.stats.errorCount;
+
+          result.uploaded.forEach((upload) => {
+            const apiFile = apiFiles.find(
+              (f) => f.name === upload.originalName,
+            );
+            if (apiFile) {
+              writeLog(`SUCCESS: ${apiFile.path} -> ${apiFile.uploadPath}`);
+              processedPaths.add(apiFile.uploadPath);
+            }
+          });
+
+          result.errors.forEach((error) => {
+            writeLog(
+              `ERROR: ${error.fileName}: ${error.error} (${error.step})`,
+            );
+            messageBuffer.push(
+              `❌ ${error.fileName}: ${error.error} (${error.step})`,
+            );
+          });
+        } catch (error) {
+          totalErrors += apiFiles.length;
+          apiFiles.forEach((file) => {
+            writeLog(`ERROR: ${file.path}: ${error.message}`);
+            messageBuffer.push(`❌ ${file.name}: ${error.message}`);
+          });
+        }
+      }
 
-    if (exists) {
-      result.skipped = true;
-      result.message = `Skipped (already exists): ${currentFile}`;
-      writeLog(`SKIPPED: ${currentFile} -> ${uploadPath}`);
-      await sendLogToSupabase({
-        file: currentFile,
-        uploadPath,
-        status: 'skipped',
-        message: 'Already exists in bucket',
+      progressBar.update(i + batch.length, {
+        successCount: totalUploaded,
+        failureCount: totalErrors,
+        skippedCount: totalSkipped,
       });
-      return result;
-    }
-
-    const { error } = await uploadWithRetry(() =>
-      supabase.storage.from(bucket).upload(uploadPath, content, {
-        upsert: true,
-        contentType,
-        metadata: {
-          originalName: path.basename(currentFile),
-          sanitizedName: path.basename(uploadPath),
-          clientPath: path.posix.join(
-            basePath,
-            folder,
-            path.relative(sourcePath, currentFile).replace(/\\/g, '/'),
-          ),
-          arelaVersion: version,
-        },
-      }),
-    );
 
-    if (error) {
-      result.error = error.message || JSON.stringify(error);
-      writeLog(`ERROR: ${currentFile} -> ${uploadPath} | ${result.error}`);
-      await sendLogToSupabase({
-        file: currentFile,
-        uploadPath,
-        status: 'error',
-        message: result.error,
+      if (i + batchSize < files.length) {
+        await new Promise((resolve) => setTimeout(resolve, 200));
+      }
+    }
+  } else {
+    // Direct Supabase mode
+    for (let i = 0; i < files.length; i++) {
+      const file = files[i];
+      try {
+        const relativePath = path.relative(basePath, file);
+        let uploadPath;
+
+        // Handle combined folder structure + auto-detection
+        if (options.folderStructure && options.autoDetectStructure) {
+          const detection = extractYearAndPedimentoFromPath(file, basePath);
+          if (detection.detected) {
+            const autoStructure = `${detection.year}/${detection.pedimento}`;
+            const combinedStructure = `${options.folderStructure}/${autoStructure}`;
+            const fileName = path.basename(file);
+            uploadPath = path.join(combinedStructure, fileName);
+            console.log(
+              `📁 Combined structure: ${options.folderStructure}/${autoStructure} for ${fileName}`,
+            );
+          } else {
+            // Fallback to just custom structure if auto-detection fails
+            const fileName = path.basename(file);
+            uploadPath = path.join(options.folderStructure, fileName);
+            console.log(
+              `📁 Custom structure (auto-detection failed): ${uploadPath}`,
+            );
+          }
+        } else if (options.folderStructure) {
+          // Use custom folder structure only
+          const fileName = path.basename(file);
+          uploadPath = path.join(options.folderStructure, fileName);
+          console.log(`📁 Custom structure: ${uploadPath}`);
+        } else if (options.autoDetectStructure) {
+          // Auto-detect structure from path if enabled
+          const detection = extractYearAndPedimentoFromPath(file, basePath);
+          if (detection.detected) {
+            const autoStructure = `${detection.year}/${detection.pedimento}`;
+            const fileName = path.basename(file);
+            uploadPath = path.join(autoStructure, fileName);
+          } else {
+            uploadPath = options.prefix
+              ? path.join(options.prefix, relativePath)
+              : relativePath;
+          }
+        } else {
+          uploadPath = options.prefix
+            ? path.join(options.prefix, relativePath)
+            : relativePath;
+        }
+
+        if (processedPaths.has(uploadPath)) {
+          totalSkipped++;
+          writeLog(`SKIPPED: ${file} -> ${uploadPath}`);
+        } else {
+          await uploadToSupabase(file, uploadPath);
+          totalUploaded++;
+          writeLog(`SUCCESS: ${file} -> ${uploadPath}`);
+          processedPaths.add(uploadPath);
+        }
+      } catch (error) {
+        totalErrors++;
+        writeLog(`ERROR: ${file}: ${error.message}`);
+        messageBuffer.push(`❌ ${path.basename(file)}: ${error.message}`);
+      }
+
+      progressBar.update(i + 1, {
+        successCount: totalUploaded,
+        failureCount: totalErrors,
+        skippedCount: totalSkipped,
       });
+    }
+  }
+
+  progressBar.stop();
+
+  const errorMessages = messageBuffer.filter((msg) => msg.startsWith('❌'));
+  if (errorMessages.length > 0) {
+    console.log('\n🚨 Errors encountered during processing:');
+    errorMessages.forEach((msg) => console.error(msg));
+  }
+
+  return {
+    successCount: totalUploaded,
+    detectedCount: totalDetected,
+    organizedCount: totalOrganized,
+    failureCount: totalErrors,
+    skippedCount: totalSkipped,
+  };
+};
+
+/**
+ * Upload files to Arela API based on specific RFC values
+ */
+const uploadFilesByRfc = async (options = {}) => {
+  if (!supabase) {
+    console.error('❌ Supabase client not initialized');
+    process.exit(1);
+  }
+
+  if (!API_BASE_URL || !API_TOKEN) {
+    console.error('❌ Arela API configuration missing. Please set ARELA_API_URL and ARELA_API_TOKEN environment variables.');
+    process.exit(1);
+  }
+
+  if (!uploadRfcs || uploadRfcs.length === 0) {
+    console.error('❌ No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.');
+    console.error('   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"');
+    process.exit(1);
+  }
+
+  console.log('🎯 RFC-based Upload Mode');
+  console.log(`📋 Target RFCs: ${uploadRfcs.join(', ')}`);
+  console.log('🔍 Searching for files to upload...');
+
+  // Step 1: Get all records that match the specified RFCs and have arela_path
+  const { data: rfcRecords, error: rfcError } = await supabase
+    .from('uploader')
+    .select('arela_path')
+    .in('rfc', uploadRfcs)
+    .not('arela_path', 'is', null);
+
+  if (rfcError) {
+    console.error('❌ Error fetching RFC records:', rfcError.message);
+    return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
+  }
+
+  if (!rfcRecords || rfcRecords.length === 0) {
+    console.log('ℹ️  No files found for the specified RFCs with arela_path');
+    console.log(`   Make sure files for RFCs [${uploadRfcs.join(', ')}] have been processed and have arela_path values`);
+    return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+  }
+
+  // Step 2: Get unique arela_paths from the RFC matches
+  const uniqueArelaPaths = [...new Set(rfcRecords.map(r => r.arela_path))];
+  console.log(`� Found ${uniqueArelaPaths.length} unique arela_path(s) for the specified RFCs`);
+
+  // Step 3: Get ALL files that have these arela_paths (including supporting documents)
+  // Use pagination to ensure we get all files, regardless of count
+  let allRelatedFiles = [];
+  let hasMore = true;
+  let offset = 0;
+  const queryBatchSize = 1000;
+
+  console.log('📥 Fetching all related files (with pagination)...');
+  
+  while (hasMore) {
+    const { data: batch, error: queryError } = await supabase
+      .from('uploader')
+      .select('id, original_path, arela_path, filename, rfc, document_type')
+      .in('arela_path', uniqueArelaPaths)
+      .not('original_path', 'is', null)
+      .range(offset, offset + queryBatchSize - 1);
+
+    if (queryError) {
+      console.error('❌ Error fetching related files:', queryError.message);
+      return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
+    }
+
+    if (!batch || batch.length === 0) {
+      hasMore = false;
     } else {
-      result.success = true;
-      result.message = `Uploaded ${currentFile} -> ${uploadPath}`;
-      writeLog(`SUCCESS: ${currentFile} -> ${uploadPath}`);
-      await sendLogToSupabase({
-        file: currentFile,
-        uploadPath,
-        status: 'success',
-        message: 'Uploaded successfully',
-      });
+      allRelatedFiles = allRelatedFiles.concat(batch);
+      offset += queryBatchSize;
+      
+      // If we got less than queryBatchSize, we've reached the end
+      if (batch.length < queryBatchSize) {
+        hasMore = false;
+      }
+    }
+  }
+
+  if (!allRelatedFiles || allRelatedFiles.length === 0) {
+    console.log('ℹ️  No related files found for the arela_paths');
+    return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+  }
+
+  console.log(`📁 Found ${allRelatedFiles.length} total files to upload (including supporting documents)`);
+  
+  // Group by RFC and arela_path for better organization
+  const filesByRfc = allRelatedFiles.reduce((acc, record) => {
+    const rfc = record.rfc || 'No RFC';
+    if (!acc[rfc]) {
+      acc[rfc] = [];
+    }
+    acc[rfc].push(record);
+    return acc;
+  }, {});
+
+  console.log('📊 Files by RFC (including supporting documents):');
+  for (const [rfc, files] of Object.entries(filesByRfc)) {
+    const documentTypes = [...new Set(files.map(f => f.document_type || 'Unknown'))];
+    console.log(`   ${rfc}: ${files.length} files (${documentTypes.join(', ')})`);
+  }
+
+  // Group by arela_path for upload organization
+  const filesByPath = allRelatedFiles.reduce((acc, record) => {
+    const path = record.arela_path;
+    if (!acc[path]) {
+      acc[path] = [];
     }
-  } catch (err) {
-    result.error = err.message || JSON.stringify(err);
-    writeLog(`ERROR: ${currentFile} | ${result.error}`);
-    await sendLogToSupabase({
-      file: currentFile,
-      uploadPath: currentFile,
-      status: 'error',
-      message: result.error,
+    acc[path].push(record);
+    return acc;
+  }, {});
+
+  console.log('� Files grouped by arela_path:');
+  for (const [path, files] of Object.entries(filesByPath)) {
+    console.log(`   ${path}: ${files.length} files`);
+  }
+
+  let totalProcessed = 0;
+  let totalUploaded = 0;
+  let totalErrors = 0;
+  let totalSkipped = 0;
+
+  // Create progress bar
+  const progressBar = new cliProgress.SingleBar({
+    format: '🚀 Uploading files |{bar}| {percentage}% | {value}/{total} | Uploaded: {uploaded} | Errors: {errors} | Skipped: {skipped}',
+    barCompleteChar: '█',
+    barIncompleteChar: '░',
+    hideCursor: true,
+  });
+
+  if (options.showProgress !== false) {
+    progressBar.start(allRelatedFiles.length, 0, {
+      uploaded: 0,
+      errors: 0,
+      skipped: 0,
     });
   }
 
-  return result;
+  const batchSize = parseInt(options.batchSize) || 10;
+  console.log(`📦 Processing in batches of ${batchSize} files`);
+
+  // Process files in batches
+  for (let i = 0; i < allRelatedFiles.length; i += batchSize) {
+    const batch = allRelatedFiles.slice(i, i + batchSize);
+    const batchNumber = Math.floor(i / batchSize) + 1;
+    const totalBatches = Math.ceil(allRelatedFiles.length / batchSize);
+    
+    console.log(`\n📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} files)`);
+
+    // Prepare files for upload
+    const filesToUpload = [];
+    
+    for (const record of batch) {
+      totalProcessed++;
+      
+      try {
+        const originalPath = record.original_path;
+        
+        // Check if file exists
+        if (!fs.existsSync(originalPath)) {
+          console.log(`   ⚠️  File not found: ${originalPath}`);
+          totalSkipped++;
+          continue;
+        }
+
+        const fileStats = fs.statSync(originalPath);
+        const fileBuffer = fs.readFileSync(originalPath);
+        
+        filesToUpload.push({
+          path: originalPath,
+          buffer: fileBuffer,
+          size: fileStats.size,
+          name: record.filename,
+          arelaPath: record.arela_path,
+          rfc: record.rfc,
+          documentType: record.document_type,
+        });
+        
+      } catch (error) {
+        console.error(`   ❌ Error reading file ${record.original_path}:`, error.message);
+        totalErrors++;
+      }
+      
+      if (options.showProgress !== false) {
+        progressBar.update(totalProcessed, {
+          uploaded: totalUploaded,
+          errors: totalErrors,
+          skipped: totalSkipped,
+        });
+      }
+    }
+
+    // Upload the batch if we have files
+    if (filesToUpload.length > 0) {
+      try {
+        console.log(`   🚀 Uploading ${filesToUpload.length} files to Arela API...`);
+        
+        const formData = new FormData();
+        
+        // Add files to form data
+        filesToUpload.forEach((file, index) => {
+          formData.append(`files`, file.buffer, {
+            filename: file.name,
+            contentType: mime.lookup(file.name) || 'application/octet-stream',
+          });
+        });
+
+        // Instead of using per-file folder structures, we'll group by arela_path and upload separately
+        // Group files by their arela_path to upload them in correct structure
+        const filesByPath = filesToUpload.reduce((acc, file) => {
+          const path = file.arelaPath.replace(/\/$/, '');
+          if (!acc[path]) {
+            acc[path] = [];
+          }
+          acc[path].push(file);
+          return acc;
+        }, {});
+
+        // Upload each group separately with its folder structure
+        for (const [arelaPath, pathFiles] of Object.entries(filesByPath)) {
+          const pathFormData = new FormData();
+          
+          pathFiles.forEach((file) => {
+            pathFormData.append('files', file.buffer, {
+              filename: file.name,
+              contentType: mime.lookup(file.name) || 'application/octet-stream',
+            });
+          });
+
+          // Set folder structure for this group - concatenate custom prefix with arela_path
+          const folderStructure = options.folderStructure 
+            ? `${options.folderStructure}/${arelaPath}`.replace(/\/+/g, '/').replace(/\/$/, '')
+            : arelaPath;
+          pathFormData.append('folderStructure', folderStructure);
+          pathFormData.append('autoDetect', 'true');
+          pathFormData.append('autoOrganize', 'false');
+          pathFormData.append('batchSize', String(pathFiles.length));
+          pathFormData.append('clientVersion', packageVersion);
+          if (bucket) {
+            pathFormData.append('bucket', bucket);
+          }
+
+          console.log(`     📁 Uploading ${pathFiles.length} files to: ${folderStructure}`);
+
+          const response = await fetch(`${API_BASE_URL}/api/storage/batch-upload-and-process`, {
+            method: 'POST',
+            headers: {
+              'x-api-key': API_TOKEN,
+            },
+            body: pathFormData,
+          });
+
+          if (!response.ok) {
+            const errorText = await response.text();
+            throw new Error(`HTTP ${response.status}: ${errorText}`);
+          }
+
+          const result = await response.json();
+          
+          // Check if upload was successful based on stats rather than success field
+          const isSuccessful = result.stats && result.stats.uploadedCount > 0 && result.stats.errorCount === 0;
+          
+          if (isSuccessful) {
+            console.log(`     ✅ Group uploaded: ${result.stats.uploadedCount} files to ${folderStructure}`);
+            totalUploaded += result.stats.uploadedCount;
+            
+            if (result.stats.detectedCount > 0) {
+              console.log(`     🔍 Files detected: ${result.stats.detectedCount}`);
+            }
+            if (result.stats.organizedCount > 0) {
+              console.log(`     📁 Files organized: ${result.stats.organizedCount}`);
+            }
+          } else {
+            console.error(`     ❌ Upload failed for ${folderStructure}:`);
+            if (result.errors && result.errors.length > 0) {
+              result.errors.forEach(error => {
+                console.error(`        - ${error.fileName}: ${error.error}`);
+              });
+            }
+            totalErrors += pathFiles.length;
+          }
+
+          // Small delay between path groups
+          await new Promise(resolve => setTimeout(resolve, 100));
+        }
+
+      } catch (error) {
+        console.error(`   ❌ Error uploading batch ${batchNumber}:`, error.message);
+        totalErrors += filesToUpload.length;
+      }
+    }
+
+    // Small delay between batches
+    if (i + batchSize < allRelatedFiles.length) {
+      await new Promise(resolve => setTimeout(resolve, 200));
+    }
+  }
+
+  if (options.showProgress !== false) {
+    progressBar.stop();
+  }
+
+  console.log(`\n${'='.repeat(60)}`);
+  console.log(`🎯 RFC-BASED UPLOAD COMPLETED`);
+  console.log(`${'='.repeat(60)}`);
+  console.log(`   📋 Files processed: ${totalProcessed}`);
+  console.log(`   ✅ Files uploaded: ${totalUploaded}`);
+  console.log(`   ⏭️  Files skipped: ${totalSkipped}`);
+  console.log(`   ❌ Errors: ${totalErrors}`);
+  console.log(`${'='.repeat(60)}\n`);
+
+  return {
+    processedCount: totalProcessed,
+    uploadedCount: totalUploaded,
+    skippedCount: totalSkipped,
+    errorCount: totalErrors,
+  };
 };
 
-// Function to process files in parallel batches
-const processFilesInBatches = async (
-  files,
-  batchSize,
-  options,
-  basePath,
-  folder,
-  sourcePath,
-  processedPaths,
-) => {
-  let successCount = 0;
-  let failureCount = 0;
-  let skippedCount = 0;
+/**
+ * Propagate arela_path from pedimento_simplificado records to related files with same base path
+ */
+const propagateArelaPath = async (options = {}) => {
+  if (!supabase) {
+    console.error('❌ Supabase client not initialized');
+    process.exit(1);
+  }
 
-  // Buffer for messages to show after progress bar completes
-  const messageBuffer = [];
+  console.log('🔍 Finding pedimento_simplificado records with arela_path...');
+
+  // Get all pedimento_simplificado records that have arela_path
+  const { data: pedimentoRecords, error: pedimentoError } = await supabase
+    .from('uploader')
+    .select('id, original_path, arela_path, filename')
+    .eq('document_type', 'pedimento_simplificado')
+    .not('arela_path', 'is', null);
 
+  if (pedimentoError) {
+    console.error('❌ Error fetching pedimento records:', pedimentoError.message);
+    return { processedCount: 0, updatedCount: 0, errorCount: 1 };
+  }
+
+  if (!pedimentoRecords || pedimentoRecords.length === 0) {
+    console.log('ℹ️  No pedimento_simplificado records with arela_path found');
+    return { processedCount: 0, updatedCount: 0, errorCount: 0 };
+  }
+
+  console.log(`📋 Found ${pedimentoRecords.length} pedimento records with arela_path`);
+
+  let totalProcessed = 0;
+  let totalUpdated = 0;
+  let totalErrors = 0;
+
+  // Create progress bar
   const progressBar = new cliProgress.SingleBar({
-    format:
-      '📂 Processing [{bar}] {percentage}% | {value}/{total} files | Success: {successCount} | Errors: {failureCount} | Skipped: {skippedCount}',
+    format: '🔄 Propagating paths |{bar}| {percentage}% | {value}/{total} | Updated: {updated} | Errors: {errors}',
     barCompleteChar: '█',
     barIncompleteChar: '░',
     hideCursor: true,
   });
 
-  progressBar.start(files.length, 0, {
-    successCount: 0,
-    failureCount: 0,
-    skippedCount: 0,
-  });
+  if (options.showProgress !== false) {
+    progressBar.start(pedimentoRecords.length, 0, {
+      updated: 0,
+      errors: 0,
+    });
+  }
 
-  for (let i = 0; i < files.length; i += batchSize) {
-    const batch = files.slice(i, i + batchSize);
+  // Process each pedimento record
+  for (const pedimento of pedimentoRecords) {
+    try {
+      totalProcessed++;
+      
+      // Extract base path from original_path (remove filename)
+      const basePath = path.dirname(pedimento.original_path);
+      
+      console.log(`\n🔍 Processing: ${pedimento.filename}`);
+      console.log(`   📁 Base path: ${basePath}`);
+      
+      // Extract folder part from existing arela_path by removing the filename
+      const existingPath = pedimento.arela_path;
+      const folderArelaPath = existingPath.includes('/') ? 
+        existingPath.substring(0, existingPath.lastIndexOf('/')) + '/' : 
+        existingPath.endsWith('/') ? existingPath : existingPath + '/';
+        
+      console.log(`   🎯 Original arela path: ${existingPath}`);
+      console.log(`   📁 Folder arela path: ${folderArelaPath}`);
+
+      // Find all files with the same base path that don't have arela_path yet
+      const { data: relatedFiles, error: relatedError } = await supabase
+        .from('uploader')
+        .select('id, filename, original_path')
+        .like('original_path', `${basePath}%`)
+        .is('arela_path', null)
+        .neq('id', pedimento.id); // Exclude the pedimento itself
+
+      if (relatedError) {
+        console.error(`❌ Error finding related files for ${pedimento.filename}:`, relatedError.message);
+        totalErrors++;
+        continue;
+      }
 
-    // Process batch in parallel
-    const batchResults = await Promise.all(
-      batch.map((file) =>
-        processFile(
-          file,
-          options,
-          basePath,
-          folder,
-          sourcePath,
-          processedPaths,
-        ),
-      ),
-    );
+      if (!relatedFiles || relatedFiles.length === 0) {
+        console.log(`   ℹ️  No related files found needing arela_path update`);
+        continue;
+      }
+
+      console.log(`   📄 Found ${relatedFiles.length} related files to update:`);
+      
+      // Show first 10 files, then indicate if there are more
+      const filesToShow = relatedFiles.slice(0, 10);
+      filesToShow.forEach(file => {
+        console.log(`      - ${file.filename}`);
+      });
+      
+      if (relatedFiles.length > 10) {
+        console.log(`      ... and ${relatedFiles.length - 10} more files`);
+      }
 
-    // Update counters and buffer messages (don't print them yet)
-    for (const result of batchResults) {
-      if (result.success) {
-        successCount++;
-        // Only buffer verbose success messages if needed
-        if (
-          process.env.UPLOAD_VERBOSE === 'true' &&
-          result.message &&
-          !result.error
-        ) {
-          messageBuffer.push(`✅ ${result.message}`);
+      // Process files in batches to avoid URI length limitations
+      const BATCH_SIZE = 50; // Process 50 files at a time
+      const fileIds = relatedFiles.map(f => f.id);
+      let batchErrors = 0;
+      let batchUpdated = 0;
+
+      console.log(`   🔄 Processing ${relatedFiles.length} files in batches of ${BATCH_SIZE}...`);
+
+      for (let i = 0; i < fileIds.length; i += BATCH_SIZE) {
+        const batchIds = fileIds.slice(i, i + BATCH_SIZE);
+        const batchNumber = Math.floor(i / BATCH_SIZE) + 1;
+        const totalBatches = Math.ceil(fileIds.length / BATCH_SIZE);
+        
+        console.log(`   📦 Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`);
+
+        try {
+          const { error: updateError } = await supabase
+            .from('uploader')
+            .update({ arela_path: folderArelaPath })
+            .in('id', batchIds);
+
+          if (updateError) {
+            console.error(`   ❌ Error in batch ${batchNumber}:`, updateError.message);
+            batchErrors++;
+          } else {
+            console.log(`   ✅ Batch ${batchNumber} completed: ${batchIds.length} files updated`);
+            batchUpdated += batchIds.length;
+          }
+        } catch (error) {
+          console.error(`   ❌ Exception in batch ${batchNumber}:`, error.message);
+          batchErrors++;
         }
-      } else if (result.skipped) {
-        skippedCount++;
-        // Only buffer verbose skip messages if needed
-        if (process.env.UPLOAD_VERBOSE === 'true' && result.message) {
-          messageBuffer.push(`⏭️  ${result.message}`);
+
+        // Small delay between batches to avoid overwhelming the database
+        if (i + BATCH_SIZE < fileIds.length) {
+          await new Promise(resolve => setTimeout(resolve, 100));
         }
-      } else if (result.error) {
-        failureCount++;
-        // Always buffer error messages to show later
-        messageBuffer.push(`❌ ${result.error}`);
       }
-    }
 
-    progressBar.update(i + batch.length, {
-      successCount,
-      failureCount,
-      skippedCount,
-    });
+      if (batchErrors > 0) {
+        console.error(`❌ ${batchErrors} batch(es) failed for ${pedimento.filename}`);
+        totalErrors++;
+      } else {
+        console.log(`   🎯 Successfully updated ${batchUpdated} related files`);
+        totalUpdated += batchUpdated;
+      }
 
-    // Manage cache size periodically (every 100 files processed)
-    if ((i + batch.length) % 100 === 0) {
-      manageCaches();
-      // Also flush logs every 100 files to maintain responsiveness
-      await logBatcher.flush();
+    } catch (error) {
+      console.error(`❌ Error processing ${pedimento.filename}:`, error.message);
+      totalErrors++;
     }
 
-    // Small delay between batches to prevent overwhelming the server
-    if (i + batchSize < files.length) {
-      await delay(100);
+    if (options.showProgress !== false) {
+      progressBar.update(totalProcessed, {
+        updated: totalUpdated,
+        errors: totalErrors,
+      });
     }
   }
 
-  // Stop progress bar cleanly before showing any messages
-  progressBar.stop();
-
-  // Now show buffered messages if there are any important ones to show
-  const errorMessages = messageBuffer.filter((msg) => msg.startsWith('❌'));
-  if (errorMessages.length > 0) {
-    console.log('\n🚨 Errors encountered during processing:');
-    errorMessages.forEach((msg) => console.error(msg));
+  if (options.showProgress !== false) {
+    progressBar.stop();
   }
 
-  // Show verbose messages only if requested
-  if (process.env.UPLOAD_VERBOSE === 'true') {
-    const otherMessages = messageBuffer.filter((msg) => !msg.startsWith('❌'));
-    if (otherMessages.length > 0) {
-      console.log('\n📝 Detailed processing log:');
-      otherMessages.forEach((msg) => console.log(msg));
-    }
-  }
+  console.log(`\n${'='.repeat(60)}`);
+  console.log(`🎯 ARELA PATH PROPAGATION COMPLETED`);
+  console.log(`${'='.repeat(60)}`);
+  console.log(`   📋 Pedimento records processed: ${totalProcessed}`);
+  console.log(`   ✅ Related files updated: ${totalUpdated}`);
+  console.log(`   ❌ Errors: ${totalErrors}`);
+  console.log(`${'='.repeat(60)}\n`);
 
   return {
-    successCount,
-    failureCount,
-    skippedCount,
+    processedCount: totalProcessed,
+    updatedCount: totalUpdated,
+    errorCount: totalErrors,
   };
 };
 
 program
-  .name('supabase-uploader')
-  .description('CLI to upload folders from a base path to Supabase Storage')
+  .name('arela-uploader')
+  .description(
+    'CLI to upload folders to Arela API or Supabase Storage with automatic processing',
+  )
   .option('-v, --version', 'output the version number')
   .option('-p, --prefix <prefix>', 'Prefix path in bucket', '')
+  .option('-b, --bucket <bucket>', 'Bucket name override')
+  .option('--force-supabase', 'Force direct Supabase upload (skip API)')
   .option(
-    '-r, --rename-files',
-    'Rename files with problematic characters before uploading',
+    '--no-auto-detect',
+    'Disable automatic file detection (API mode only)',
   )
   .option(
-    '--dry-run',
-    'Show what files would be renamed without actually renaming them',
+    '--no-auto-organize',
+    'Disable automatic file organization (API mode only)',
   )
   .option(
     '-c, --concurrency <number>',
-    'Number of files to process concurrently (default: 3)',
-    '3',
+    'Files per batch for processing (default: 10)',
+    '10',
   )
+  .option('--batch-size <number>', 'API batch size (default: 10)', '10')
+  .option('--show-stats', 'Show detailed processing statistics')
   .option(
-    '--show-cache-stats',
-    'Show cache statistics for performance analysis',
+    '--folder-structure <structure>',
+    'Custom folder structure (e.g., "2024/4023260" or "cliente1/pedimentos")',
   )
   .option(
-    '--batch-size <number>',
-    'Number of logs to batch before sending to Supabase (default: 50)',
-    '50',
+    '--auto-detect-structure',
+    'Automatically detect year/pedimento from file paths',
   )
+  .option('--client-path <path>', 'Client path for metadata tracking')
+  .option('--stats-only', 'Only read file stats and insert to uploader table, skip file upload')
+  .option('--no-detect', 'Disable document type detection in stats-only mode')
+  .option('--propagate-arela-path', 'Propagate arela_path from pedimento_simplificado records to related files with same base path')
+  .option('--upload-by-rfc', 'Upload files to Arela API based on RFC values from UPLOAD_RFCS environment variable')
   .action(async (options) => {
-    // Handle version option
     if (options.version) {
-      console.log(version);
+      console.log(packageVersion);
       process.exit(0);
     }
 
+    // Handle propagate-arela-path option
+    if (options.propagateArelaPath) {
+      // Initialize Supabase credentials for propagation
+      await checkCredentials(true); // Force Supabase mode
+      
+      const result = await propagateArelaPath({
+        showProgress: options.showStats || true,
+      });
+      
+      if (result.errorCount > 0) {
+        process.exit(1);
+      }
+      return;
+    }
+
+    // Handle upload-by-rfc option
+    if (options.uploadByRfc) {
+      // RFC upload needs both Supabase (for database queries) and API (for uploads)
+      await checkCredentials(false); // Initialize API mode
+      
+      // Also initialize Supabase for database queries
+      if (!supabase) {
+        if (!supabaseUrl || !supabaseKey) {
+          console.error('❌ RFC upload requires Supabase credentials for database queries.');
+          console.error('   Please set SUPABASE_URL and SUPABASE_KEY environment variables.');
+          process.exit(1);
+        }
+        
+        supabase = createClient(supabaseUrl, supabaseKey);
+        console.log('✅ Connected to Supabase for database queries');
+      }
+      
+      const result = await uploadFilesByRfc({
+        showProgress: options.showStats || true,
+        batchSize: parseInt(options.batchSize) || 10,
+        folderStructure: options.folderStructure,
+      });
+      
+      if (result.errorCount > 0) {
+        process.exit(1);
+      }
+      return;
+    }
+
+    // Initialize credentials with force supabase flag (for stats mode, always need Supabase)
+    await checkCredentials(options.forceSupabase || options.statsOnly);
+
     if (!basePath || !sources || sources.length === 0) {
       console.error(
         '⚠️ UPLOAD_BASE_PATH or UPLOAD_SOURCES not defined in environment variables.',
@@ -768,18 +1486,31 @@ program
       process.exit(1);
     }
 
-    const concurrency = parseInt(options.concurrency) || 3;
-    const batchSize = parseInt(options.batchSize) || 50;
-
-    // Configure log batcher with custom batch size
-    logBatcher.batchSize = batchSize;
+    const batchSize = parseInt(options.batchSize) || 10;
+    const concurrency = parseInt(options.concurrency) || 10;
 
-    console.log(`🚀 Using concurrency level: ${concurrency}`);
-    console.log(`📦 Using log batch size: ${batchSize}`);
+    if (options.statsOnly) {
+      console.log('📊 Mode: Stats Only - Reading file stats and inserting to uploader table');
+      console.log('🚫 Files will NOT be uploaded');
+      if (options.detect !== false) {
+        console.log('🔍 Document type detection ENABLED for supported files');
+      } else {
+        console.log('🔍 Document type detection DISABLED');
+      }
+    } else {
+      console.log(
+        `🚀 Mode: ${apiMode ? 'Arela API with auto-processing' : 'Direct Supabase'}`,
+      );
+    }
+    console.log(`📦 Batch size: ${batchSize}`);
+    console.log(`⚡ Concurrency: ${concurrency}`);
 
-    const processedPaths = await getProcessedPaths();
+    const processedPaths = getProcessedPaths();
     let globalSuccess = 0;
+    let globalDetected = 0;
+    let globalOrganized = 0;
     let globalFailure = 0;
+    let globalSkipped = 0;
 
     for (const folder of sources) {
       const sourcePath = path.resolve(basePath, folder).replace(/\\/g, '/');
@@ -793,85 +1524,78 @@ program
 
         console.log(`📊 Found ${files.length} files to process`);
 
-        // Process files in parallel batches
-        const { successCount, failureCount, skippedCount } =
-          await processFilesInBatches(
-            files,
-            concurrency,
-            options,
-            basePath,
-            folder,
-            sourcePath,
-            processedPaths,
-          );
-
-        globalSuccess += successCount;
-        globalFailure += failureCount;
-
-        // Small delay to ensure progress bar is fully cleared
-        await delay(100);
-
-        console.log(`\n📦 Upload Summary for ${folder}:`);
-        console.log(`   ✅ Successfully uploaded files: ${successCount}`);
-        console.log(`   ❌ Files with errors: ${failureCount}`);
-        console.log(`   ⏭️  Files skipped (already exist): ${skippedCount}`);
-        console.log(`   📜 Log file: ${logFilePath} \n`);
+        const result = await processFilesInBatches(
+          files,
+          batchSize,
+          options,
+          basePath,
+          folder,
+          sourcePath,
+          processedPaths,
+        );
 
-        // Show cache statistics if requested
-        if (options.showCacheStats) {
-          console.log(`📊 Cache Statistics:`);
-          console.log(
-            `   🗂️  Filename sanitization cache: ${sanitizationCache.size} entries`,
-          );
-          console.log(
-            `   📁 Path sanitization cache: ${pathSanitizationCache.size} entries`,
-          );
-          console.log(
-            `   📋 Log batch pending: ${logBatcher.batch.length} entries`,
-          );
+        globalSuccess += result.successCount;
+        globalDetected += result.detectedCount || 0;
+        globalOrganized += result.organizedCount || 0;
+        globalFailure += result.failureCount;
+        globalSkipped += result.skippedCount;
 
-          // Calculate cache hit rate (rough estimation)
-          const totalProcessed = successCount + failureCount + skippedCount;
-          const estimatedCacheHitRate =
-            totalProcessed > 0
-              ? Math.round(
-                  ((totalProcessed - sanitizationCache.size) / totalProcessed) *
-                    100,
-                )
-              : 0;
-          console.log(
-            `   🎯 Estimated cache hit rate: ${Math.max(0, estimatedCacheHitRate)}%\n`,
-          );
+        console.log(`\n📦 Summary for ${folder}:`);
+        if (options.statsOnly) {
+          console.log(`   📊 Stats recorded: ${result.successCount}`);
+        } else {
+          console.log(`   ✅ Uploaded: ${result.successCount}`);
+          if (apiMode) {
+            console.log(`   🔍 Detected: ${result.detectedCount || 0}`);
+            console.log(`   📁 Organized: ${result.organizedCount || 0}`);
+          }
+        }
+        console.log(`   ❌ Errors: ${result.failureCount}`);
+        if (options.statsOnly) {
+          console.log(`   ⏭️  Duplicates: ${result.skippedCount}`);
+        } else {
+          console.log(`   ⏭️  Skipped: ${result.skippedCount}`);
         }
 
         writeLog(
-          `📦 Upload Summary for folder ${folder}: Success: ${successCount}, Errors: ${failureCount}, Skipped: ${skippedCount}`,
+          `📦 Summary for ${folder}: Success: ${result.successCount}, Detected: ${result.detectedCount || 0}, Organized: ${result.organizedCount || 0}, Errors: ${result.failureCount}, ${options.statsOnly ? 'Duplicates' : 'Skipped'}: ${result.skippedCount}`,
         );
       } catch (err) {
         console.error(`⚠️ Error processing folder ${folder}:`, err.message);
         writeLog(`⚠️ Error processing folder ${folder}: ${err.message}`);
-        await sendLogToSupabase({
-          file: folder,
-          uploadPath: folder,
-          status: 'error',
-          message: err.message,
-        });
         globalFailure++;
       }
     }
 
-    // Force flush any remaining logs before finishing
-    console.log(`\n📤 Flushing remaining logs...`);
-    await logBatcher.forceFlush();
-
-    // Final summary with clear separation
-    console.log(`\n${'='.repeat(50)}`);
-    console.log(`🎯 UPLOAD COMPLETED`);
-    console.log(`${'='.repeat(50)}`);
-    console.log(`   ✅ Total uploaded: ${globalSuccess}`);
-    console.log(`   ❌ Total with errors: ${globalFailure}`);
+    console.log(`\n${'='.repeat(60)}`);
+    if (options.statsOnly) {
+      console.log(`📊 STATS COLLECTION COMPLETED`);
+      console.log(`${'='.repeat(60)}`);
+      console.log(`   📊 Total stats recorded: ${globalSuccess}`);
+    } else {
+      console.log(`🎯 ${apiMode ? 'ARELA API' : 'SUPABASE'} UPLOAD COMPLETED`);
+      console.log(`${'='.repeat(60)}`);
+      console.log(`   ✅ Total uploaded: ${globalSuccess}`);
+      if (apiMode) {
+        console.log(`   🔍 Total detected: ${globalDetected}`);
+        console.log(`   📁 Total organized: ${globalOrganized}`);
+      }
+    }
+    if (options.statsOnly) {
+      console.log(`   ⏭️  Total duplicates: ${globalSkipped}`);
+    } else {
+      console.log(`   ⏭️  Total skipped: ${globalSkipped}`);
+    }
+    console.log(`   ❌ Total errors: ${globalFailure}`);
     console.log(`   📜 Log file: ${logFilePath}`);
-    console.log(`${'='.repeat(50)}\n`);
+    console.log(`${'='.repeat(60)}\n`);
+
+    if (options.showStats && sanitizationCache.size > 0) {
+      console.log(`📊 Performance Statistics:`);
+      console.log(
+        `   🗂️  Sanitization cache entries: ${sanitizationCache.size}`,
+      );
+    }
   });
 
 program.parse();