@arela/uploader 0.1.0 → 0.2.1

package/src/index.js

@@ -3,32 +3,56 @@ import { createClient } from '@supabase/supabase-js';
 import cliProgress from 'cli-progress';
 import { Command } from 'commander';
 import { config } from 'dotenv';
+import FormData from 'form-data';
 import fs from 'fs';
 import { globby } from 'globby';
 import mime from 'mime-types';
-import { createRequire } from 'module';
+import fetch from 'node-fetch';
 import path from 'path';
-
-const require = createRequire(import.meta.url);
-const { version } = require('../package.json');
+import { FileDetectionService } from './file-detection.js';
 
 config();
 
 const program = new Command();
 
+// Read package.json version at startup
+let packageVersion = '1.0.0'; // fallback
+try {
+  const __filename = new URL(import.meta.url).pathname;
+  const __dirname = path.dirname(__filename);
+  const packageJsonPath = path.resolve(__dirname, '../package.json');
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
+  packageVersion = packageJson.version || '1.0.0';
+} catch (error) {
+  console.warn('⚠️ Could not read package.json version, using fallback');
+}
+
+// Configuración de Supabase (original)
 const supabaseUrl = process.env.SUPABASE_URL;
 const supabaseKey = process.env.SUPABASE_KEY;
 const bucket = process.env.SUPABASE_BUCKET;
+
+// Configuración de API (nueva)
+const API_BASE_URL = process.env.ARELA_API_URL;
+const API_TOKEN = process.env.ARELA_API_TOKEN;
+
+// Configuración del uploader mejorado
 const basePath = process.env.UPLOAD_BASE_PATH;
 const sources = process.env.UPLOAD_SOURCES?.split('|')
   .map((s) => s.trim())
   .filter(Boolean);
 
-const supabase = createClient(supabaseUrl, supabaseKey);
+// Configuración de RFCs para upload
+console.log('🔧 Configured RFCs for upload:', process.env.UPLOAD_RFCS);
+const uploadRfcs = process.env.UPLOAD_RFCS?.split('|')
+  .map((s) => s.trim())
+  .filter(Boolean);
+
+let supabase;
+let apiMode = false;
 
-// Pre-compiled regex patterns for better performance
+// Pre-compiled regex patterns for better performance (from original complex uploader)
 const SANITIZATION_PATTERNS = [
-  // Character replacements (grouped for efficiency)
   [/[áàâäãåāăą]/gi, 'a'],
   [/[éèêëēĕėę]/gi, 'e'],
   [/[íìîïīĭį]/gi, 'i'],
@@ -37,579 +61,835 @@ const SANITIZATION_PATTERNS = [
   [/[ñň]/gi, 'n'],
   [/[ç]/gi, 'c'],
   [/[ý]/gi, 'y'],
-  // Korean characters (compiled once)
   [/[멕]/g, 'meok'],
   [/[시]/g, 'si'],
   [/[코]/g, 'ko'],
   [/[용]/g, 'yong'],
   [/[가-힣]/g, 'kr'],
-  // Unicode diacritics (after normalize)
   [/[\u0300-\u036f]/g, ''],
-  // Problematic symbols
   [/[\\?%*:|"<>[\]~`^]/g, '-'],
   [/[{}]/g, '-'],
   [/[&]/g, 'and'],
-  [/[()]/g, ''], // Remove parentheses
-  // Cleanup patterns
-  [/\s+/g, '-'], // Replace spaces with dashes
-  [/-+/g, '-'], // Replace multiple dashes with single dash
-  [/^-+|-+$/g, ''], // Remove leading/trailing dashes
-  [/^\.+/, ''], // Remove leading dots
-  [/[^\w.-]/g, ''], // Remove any remaining non-alphanumeric chars
+  [/[()]/g, ''],
+  [/\s+/g, '-'],
+  [/-+/g, '-'],
+  [/^-+|-+$/g, ''],
+  [/^\.+/, ''],
+  [/[^\w.-]/g, ''],
 ];
 
-// Cache for sanitized filenames to avoid repeated processing
 const sanitizationCache = new Map();
 
-// Enhanced sanitization function with caching and pre-compiled regex
 const sanitizeFileName = (fileName) => {
-  // Check cache first
   if (sanitizationCache.has(fileName)) {
     return sanitizationCache.get(fileName);
   }
 
-  // Get file extension
   const ext = path.extname(fileName);
   const nameWithoutExt = path.basename(fileName, ext);
 
-  // Fast path for already clean filenames
   if (/^[a-zA-Z0-9._-]+$/.test(nameWithoutExt)) {
     const result = fileName;
     sanitizationCache.set(fileName, result);
     return result;
   }
 
-  // Normalize unicode first (more efficient to do once)
   let sanitized = nameWithoutExt.normalize('NFD');
 
-  // Apply all sanitization patterns
   for (const [pattern, replacement] of SANITIZATION_PATTERNS) {
     sanitized = sanitized.replace(pattern, replacement);
   }
 
-  // Ensure the filename is not empty
+  // Additional sanitization for problematic characters
+  sanitized = sanitized
+    .replace(/~/g, '-') // Replace tildes
+    .replace(/\s+/g, '-') // Replace spaces with dashes
+    .replace(/\.+/g, '-') // Replace multiple dots with dashes
+    .replace(/-+/g, '-') // Collapse multiple dashes
+    .replace(/^-+|-+$/g, ''); // Remove leading/trailing dashes
+
   if (!sanitized) {
     sanitized = 'unnamed_file';
   }
 
   const result = sanitized + ext;
-
-  // Cache the result for future use
   sanitizationCache.set(fileName, result);
-
   return result;
 };
 
-// Pre-compiled regex patterns for path sanitization
-const PATH_SANITIZATION_PATTERNS = [
-  [/[\\?%*:|"<>[\]~]/g, '-'],
-  [/ +/g, ' '],
-  [/^\.+/, ''],
-  [/\/+/g, '/'],
-];
+const checkCredentials = async (forceSupabase = false) => {
+  // Force Supabase mode if explicitly requested
+  if (forceSupabase) {
+    console.log('🔧 Force Supabase mode enabled - skipping API');
+    apiMode = false;
+  } else if (API_BASE_URL && API_TOKEN) {
+    console.log(
+      '🌐 API mode enabled - files will be uploaded to Arela API with automatic processing',
+    );
+    apiMode = true;
 
-// Cache for sanitized paths
-const pathSanitizationCache = new Map();
-
-// Batch logging system for performance
-class LogBatcher {
-  constructor(batchSize = 50, flushInterval = 5000) {
-    this.batch = [];
-    this.batchSize = batchSize;
-    this.flushInterval = flushInterval;
-    this.lastFlush = Date.now();
-    this.flushTimer = null;
-  }
-
-  add(logEntry) {
-    this.batch.push({
-      filename: path.basename(logEntry.file),
-      path: logEntry.uploadPath,
-      status: logEntry.status,
-      message: logEntry.message,
-    });
+    try {
+      const response = await fetch(`${API_BASE_URL}/api/health`, {
+        headers: {
+          'x-api-key': API_TOKEN,
+        },
+      });
 
-    // Auto-flush if batch is full or enough time has passed
-    if (
-      this.batch.length >= this.batchSize ||
-      Date.now() - this.lastFlush > this.flushInterval
-    ) {
-      this.flush();
+      if (!response.ok) {
+        console.warn(
+          '⚠️ API connection failed, falling back to direct Supabase upload',
+        );
+        apiMode = false;
+      } else {
+        console.log('✅ Connected to Arela API');
+        return;
+      }
+    } catch (err) {
+      console.warn(
+        '⚠️ API connection failed, falling back to direct Supabase upload',
+      );
+      apiMode = false;
     }
   }
 
-  async flush() {
-    if (this.batch.length === 0) return;
-
-    const logsToSend = [...this.batch];
-    this.batch = [];
-    this.lastFlush = Date.now();
-
-    // Clear any pending timer
-    if (this.flushTimer) {
-      clearTimeout(this.flushTimer);
-      this.flushTimer = null;
+  // Initialize Supabase client if not in API mode or if forced
+  if (!apiMode || forceSupabase) {
+    if (!supabaseUrl || !supabaseKey || !bucket) {
+      console.error(
+        '⚠️ Missing credentials. Please set either:\n' +
+        '   - ARELA_API_URL and ARELA_API_TOKEN for API mode, or\n' +
+        '   - SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET for direct mode',
+      );
+      process.exit(1);
     }
 
+    supabase = createClient(supabaseUrl, supabaseKey);
+
     try {
-      const { error } = await supabase.from('upload_logs').insert(logsToSend);
+      const { error } = await supabase.storage.from(bucket).list('');
       if (error) {
-        console.error(
-          `⚠️ Error saving batch of ${logsToSend.length} logs to Supabase: ${error.message}`,
-        );
-        // Re-add failed logs to batch for retry (optional)
-        this.batch.unshift(...logsToSend);
-      } else {
-        // Only show verbose output if requested
-        if (process.env.LOG_BATCH_VERBOSE === 'true') {
-          console.log(`📊 Flushed ${logsToSend.length} logs to Supabase`);
-        }
+        console.error('⚠️ Error connecting to Supabase:', error.message);
+        process.exit(1);
       }
+      console.log('✅ Connected to Supabase (direct mode)');
     } catch (err) {
-      console.error(`⚠️ Error during batch flush: ${err.message}`);
-      // Re-add failed logs to batch for retry (optional)
-      this.batch.unshift(...logsToSend);
-    }
-  }
-
-  // Schedule auto-flush if not already scheduled
-  scheduleFlush() {
-    if (!this.flushTimer && this.batch.length > 0) {
-      this.flushTimer = setTimeout(() => {
-        this.flush();
-      }, this.flushInterval);
+      console.error('⚠️ Error:', err.message);
+      process.exit(1);
     }
   }
+};
 
-  // Force flush all pending logs (called at end of process)
-  async forceFlush() {
-    if (this.flushTimer) {
-      clearTimeout(this.flushTimer);
-      this.flushTimer = null;
-    }
-    await this.flush();
-  }
-}
+const logFilePath = path.resolve(process.cwd(), 'arela-upload.log');
 
-// Global log batcher instance
-const logBatcher = new LogBatcher();
+/**
+ * OPTIMIZED: Log buffer to reduce I/O operations
+ */
+let logBuffer = [];
+const LOG_BUFFER_SIZE = 100; // Flush every 100 log entries
+let lastFlushTime = Date.now();
+const LOG_FLUSH_INTERVAL = 5000; // Flush every 5 seconds
 
-// Function to manage cache size (prevent memory issues in long sessions)
-const manageCaches = () => {
-  const MAX_CACHE_SIZE = 1000;
+const flushLogBuffer = () => {
+  if (logBuffer.length === 0) return;
 
-  if (sanitizationCache.size > MAX_CACHE_SIZE) {
-    // Keep only the most recent 500 entries
-    const entries = Array.from(sanitizationCache.entries());
-    sanitizationCache.clear();
-    entries.slice(-500).forEach(([key, value]) => {
-      sanitizationCache.set(key, value);
-    });
+  try {
+    const logContent = logBuffer.join('\n') + '\n';
+    fs.appendFileSync(logFilePath, logContent);
+    logBuffer = [];
+    lastFlushTime = Date.now();
+  } catch (error) {
+    console.error(`❌ Error writing to log file: ${error.code} | ${error.message} | path: ${logFilePath}`);
   }
+};
 
-  if (pathSanitizationCache.size > MAX_CACHE_SIZE) {
-    const entries = Array.from(pathSanitizationCache.entries());
-    pathSanitizationCache.clear();
-    entries.slice(-500).forEach(([key, value]) => {
-      pathSanitizationCache.set(key, value);
-    });
+const writeLog = (message) => {
+  try {
+    const timestamp = new Date().toISOString();
+    logBuffer.push(`[${timestamp}] ${message}`);
+
+    // Flush if buffer is full or enough time has passed
+    const now = Date.now();
+    if (logBuffer.length >= LOG_BUFFER_SIZE || (now - lastFlushTime) >= LOG_FLUSH_INTERVAL) {
+      flushLogBuffer();
+    }
+  } catch (error) {
+    console.error(`❌ Error buffering log message: ${error.message}`);
   }
 };
 
-const sanitizePath = (inputPath) => {
-  // Check cache first
-  if (pathSanitizationCache.has(inputPath)) {
-    return pathSanitizationCache.get(inputPath);
+// Ensure logs are flushed on process exit
+process.on('exit', flushLogBuffer);
+process.on('SIGINT', () => {
+  flushLogBuffer();
+  process.exit(0);
+});
+process.on('SIGTERM', () => {
+  flushLogBuffer();
+  process.exit(0);
+});
+
+/**
+ * OPTIMIZED: Conditional logging to reduce console overhead
+ */
+const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true';
+const BATCH_DELAY = parseInt(process.env.BATCH_DELAY) || 100; // Configurable delay between batches
+const PROGRESS_UPDATE_INTERVAL = parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10; // Update progress every N items
+
+const logVerbose = (message) => {
+  if (VERBOSE_LOGGING) {
+    console.log(message);
   }
+};
+const batchReadFileStats = (filePaths) => {
+  const results = [];
 
-  // Fast path for already clean paths
-  if (!/[\\?%*:|"<>[\]~]|^ +|^\.+|\/\/+/.test(inputPath)) {
-    pathSanitizationCache.set(inputPath, inputPath);
-    return inputPath;
+  for (const filePath of filePaths) {
+    try {
+      const stats = fs.statSync(filePath);
+      results.push({ path: filePath, stats, error: null });
+    } catch (error) {
+      results.push({ path: filePath, stats: null, error: error.message });
+    }
   }
 
-  let sanitized = inputPath;
+  return results;
+};
+
+/**
+ * OPTIMIZED: Cache for year/pedimento detection results to avoid redundant parsing
+ */
+const pathDetectionCache = new Map();
 
-  // Apply path sanitization patterns
-  for (const [pattern, replacement] of PATH_SANITIZATION_PATTERNS) {
-    sanitized = sanitized.replace(pattern, replacement);
+/**
+ * OPTIMIZED: Clear the path detection cache (useful for testing or long-running processes)
+ */
+const clearPathDetectionCache = () => {
+  pathDetectionCache.clear();
+};
+
+/**
+ * OPTIMIZED: Get detection results with caching
+ */
+const getCachedPathDetection = (filePath, basePath) => {
+  const cacheKey = `${filePath}|${basePath}`;
+
+  if (pathDetectionCache.has(cacheKey)) {
+    return pathDetectionCache.get(cacheKey);
   }
 
-  // Cache the result
-  pathSanitizationCache.set(inputPath, sanitized);
+  const detection = extractYearAndPedimentoFromPath(filePath, basePath);
+  pathDetectionCache.set(cacheKey, detection);
 
-  return sanitized;
+  return detection;
 };
 
-const sendLogToSupabase = async ({ file, uploadPath, status, message }) => {
-  // Add to batch instead of sending immediately
-  logBatcher.add({ file, uploadPath, status, message });
+/**
+ * Extracts year and pedimento number from file path
+ * Supports patterns like:
+ * - /path/to/2024/4023260/file.pdf
+ * - /path/to/pedimentos/2024/4023260/file.pdf
+ * - /path/to/docs/año2024/ped4023260/file.pdf
+ */
+const extractYearAndPedimentoFromPath = (filePath, basePath) => {
+  try {
+    const relativePath = path.relative(basePath, filePath);
+    const pathParts = relativePath.split(path.sep);
+
+    let year = null;
+    let pedimento = null;
+
+    // Pattern 1: Direct year/pedimento structure (2024/4023260)
+    for (let i = 0; i < pathParts.length - 1; i++) {
+      const part = pathParts[i];
+      const nextPart = pathParts[i + 1];
+
+      // Check if current part looks like a year (2020-2030)
+      const yearMatch = part.match(/^(202[0-9])$/);
+      if (yearMatch && nextPart) {
+        year = yearMatch[1];
+
+        // Check if next part looks like a pedimento (4-8 digits)
+        const pedimentoMatch = nextPart.match(/^(\d{4,8})$/);
+        if (pedimentoMatch) {
+          pedimento = pedimentoMatch[1];
+          break;
+        }
+      }
+    }
+
+    // Pattern 2: Named patterns (año2024, ped4023260)
+    if (!year || !pedimento) {
+      for (const part of pathParts) {
+        if (!year) {
+          const namedYearMatch = part.match(/(?:año|year|anio)(\d{4})/i);
+          if (namedYearMatch) {
+            year = namedYearMatch[1];
+          }
+        }
 
-  // Schedule auto-flush if needed
-  logBatcher.scheduleFlush();
-};
+        if (!pedimento) {
+          const namedPedimentoMatch = part.match(
+            /(?:ped|pedimento|pedi)(\d{4,8})/i,
+          );
+          if (namedPedimentoMatch) {
+            pedimento = namedPedimentoMatch[1];
+          }
+        }
+      }
+    }
 
-// Enhanced version for immediate sending (used for critical errors)
-const sendLogToSupabaseImmediate = async ({
-  file,
-  uploadPath,
-  status,
-  message,
-}) => {
-  const { error } = await supabase.from('upload_logs').insert([
-    {
-      filename: path.basename(file),
-      path: uploadPath,
-      status,
-      message,
-    },
-  ]);
+    // Pattern 3: Loose year detection in any part
+    if (!year) {
+      for (const part of pathParts) {
+        const yearMatch = part.match(/(202[0-9])/);
+        if (yearMatch) {
+          year = yearMatch[1];
+          break;
+        }
+      }
+    }
 
-  if (error) {
-    console.error(
-      `⚠️ Error saving immediate log to Supabase: ${error.message}`,
-    );
+    // Pattern 4: Loose pedimento detection (4-8 consecutive digits)
+    if (!pedimento) {
+      for (const part of pathParts) {
+        const pedimentoMatch = part.match(/(\d{4,8})/);
+        if (pedimentoMatch && pedimentoMatch[1].length >= 4) {
+          pedimento = pedimentoMatch[1];
+          break;
+        }
+      }
+    }
+
+    return { year, pedimento, detected: !!(year && pedimento) };
+  } catch (error) {
+    return {
+      year: null,
+      pedimento: null,
+      detected: false,
+      error: error.message,
+    };
   }
 };
 
-const checkCredentials = async () => {
-  if (!supabaseUrl || !supabaseKey || !bucket) {
-    console.error(
-      '⚠️ Missing Supabase credentials. Please set SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET in your environment variables.',
-    );
-    writeLog('⚠️ Missing Supabase credentials.');
-    await sendLogToSupabaseImmediate({
-      file: 'Error',
-      uploadPath: 'Error',
-      status: 'error',
-      message: 'Missing Supabase credentials.',
-    });
-    process.exit(1);
-  }
+/**
+ * OPTIMIZED: Get processed paths with caching and buffered log reading
+ */
+let processedPathsCache = null;
+let lastLogModTime = 0;
 
+const getProcessedPaths = () => {
   try {
-    const { error } = await supabase.storage.from(bucket).list('');
-    if (error) {
-      console.error('⚠️ Error connecting to Supabase:', error.message);
-      writeLog(`⚠️ Error connecting to Supabase: ${error.message}`);
-      await sendLogToSupabaseImmediate({
-        file: 'Error',
-        uploadPath: 'Error',
-        status: 'error',
-        message: error.message,
-      });
-      process.exit(1);
+    // Check if log file exists
+    if (!fs.existsSync(logFilePath)) {
+      return new Set();
+    }
+
+    // Check if cache is still valid
+    const logStats = fs.statSync(logFilePath);
+    if (processedPathsCache && logStats.mtime.getTime() === lastLogModTime) {
+      return processedPathsCache;
     }
-  } catch (err) {
-    console.error('⚠️ Error:', err.message);
-    writeLog(`⚠️ Error: ${err.message}`);
-    await sendLogToSupabaseImmediate({
-      file: 'Error',
-      uploadPath: 'Error',
-      status: 'error',
-      message: err.message,
+
+    // Read and parse log file
+    const processed = new Set();
+    const content = fs.readFileSync(logFilePath, 'utf-8');
+
+    // Use more efficient regex with global flag
+    const regex = /(SUCCESS|SKIPPED): .*? -> (.+)/g;
+    let match;
+
+    while ((match = regex.exec(content)) !== null) {
+      const path = match[2];
+      if (path) {
+        processed.add(path.trim());
+      }
+    }
+
+    // Update cache
+    processedPathsCache = processed;
+    lastLogModTime = logStats.mtime.getTime();
+
+    return processed;
+  } catch (error) {
+    console.error(`⚠️ Error reading processed paths: ${error.message}`);
+    return new Set();
+  }
+};
+
+/**
+ * Upload files to Arela API with automatic detection and organization
+ */
+const uploadToApi = async (files, options) => {
+  const formData = new FormData();
+
+  files.forEach((file) => {
+    const fileBuffer = fs.readFileSync(file.path);
+    formData.append('files', fileBuffer, {
+      filename: file.name,
+      contentType: file.contentType,
     });
-    process.exit(1);
+  });
+
+  if (bucket) formData.append('bucket', bucket);
+  if (options.prefix) formData.append('prefix', options.prefix);
+
+  // Nueva funcionalidad: estructura de carpetas personalizada
+  let combinedStructure = null;
+  let cachedDetection = null; // Cache detection result to avoid redundant calls
+
+  if (
+    options.folderStructure &&
+    options.autoDetectStructure &&
+    files.length > 0
+  ) {
+    // Combine custom folder structure with auto-detection
+    const firstFile = files[0];
+    cachedDetection = getCachedPathDetection(firstFile.path, process.cwd());
+
+    if (cachedDetection.detected) {
+      const autoStructure = `${cachedDetection.year}/${cachedDetection.pedimento}`;
+      combinedStructure = `${options.folderStructure}/${autoStructure}`;
+      formData.append('folderStructure', combinedStructure);
+      console.log(
+        `📁 Combined folder structure: ${options.folderStructure} + ${autoStructure} = ${combinedStructure}`,
+      );
+    } else {
+      // Fallback to just custom structure if auto-detection fails
+      formData.append('folderStructure', options.folderStructure);
+      console.log(
+        `📁 Using custom folder structure (auto-detection failed): ${options.folderStructure}`,
+      );
+    }
+  } else if (options.folderStructure) {
+    formData.append('folderStructure', options.folderStructure);
+    console.log(`📁 Using custom folder structure: ${options.folderStructure}`);
+  } else if (options.autoDetectStructure && files.length > 0) {
+    // Try to auto-detect from the first file if no explicit structure is provided
+    const firstFile = files[0];
+    cachedDetection = getCachedPathDetection(firstFile.path, process.cwd());
+
+    if (cachedDetection.detected) {
+      const autoStructure = `${cachedDetection.year}/${cachedDetection.pedimento}`;
+      formData.append('folderStructure', autoStructure);
+    }
+  }
+
+  // Si se especifica clientPath para user_metadata
+  if (options.clientPath) {
+    formData.append('clientPath', options.clientPath);
+  }
+
+  formData.append('autoDetect', String(options.autoDetect ?? true));
+  formData.append('autoOrganize', String(options.autoOrganize ?? true));
+  formData.append('batchSize', String(options.batchSize || 10));
+  formData.append('clientVersion', packageVersion);
+
+  const response = await fetch(
+    `${API_BASE_URL}/api/storage/batch-upload-and-process`,
+    {
+      method: 'POST',
+      headers: {
+        'x-api-key': API_TOKEN,
+      },
+      body: formData,
+    },
+  );
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(
+      `API request failed: ${response.status} ${response.statusText} - ${errorText}`,
+    );
   }
+
+  return response.json();
 };
 
-await checkCredentials();
+/**
+ * Upload file directly to Supabase (fallback method)
+ */
+const uploadToSupabase = async (file, uploadPath) => {
+  const content = fs.readFileSync(file);
+  const contentType = mime.lookup(file) || 'application/octet-stream';
 
-const fileExistsInBucket = async (pathInBucket) => {
-  const dir = path.dirname(pathInBucket);
-  const filename = path.basename(pathInBucket);
   const { data, error } = await supabase.storage
     .from(bucket)
-    .list(dir === '.' ? '' : dir, { limit: 1000 });
-  if (error) {
-    console.error(`⚠️ Could not verify duplicate: ${error.message}`);
-    writeLog(`⚠️ Could not verify duplicate: ${error.message}`);
-    await sendLogToSupabaseImmediate({
-      file: 'Error',
-      uploadPath: 'Error',
-      status: 'error',
-      message: error.message,
+    .upload(uploadPath.replace(/\\/g, '/'), content, {
+      upsert: true,
+      contentType,
     });
-    return false;
+
+  if (error) {
+    throw new Error(error.message);
   }
-  return data.some((file) => file.name === filename);
+
+  return data;
 };
 
-const logFilePath = path.resolve(process.cwd(), 'upload.log');
-const writeLog = (message) => {
-  try {
-    const timestamp = new Date().toISOString();
-    fs.appendFileSync(logFilePath, `[${timestamp}] ${message}\n`);
-  } catch (error) {
-    console.error(`❌ Error writing to log file: ${error.message}`);
+/**
+ * Insert file stats into uploader table with document detection
+ */
+const insertStatsToUploaderTable = async (files, options) => {
+  if (!supabase) {
+    throw new Error('Supabase client not initialized. Stats mode requires Supabase connection.');
   }
-};
 
-// Modified to fetch from Supabase first, then fallback to local log
-const getProcessedPaths = async () => {
-  const processed = new Set();
+  const detectionService = new FileDetectionService();
+  const records = [];
 
-  // Try to fetch from Supabase first
-  try {
-    const { data, error } = await supabase
-      .from('upload_logs')
-      .select('path')
-      .in('status', ['success', 'skipped']);
+  for (const file of files) {
+    // OPTIMIZED: Use pre-computed stats if available, otherwise call fs.statSync
+    const stats = file.stats || fs.statSync(file.path);
+    const originalPath = options.clientPath || file.path;
 
-    if (error) {
-      console.warn(
-        `⚠️ Could not fetch processed paths from Supabase: ${error.message}. Falling back to local log.`,
-      );
-      // Fallback to local log if Supabase fetch fails
-      const lines = fs.existsSync(logFilePath)
-        ? fs.readFileSync(logFilePath, 'utf-8').split('\\n')
-        : [];
-      for (const line of lines) {
-        const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
-        if (match) {
-          const [, , path] = match;
-          processed.add(path.trim());
-        }
-      }
-    } else if (data) {
-      data.forEach((log) => {
-        if (log.path) {
-          processed.add(log.path.trim());
-        }
-      });
-      // Also read from local log to ensure any paths logged before this change or during a Supabase outage are included
-      const lines = fs.existsSync(logFilePath)
-        ? fs.readFileSync(logFilePath, 'utf-8').split('\\n')
-        : [];
-      for (const line of lines) {
-        const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
-        if (match) {
-          const [, , pathValue] = match;
-          if (pathValue) {
-            processed.add(pathValue.trim());
+    // Check if record already exists
+    const { data: existingRecords, error: checkError } = await supabase
+      .from('uploader')
+      .select('id, original_path')
+      .eq('original_path', originalPath)
+      .limit(1);
+
+    if (checkError) {
+      console.error(`❌ Error checking for existing record: ${checkError.message}`);
+      continue;
+    }
+
+    if (existingRecords && existingRecords.length > 0) {
+      console.log(`⏭️ Skipping duplicate: ${path.basename(file.path)}`);
+      continue;
+    }
+
+    // Initialize record with basic file stats
+    const record = {
+      document_type: null,
+      size: stats.size,
+      num_pedimento: null,
+      filename: file.originalName || path.basename(file.path),
+      original_path: originalPath,
+      arela_path: null,
+      status: 'stats',
+      rfc: null,
+      message: null
+    };
+
+    // Try to detect document type for supported files
+    if (detectionService.isSupportedFileType(file.path)) {
+      try {
+        const detection = await detectionService.detectFile(file.path);
+
+        if (detection.detectedType) {
+          record.document_type = detection.detectedType;
+          record.num_pedimento = detection.detectedPedimento;
+          record.status = 'detected';
+
+          // Set arela_path for pedimento_simplificado documents
+          if (detection.arelaPath) {
+            record.arela_path = detection.arelaPath;
+          }
+
+          // Extract RFC from fields if available
+          const rfcField = detection.fields.find(f => f.name === 'rfc' && f.found);
+          if (rfcField) {
+            record.rfc = rfcField.value;
+          }
+        } else {
+          record.status = 'not-detected';
+          if (detection.error) {
+            record.message = detection.error;
           }
         }
+      } catch (error) {
+        console.error(`❌ Error detecting ${record.filename}:`, error.message);
+        record.status = 'detection-error';
+        record.message = error.message;
       }
+    } else {
+      record.status = 'unsupported';
+      record.message = 'File type not supported for detection';
     }
-  } catch (e) {
-    console.warn(
-      `⚠️ Error fetching from Supabase or reading local log: ${e.message}. Proceeding with an empty set of processed paths initially.`,
-    );
-    // Ensure local log is still attempted if Supabase connection itself fails
-    const lines = fs.existsSync(logFilePath)
-      ? fs.readFileSync(logFilePath, 'utf-8').split('\\n')
-      : [];
-    for (const line of lines) {
-      const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
-      if (match) {
-        const [, , path] = match;
-        processed.add(path.trim());
-      }
-    }
+
+    records.push(record);
   }
-  return processed;
+
+  if (records.length === 0) {
+    console.log('📝 No new records to insert (all were duplicates or errors)');
+    return [];
+  }
+
+  console.log(`💾 Inserting ${records.length} new records into uploader table...`);
+
+  const { data, error } = await supabase
+    .from('uploader')
+    .insert(records)
+    .select();
+
+  if (error) {
+    throw new Error(`Failed to insert stats records: ${error.message}`);
+  }
+
+  return data;
 };
 
-const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+ * OPTIMIZED: Insert ONLY file stats into uploader table (Phase 1)
+ * No file reading, no detection - just filesystem metadata
+ * Returns summary statistics instead of full records for better performance
+ */
+const insertStatsOnlyToUploaderTable = async (files, options) => {
+  if (!supabase) {
+    throw new Error('Supabase client not initialized. Stats mode requires Supabase connection.');
+  }
 
-const uploadWithRetry = async (uploadFn, maxRetries = 5, delayMs = 2000) => {
-  let attempt = 0;
-  let lastError;
+  const batchSize = 1000; // Large batch size for performance
+  const allRecords = [];
 
-  while (attempt < maxRetries) {
+  // Prepare all file stats data first - OPTIMIZED to use pre-computed stats
+  console.log('📊 Collecting filesystem stats...');
+  for (const file of files) {
     try {
-      const result = await uploadFn();
-      if (!result.error) return result;
-      lastError = result.error;
-      attempt++;
-
-      // Exponential backoff with jitter
-      if (attempt < maxRetries) {
-        const backoffDelay =
-          delayMs * Math.pow(2, attempt - 1) + Math.random() * 1000;
-        console.log(
-          `Retry ${attempt}/${maxRetries} after ${Math.round(backoffDelay)}ms...`,
-        );
-        await delay(backoffDelay);
-      }
+      // Use pre-computed stats if available, otherwise call fs.statSync
+      const stats = file.stats || fs.statSync(file.path);
+      const originalPath = options.clientPath || file.path;
+      const fileExtension = path.extname(file.path).toLowerCase().replace('.', '');
+
+      const record = {
+        document_type: null,
+        size: stats.size,
+        num_pedimento: null,
+        filename: file.originalName || path.basename(file.path),
+        original_path: originalPath,
+        arela_path: null,
+        status: 'fs-stats',
+        rfc: null,
+        message: null,
+        file_extension: fileExtension,
+        created_at: new Date().toISOString(),
+        modified_at: stats.mtime.toISOString()
+      };
+
+      allRecords.push(record);
     } catch (error) {
-      lastError = error;
-      attempt++;
+      console.error(`❌ Error reading stats for ${file.path}:`, error.message);
+    }
+  }
 
-      if (attempt < maxRetries) {
-        const backoffDelay =
-          delayMs * Math.pow(2, attempt - 1) + Math.random() * 1000;
-        console.log(
-          `Retry ${attempt}/${maxRetries} after ${Math.round(backoffDelay)}ms due to exception...`,
-        );
-        await delay(backoffDelay);
+  if (allRecords.length === 0) {
+    console.log('📝 No file stats to insert');
+    return { totalInserted: 0, totalSkipped: 0, totalProcessed: 0 };
+  }
+
+  console.log(`💾 Bulk inserting ${allRecords.length} file stats in batches of ${batchSize}...`);
+
+  let totalInserted = 0;
+  let totalSkipped = 0;
+
+  // Process in batches for optimal performance
+  for (let i = 0; i < allRecords.length; i += batchSize) {
+    const batch = allRecords.slice(i, i + batchSize);
+
+    try {
+      // OPTIMIZED: Use upsert without select to avoid unnecessary data transfer
+      const { error, count } = await supabase
+        .from('uploader')
+        .upsert(batch, {
+          onConflict: 'original_path',
+          ignoreDuplicates: false,
+          count: 'exact'
+        });
+
+      if (error) {
+        console.error(`❌ Error inserting batch ${Math.floor(i / batchSize) + 1}:`, error.message);
+        continue;
       }
+
+      // For upsert operations, we can't easily distinguish between inserts and updates
+      // from the count alone, but we can estimate based on the assumption that most
+      // operations in --stats-only mode are likely new inserts
+      const batchProcessed = batch.length;
+
+      // Since we're using upsert with ignoreDuplicates: false, the count represents
+      // the actual number of rows affected (both inserts and updates)
+      const affected = count || batchProcessed;
+
+      // For simplicity and performance, we'll assume most are new inserts in stats-only mode
+      // This is reasonable since stats-only is typically run on new file sets
+      totalInserted += affected;
+
+      console.log(`✅ Batch ${Math.floor(i / batchSize) + 1}: ${affected} rows processed`);
+    } catch (error) {
+      console.error(`❌ Unexpected error in batch ${Math.floor(i / batchSize) + 1}:`, error.message);
     }
   }
 
+  // Calculate skipped as difference between total records and inserted
+  totalSkipped = allRecords.length - totalInserted;
+
+  console.log(`📊 Phase 1 Summary: ${totalInserted} records processed, estimated ${totalSkipped} were updates`);
+
   return {
-    error: new Error(
-      `Max retries exceeded. Last error: ${lastError?.message || 'Unknown error'}`,
-    ),
+    totalInserted,
+    totalSkipped,
+    totalProcessed: allRecords.length
   };
 };
 
-// Function to process a single file
-const processFile = async (
-  file,
-  options,
-  basePath,
-  folder,
-  sourcePath,
-  processedPaths,
-) => {
-  let currentFile = file;
-  let result = {
-    success: false,
-    skipped: false,
-    error: null,
-    message: '',
-  };
+/**
+ * PHASE 2: Process PDF files for pedimento-simplificado detection
+ * Only processes files with status 'fs-stats' and file_extension 'pdf'
+ */
+const detectPedimentosInDatabase = async (options = {}) => {
+  if (!supabase) {
+    throw new Error('Supabase client not initialized.');
+  }
 
-  try {
-    // Check if we need to rename the file
-    if (options.renameFiles) {
-      const originalName = path.basename(file);
-      const sanitizedName = sanitizeFileName(originalName);
-
-      if (originalName !== sanitizedName) {
-        const newFilePath = path.join(path.dirname(file), sanitizedName);
-
-        if (options.dryRun) {
-          result.message = `Would rename: ${originalName} → ${sanitizedName}`;
-          result.skipped = true;
-          return result;
-        } else {
-          try {
-            fs.renameSync(file, newFilePath);
-            currentFile = newFilePath;
-            writeLog(`RENAMED: ${originalName} → ${sanitizedName}`);
-            await sendLogToSupabase({
-              file: originalName,
-              uploadPath: sanitizedName,
-              status: 'renamed',
-              message: `Renamed from ${originalName}`,
-            });
-          } catch (renameError) {
-            result.error = `Failed to rename ${originalName}: ${renameError.message}`;
-            writeLog(`RENAME_ERROR: ${originalName} | ${renameError.message}`);
-            return result;
-          }
-        }
-      }
-    }
+  console.log('🔍 Phase 2: Starting PDF detection for pedimento-simplificado documents...');
 
-    const content = fs.readFileSync(currentFile);
-    const relativePathRaw = path
-      .relative(basePath, currentFile)
-      .replace(/^[\\/]+/, '')
-      .replace(/\\/g, '/');
-
-    // Always sanitize the filename for upload path
-    const pathParts = relativePathRaw.split('/');
-    const originalFileName = pathParts[pathParts.length - 1];
-    const sanitizedFileName = sanitizeFileName(originalFileName);
-    pathParts[pathParts.length - 1] = sanitizedFileName;
-    const sanitizedRelativePath = pathParts.join('/');
-
-    const uploadPathRaw = options.prefix
-      ? path.posix.join(options.prefix, sanitizedRelativePath)
-      : sanitizedRelativePath;
-    const uploadPath = sanitizePath(uploadPathRaw);
-
-    if (
-      uploadPath !== uploadPathRaw ||
-      originalFileName !== sanitizedFileName
-    ) {
-      writeLog(`SANITIZED: ${relativePathRaw} → ${uploadPath}`);
-      await sendLogToSupabase({
-        file: currentFile,
-        uploadPath: relativePathRaw,
-        status: 'sanitized',
-        message: `Sanitized to ${uploadPath} (Arela Version: ${version})`,
-      });
+  // Get all PDF files that need detection (status = 'fs-stats' and extension = 'pdf')
+  let allPdfRecords = [];
+  let hasMore = true;
+  let offset = 0;
+  const queryBatchSize = 1000;
+
+  console.log('📥 Fetching PDF files from database...');
+
+  while (hasMore) {
+    const { data: batch, error: queryError } = await supabase
+      .from('uploader')
+      .select('id, original_path, filename, file_extension, status')
+      .eq('status', 'fs-stats')
+      .eq('file_extension', 'pdf')
+      .ilike('filename', '%simp%')
+      .range(offset, offset + queryBatchSize - 1);
+
+    if (queryError) {
+      throw new Error(`Failed to fetch PDF records: ${queryError.message}`);
     }
 
-    if (processedPaths.has(uploadPath)) {
-      result.skipped = true;
-      result.message = `Already processed (log): ${currentFile}`;
-      return result;
+    if (!batch || batch.length === 0) {
+      hasMore = false;
+    } else {
+      allPdfRecords.push(...batch);
+      offset += queryBatchSize;
+      console.log(`📄 Fetched ${batch.length} PDF records (total: ${allPdfRecords.length})`);
     }
+  }
 
-    const contentType = mime.lookup(currentFile) || 'application/octet-stream';
+  if (allPdfRecords.length === 0) {
+    console.log('📝 No PDF files found for detection');
+    return { detectedCount: 0, processedCount: 0, errorCount: 0 };
+  }
 
-    const exists = await fileExistsInBucket(uploadPath);
+  console.log(`🔍 Processing ${allPdfRecords.length} PDF files for detection...`);
 
-    if (exists) {
-      result.skipped = true;
-      result.message = `Skipped (already exists): ${currentFile}`;
-      writeLog(`SKIPPED: ${currentFile} -> ${uploadPath}`);
-      await sendLogToSupabase({
-        file: currentFile,
-        uploadPath,
-        status: 'skipped',
-        message: 'Already exists in bucket',
-      });
-      return result;
-    }
-
-    const { error } = await uploadWithRetry(() =>
-      supabase.storage.from(bucket).upload(uploadPath, content, {
-        upsert: true,
-        contentType,
-        metadata: {
-          originalName: path.basename(currentFile),
-          sanitizedName: path.basename(uploadPath),
-          clientPath: path.posix.join(
-            basePath,
-            folder,
-            path.relative(sourcePath, currentFile).replace(/\\/g, '/'),
-          ),
-          arelaVersion: version,
-        },
-      }),
-    );
+  const detectionService = new FileDetectionService();
+  const batchSize = parseInt(options.batchSize) || 10; // Smaller batches for file I/O
+  let totalDetected = 0;
+  let totalProcessed = 0;
+  let totalErrors = 0;
 
-    if (error) {
-      result.error = error.message || JSON.stringify(error);
-      writeLog(`ERROR: ${currentFile} -> ${uploadPath} | ${result.error}`);
-      await sendLogToSupabase({
-        file: currentFile,
-        uploadPath,
-        status: 'error',
-        message: result.error,
-      });
-    } else {
-      result.success = true;
-      result.message = `Uploaded ${currentFile} -> ${uploadPath}`;
-      writeLog(`SUCCESS: ${currentFile} -> ${uploadPath}`);
-      await sendLogToSupabase({
-        file: currentFile,
-        uploadPath,
-        status: 'success',
-        message: 'Uploaded successfully',
-      });
+  // Create progress bar
+  const progressBar = new cliProgress.SingleBar({
+    format: '🔍 PDF Detection |{bar}| {percentage}% | {value}/{total} | Detected: {detected} | Errors: {errors}',
+    barCompleteChar: '█',
+    barIncompleteChar: '░',
+    hideCursor: true,
+  });
+
+  progressBar.start(allPdfRecords.length, 0, { detected: 0, errors: 0 });
+
+  // Process files in smaller batches to avoid overwhelming the system
+  for (let i = 0; i < allPdfRecords.length; i += batchSize) {
+    const batch = allPdfRecords.slice(i, i + batchSize);
+    const updatePromises = [];
+
+    for (const record of batch) {
+      try {
+        // Check if file still exists
+        if (!fs.existsSync(record.original_path)) {
+          updatePromises.push(
+            supabase
+              .from('uploader')
+              .update({
+                status: 'file-not-found',
+                message: 'File no longer exists at original path'
+              })
+              .eq('id', record.id)
+          );
+          totalErrors++;
+          continue;
+        }
+
+        // Perform detection
+        const detection = await detectionService.detectFile(record.original_path);
+        totalProcessed++;
+
+        const updateData = {
+          status: detection.detectedType ? 'detected' : 'not-detected',
+          document_type: detection.detectedType,
+          num_pedimento: detection.detectedPedimento,
+          arela_path: detection.arelaPath,
+          message: detection.error || null
+        };
+
+        // Extract RFC from fields if available
+        if (detection.fields) {
+          const rfcField = detection.fields.find(f => f.name === 'rfc' && f.found);
+          if (rfcField) {
+            updateData.rfc = rfcField.value;
+          }
+        }
+
+        if (detection.detectedType) {
+          totalDetected++;
+        }
+
+        updatePromises.push(
+          supabase
+            .from('uploader')
+            .update(updateData)
+            .eq('id', record.id)
+        );
+
+      } catch (error) {
+        console.error(`❌ Error detecting ${record.filename}:`, error.message);
+        totalErrors++;
+
+        updatePromises.push(
+          supabase
+            .from('uploader')
+            .update({
+              status: 'detection-error',
+              message: error.message
+            })
+            .eq('id', record.id)
+        );
+      }
+    }
+
+    // Execute all updates in parallel for this batch
+    try {
+      await Promise.all(updatePromises);
+    } catch (error) {
+      console.error(`❌ Error updating batch:`, error.message);
     }
-  } catch (err) {
-    result.error = err.message || JSON.stringify(err);
-    writeLog(`ERROR: ${currentFile} | ${result.error}`);
-    await sendLogToSupabase({
-      file: currentFile,
-      uploadPath: currentFile,
-      status: 'error',
-      message: result.error,
+
+    // Update progress
+    progressBar.update(Math.min(i + batchSize, allPdfRecords.length), {
+      detected: totalDetected,
+      errors: totalErrors
     });
   }
 
-  return result;
+  progressBar.stop();
+
+  console.log(`📊 Phase 2 Summary: ${totalDetected} detected, ${totalProcessed} processed, ${totalErrors} errors`);
+  return {
+    detectedCount: totalDetected,
+    processedCount: totalProcessed,
+    errorCount: totalErrors
+  };
 };
 
-// Function to process files in parallel batches
 const processFilesInBatches = async (
   files,
   batchSize,
@@ -619,13 +899,20 @@ const processFilesInBatches = async (
   sourcePath,
   processedPaths,
 ) => {
-  let successCount = 0;
-  let failureCount = 0;
-  let skippedCount = 0;
+  let totalUploaded = 0;
+  let totalDetected = 0;
+  let totalOrganized = 0;
+  let totalErrors = 0;
+  let totalSkipped = 0;
+
+  const messageBuffer = [];
+
+  const progressBarFormat = options.statsOnly
+    ? '📊 Processing [{bar}] {percentage}% | {value}/{total} files | Stats: {successCount} | Errors: {failureCount} | Duplicates: {skippedCount}'
+    : '📂 Processing [{bar}] {percentage}% | {value}/{total} files | Success: {successCount} | Errors: {failureCount} | Skipped: {skippedCount}';
 
   const progressBar = new cliProgress.SingleBar({
-    format:
-      '📂 Processing [{bar}] {percentage}% | {value}/{total} files | Success: {successCount} | Errors: {failureCount} | Skipped: {skippedCount}',
+    format: progressBarFormat,
     barCompleteChar: '█',
     barIncompleteChar: '░',
     hideCursor: true,
@@ -637,101 +924,928 @@ const processFilesInBatches = async (
     skippedCount: 0,
   });
 
-  for (let i = 0; i < files.length; i += batchSize) {
-    const batch = files.slice(i, i + batchSize);
+  if (options.statsOnly) {
+    // OPTIMIZED Stats-only mode - Only read filesystem stats, no file detection
+    console.log('📊 Phase 1: Processing files in optimized stats-only mode (no detection)...');
+
+    for (let i = 0; i < files.length; i += batchSize) {
+      const batch = files.slice(i, i + batchSize);
+
+      // OPTIMIZED: Batch read file stats to reduce I/O overhead
+      const fileStatsResults = batchReadFileStats(batch);
+      const statsFiles = fileStatsResults
+        .filter(result => result.stats !== null) // Only include files with valid stats
+        .map((result) => {
+          const originalFileName = path.basename(result.path);
+
+          return {
+            path: result.path,
+            originalName: originalFileName,
+            stats: result.stats, // Pass pre-computed stats to avoid redundant calls
+          };
+        });
 
-    // Process batch in parallel
-    const batchResults = await Promise.all(
-      batch.map((file) =>
-        processFile(
-          file,
-          options,
-          basePath,
-          folder,
-          sourcePath,
-          processedPaths,
-        ),
-      ),
-    );
+      // Log any files that couldn't be read
+      const failedFiles = fileStatsResults.filter(result => result.error !== null);
+      if (failedFiles.length > 0) {
+        console.log(`⚠️ Could not read stats for ${failedFiles.length} files in batch`);
+        failedFiles.forEach(failed => {
+          console.error(`   ❌ ${failed.path}: ${failed.error}`);
+        });
+      }
+
+      try {
+        const result = await insertStatsOnlyToUploaderTable(statsFiles, options);
+
+        totalUploaded += result.totalInserted;
+        totalSkipped += result.totalSkipped;
+        totalErrors += failedFiles.length; // Count failed file reads as errors
+
+        progressBar.update(Math.min(i + batch.length, files.length), {
+          successCount: totalUploaded,
+          failureCount: totalErrors,
+          skippedCount: totalSkipped,
+        });
+
+      } catch (error) {
+        console.error(`❌ Error processing stats batch:`, error.message);
+        totalErrors += batch.length;
+
+        progressBar.update(Math.min(i + batch.length, files.length), {
+          successCount: totalUploaded,
+          failureCount: totalErrors,
+          skippedCount: totalSkipped,
+        });
+      }
+    }
+  } else if (apiMode && !options.forceSupabase) {
+    // API Mode - Process in batches
+    for (let i = 0; i < files.length; i += batchSize) {
+      const batch = files.slice(i, i + batchSize);
+      let sanitizedRelativePath;
+
+      const apiFiles = batch
+        .map((file) => {
+          const relativePathRaw = path
+            .relative(basePath, file)
+            .replace(/^[\\/]+/, '')
+            .replace(/\\/g, '/');
+
+          const pathParts = relativePathRaw.split('/');
+          const originalFileName = pathParts[pathParts.length - 1];
+          const sanitizedFileName = sanitizeFileName(originalFileName);
+          pathParts[pathParts.length - 1] = sanitizedFileName;
+          sanitizedRelativePath = pathParts.join('/');
+
+          let uploadPath;
+
+          // Handle combined folder structure + auto-detection
+          if (options.folderStructure && options.autoDetectStructure) {
+            // OPTIMIZED: Use cached detection to avoid redundant parsing
+            const detection = getCachedPathDetection(file, basePath);
+            if (detection.detected) {
+              const autoStructure = `${detection.year}/${detection.pedimento}`;
+              const combinedStructure = `${options.folderStructure}/${autoStructure}`;
+              uploadPath = path.posix.join(
+                combinedStructure,
+                sanitizedFileName,
+              );
+              logVerbose(
+                `📁 Combined structure: ${options.folderStructure}/${autoStructure} for ${originalFileName} -> ${uploadPath}`,
+              );
+            } else {
+              // Fallback to just custom structure if auto-detection fails
+              uploadPath = path.posix.join(
+                options.folderStructure,
+                sanitizedFileName,
+              );
+              logVerbose(
+                `📁 Custom structure (auto-detection failed): ${uploadPath}`,
+              );
+            }
+          } else if (options.folderStructure) {
+            // Use custom folder structure only
+            uploadPath = path.posix.join(
+              options.folderStructure,
+              sanitizedFileName,
+            );
+            logVerbose(`📁 Custom structure: ${uploadPath}`);
+          } else if (options.autoDetectStructure) {
+            // Auto-detect structure from path if enabled - OPTIMIZED: Use cached detection
+            const detection = getCachedPathDetection(file, basePath);
+            if (detection.detected) {
+              const autoStructure = `${detection.year}/${detection.pedimento}`;
+              uploadPath = path.posix.join(autoStructure, sanitizedFileName);
+              console.log(
+                `🔍 Auto-detected: ${autoStructure} for ${originalFileName} -> ${uploadPath}`,
+              );
+            } else {
+              uploadPath = options.prefix
+                ? path.posix.join(options.prefix, sanitizedRelativePath)
+                : sanitizedRelativePath;
+              console.log(`📁 Using relative path: ${uploadPath}`);
+            }
+          } else {
+            uploadPath = options.prefix
+              ? path.posix.join(options.prefix, sanitizedRelativePath)
+              : sanitizedRelativePath;
+            console.log(`📁 Using standard path: ${uploadPath}`);
+          }
+
+          if (processedPaths.has(uploadPath)) {
+            totalSkipped++;
+            writeLog(`SKIPPED: ${file} -> ${uploadPath}`);
+            return null;
+          }
+
+          return {
+            path: file,
+            name: sanitizedFileName,
+            originalName: originalFileName,
+            uploadPath: uploadPath.replace(/\\/g, '/'), // Ensure forward slashes
+            contentType: mime.lookup(file) || 'application/octet-stream',
+          };
+        })
+        .filter(Boolean);
+
+      if (apiFiles.length > 0) {
+        // console.log(`🔄 Processing batch of ${apiFiles.length} files`);
+        // apiFiles.forEach(f => console.log(`   📄 ${f.name} -> ${f.uploadPath}`));
+
+        try {
+          // Use clientPath from options if specified, otherwise construct from detection or folder
+          let clientPath = options.clientPath;
+
+          if (!clientPath && apiFiles.length > 0) {
+            const firstFile = apiFiles[0];
+            // OPTIMIZED: Use cached detection to avoid redundant parsing
+            const detection = getCachedPathDetection(firstFile.path, basePath);
+            if (detection.detected) {
+              // clientPath = `${detection.year}/${detection.pedimento}/`;
+              clientPath = path
+                .resolve(basePath, sanitizedRelativePath)
+                .replace(/\\/g, '/');
+            } else {
+              // Fallback to folder structure if no year/pedimento detected
+              clientPath = path.resolve(basePath, folder).replace(/\\/g, '/');
+            }
+          }
+
+          const result = await uploadToApi(apiFiles, {
+            ...options,
+            clientPath: clientPath,
+          });
+
+          totalUploaded += result.stats.uploadedCount;
+          totalDetected += result.stats.detectedCount;
+          totalOrganized += result.stats.organizedCount;
+          totalErrors += result.stats.errorCount;
+
+          result.uploaded.forEach((upload) => {
+            const apiFile = apiFiles.find(
+              (f) => f.name === upload.originalName,
+            );
+            if (apiFile) {
+              writeLog(`SUCCESS: ${apiFile.path} -> ${apiFile.uploadPath}`);
+              processedPaths.add(apiFile.uploadPath);
+            }
+          });
+
+          result.errors.forEach((error) => {
+            writeLog(
+              `ERROR: ${error.fileName}: ${error.error} (${error.step})`,
+            );
+            messageBuffer.push(
+              `❌ ${error.fileName}: ${error.error} (${error.step})`,
+            );
+          });
+        } catch (error) {
+          totalErrors += apiFiles.length;
+          apiFiles.forEach((file) => {
+            writeLog(`ERROR: ${file.path}: ${error.message}`);
+            messageBuffer.push(`❌ ${file.name}: ${error.message}`);
+          });
+        }
+      }
+
+      progressBar.update(i + batch.length, {
+        successCount: totalUploaded,
+        failureCount: totalErrors,
+        skippedCount: totalSkipped,
+      });
+
+      if (i + batchSize < files.length) {
+        await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY));
+      }
+    }
+  } else {
+    // Direct Supabase mode
+    for (let i = 0; i < files.length; i++) {
+      const file = files[i];
+      try {
+        const relativePath = path.relative(basePath, file);
+        let uploadPath;
+
+        // Handle combined folder structure + auto-detection
+        if (options.folderStructure && options.autoDetectStructure) {
+          const detection = getCachedPathDetection(file, basePath);
+          if (detection.detected) {
+            const autoStructure = `${detection.year}/${detection.pedimento}`;
+            const combinedStructure = `${options.folderStructure}/${autoStructure}`;
+            const fileName = path.basename(file);
+            uploadPath = path.join(combinedStructure, fileName);
+            console.log(
+              `📁 Combined structure: ${options.folderStructure}/${autoStructure} for ${fileName}`,
+            );
+          } else {
+            // Fallback to just custom structure if auto-detection fails
+            const fileName = path.basename(file);
+            uploadPath = path.join(options.folderStructure, fileName);
+            console.log(
+              `📁 Custom structure (auto-detection failed): ${uploadPath}`,
+            );
+          }
+        } else if (options.folderStructure) {
+          // Use custom folder structure only
+          const fileName = path.basename(file);
+          uploadPath = path.join(options.folderStructure, fileName);
+          console.log(`📁 Custom structure: ${uploadPath}`);
+        } else if (options.autoDetectStructure) {
+          // Auto-detect structure from path if enabled - OPTIMIZED: Use cached detection
+          const detection = getCachedPathDetection(file, basePath);
+          if (detection.detected) {
+            const autoStructure = `${detection.year}/${detection.pedimento}`;
+            const fileName = path.basename(file);
+            uploadPath = path.join(autoStructure, fileName);
+          } else {
+            uploadPath = options.prefix
+              ? path.join(options.prefix, relativePath)
+              : relativePath;
+          }
+        } else {
+          uploadPath = options.prefix
+            ? path.join(options.prefix, relativePath)
+            : relativePath;
+        }
 
-    // Update counters and progress
-    for (const result of batchResults) {
-      if (result.success) {
-        successCount++;
-      } else if (result.skipped) {
-        skippedCount++;
-      } else if (result.error) {
-        failureCount++;
-        console.error(`❌ ${result.error}`);
+        if (processedPaths.has(uploadPath)) {
+          totalSkipped++;
+          writeLog(`SKIPPED: ${file} -> ${uploadPath}`);
+        } else {
+          await uploadToSupabase(file, uploadPath);
+          totalUploaded++;
+          writeLog(`SUCCESS: ${file} -> ${uploadPath}`);
+          processedPaths.add(uploadPath);
+        }
+      } catch (error) {
+        totalErrors++;
+        writeLog(`ERROR: ${file}: ${error.message}`);
+        messageBuffer.push(`❌ ${path.basename(file)}: ${error.message}`);
       }
 
-      if (result.message && !result.error) {
-        console.log(`✅ ${result.message}`);
+      progressBar.update(i + 1, {
+        successCount: totalUploaded,
+        failureCount: totalErrors,
+        skippedCount: totalSkipped,
+      });
+    }
+  }
+
+  progressBar.stop();
+
+  const errorMessages = messageBuffer.filter((msg) => msg.startsWith('❌'));
+  if (errorMessages.length > 0) {
+    console.log('\n🚨 Errors encountered during processing:');
+    errorMessages.forEach((msg) => console.error(msg));
+  }
+
+  return {
+    successCount: totalUploaded,
+    detectedCount: totalDetected,
+    organizedCount: totalOrganized,
+    failureCount: totalErrors,
+    skippedCount: totalSkipped,
+  };
+};
+
+/**
+ * Upload files to Arela API based on specific RFC values
+ */
+const uploadFilesByRfc = async (options = {}) => {
+  if (!supabase) {
+    console.error('❌ Supabase client not initialized');
+    process.exit(1);
+  }
+
+  if (!API_BASE_URL || !API_TOKEN) {
+    console.error('❌ Arela API configuration missing. Please set ARELA_API_URL and ARELA_API_TOKEN environment variables.');
+    process.exit(1);
+  }
+
+  if (!uploadRfcs || uploadRfcs.length === 0) {
+    console.error('❌ No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.');
+    console.error('   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"');
+    process.exit(1);
+  }
+
+  console.log('🎯 RFC-based Upload Mode');
+  console.log(`📋 Target RFCs: ${uploadRfcs.join(', ')}`);
+  console.log('🔍 Searching for files to upload...');
+
+  // Step 1: Get all records that match the specified RFCs and have arela_path
+  const { data: rfcRecords, error: rfcError } = await supabase
+    .from('uploader')
+    .select('arela_path')
+    .in('rfc', uploadRfcs)
+    .not('arela_path', 'is', null);
+
+  if (rfcError) {
+    console.error('❌ Error fetching RFC records:', rfcError.message);
+    return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
+  }
+
+  if (!rfcRecords || rfcRecords.length === 0) {
+    console.log('ℹ️  No files found for the specified RFCs with arela_path');
+    console.log(`   Make sure files for RFCs [${uploadRfcs.join(', ')}] have been processed and have arela_path values`);
+    return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+  }
+
+  // Step 2: Get unique arela_paths from the RFC matches
+  const uniqueArelaPaths = [...new Set(rfcRecords.map(r => r.arela_path))];
+  console.log(`� Found ${uniqueArelaPaths.length} unique arela_path(s) for the specified RFCs`);
+
+  // Step 3: Get ALL files that have these arela_paths (including supporting documents)
+  // Use pagination to ensure we get all files, regardless of count
+  let allRelatedFiles = [];
+  let hasMore = true;
+  let offset = 0;
+  const queryBatchSize = 1000;
+
+  console.log('📥 Fetching all related files (with pagination)...');
+
+  while (hasMore) {
+    const { data: batch, error: queryError } = await supabase
+      .from('uploader')
+      .select('id, original_path, arela_path, filename, rfc, document_type')
+      .in('arela_path', uniqueArelaPaths)
+      .not('original_path', 'is', null)
+      .range(offset, offset + queryBatchSize - 1);
+
+    if (queryError) {
+      console.error('❌ Error fetching related files:', queryError.message);
+      return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
+    }
+
+    if (!batch || batch.length === 0) {
+      hasMore = false;
+    } else {
+      allRelatedFiles = allRelatedFiles.concat(batch);
+      offset += queryBatchSize;
+
+      // If we got less than queryBatchSize, we've reached the end
+      if (batch.length < queryBatchSize) {
+        hasMore = false;
       }
     }
+  }
+
+  if (!allRelatedFiles || allRelatedFiles.length === 0) {
+    console.log('ℹ️  No related files found for the arela_paths');
+    return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
+  }
+
+  console.log(`📁 Found ${allRelatedFiles.length} total files to upload (including supporting documents)`);
+
+  // Group by RFC and arela_path for better organization
+  const filesByRfc = allRelatedFiles.reduce((acc, record) => {
+    const rfc = record.rfc || 'No RFC';
+    if (!acc[rfc]) {
+      acc[rfc] = [];
+    }
+    acc[rfc].push(record);
+    return acc;
+  }, {});
+
+  console.log('📊 Files by RFC (including supporting documents):');
+  for (const [rfc, files] of Object.entries(filesByRfc)) {
+    const documentTypes = [...new Set(files.map(f => f.document_type || 'Unknown'))];
+    console.log(`   ${rfc}: ${files.length} files (${documentTypes.join(', ')})`);
+  }
+
+  // Group by arela_path for upload organization
+  const filesByPath = allRelatedFiles.reduce((acc, record) => {
+    const path = record.arela_path;
+    if (!acc[path]) {
+      acc[path] = [];
+    }
+    acc[path].push(record);
+    return acc;
+  }, {});
+
+  console.log('� Files grouped by arela_path:');
+  for (const [path, files] of Object.entries(filesByPath)) {
+    console.log(`   ${path}: ${files.length} files`);
+  }
+
+  let totalProcessed = 0;
+  let totalUploaded = 0;
+  let totalErrors = 0;
+  let totalSkipped = 0;
+
+  // Create progress bar
+  const progressBar = new cliProgress.SingleBar({
+    format: '🚀 Uploading files |{bar}| {percentage}% | {value}/{total} | Uploaded: {uploaded} | Errors: {errors} | Skipped: {skipped}',
+    barCompleteChar: '█',
+    barIncompleteChar: '░',
+    hideCursor: true,
+  });
 
-    progressBar.update(i + batch.length, {
-      successCount,
-      failureCount,
-      skippedCount,
+  if (options.showProgress !== false) {
+    progressBar.start(allRelatedFiles.length, 0, {
+      uploaded: 0,
+      errors: 0,
+      skipped: 0,
     });
+  }
+
+  const batchSize = parseInt(options.batchSize) || 10;
+  console.log(`📦 Processing in batches of ${batchSize} files`);
+
+  // Process files in batches
+  for (let i = 0; i < allRelatedFiles.length; i += batchSize) {
+    const batch = allRelatedFiles.slice(i, i + batchSize);
+    const batchNumber = Math.floor(i / batchSize) + 1;
+    const totalBatches = Math.ceil(allRelatedFiles.length / batchSize);
+
+    console.log(`\n📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} files)`);
+
+    // Prepare files for upload
+    const filesToUpload = [];
+
+    for (const record of batch) {
+      totalProcessed++;
+
+      try {
+        const originalPath = record.original_path;
+
+        // Check if file exists
+        if (!fs.existsSync(originalPath)) {
+          console.log(`   ⚠️  File not found: ${originalPath}`);
+          totalSkipped++;
+          continue;
+        }
+
+        // OPTIMIZED: Read file and get size from buffer instead of separate fs.statSync call
+        const fileBuffer = fs.readFileSync(originalPath);
+
+        filesToUpload.push({
+          path: originalPath,
+          buffer: fileBuffer,
+          size: fileBuffer.length, // Get size from buffer instead of fs.statSync
+          name: record.filename,
+          arelaPath: record.arela_path,
+          rfc: record.rfc,
+          documentType: record.document_type,
+        });
+
+      } catch (error) {
+        console.error(`   ❌ Error reading file ${record.original_path}:`, error.message);
+        totalErrors++;
+      }
+
+      if (options.showProgress !== false) {
+        progressBar.update(totalProcessed, {
+          uploaded: totalUploaded,
+          errors: totalErrors,
+          skipped: totalSkipped,
+        });
+      }
+    }
+
+    // Upload the batch if we have files
+    if (filesToUpload.length > 0) {
+      try {
+        console.log(`   🚀 Uploading ${filesToUpload.length} files to Arela API...`);
+
+        const formData = new FormData();
+
+        // Add files to form data
+        filesToUpload.forEach((file, index) => {
+          formData.append(`files`, file.buffer, {
+            filename: file.name,
+            contentType: mime.lookup(file.name) || 'application/octet-stream',
+          });
+        });
+
+        // Instead of using per-file folder structures, we'll group by arela_path and upload separately
+        // Group files by their arela_path to upload them in correct structure
+        const filesByPath = filesToUpload.reduce((acc, file) => {
+          const path = file.arelaPath.replace(/\/$/, '');
+          if (!acc[path]) {
+            acc[path] = [];
+          }
+          acc[path].push(file);
+          return acc;
+        }, {});
+
+        // Upload each group separately with its folder structure
+        for (const [arelaPath, pathFiles] of Object.entries(filesByPath)) {
+          const pathFormData = new FormData();
+
+          pathFiles.forEach((file) => {
+            pathFormData.append('files', file.buffer, {
+              filename: file.name,
+              contentType: mime.lookup(file.name) || 'application/octet-stream',
+            });
+          });
+
+          // Set folder structure for this group - concatenate custom prefix with arela_path
+          const folderStructure = options.folderStructure
+            ? `${options.folderStructure}/${arelaPath}`.replace(/\/+/g, '/').replace(/\/$/, '')
+            : arelaPath;
+          pathFormData.append('folderStructure', folderStructure);
+          pathFormData.append('autoDetect', 'true');
+          pathFormData.append('autoOrganize', 'false');
+          pathFormData.append('batchSize', String(pathFiles.length));
+          pathFormData.append('clientVersion', packageVersion);
+          if (bucket) {
+            pathFormData.append('bucket', bucket);
+          }
 
-    // Manage cache size periodically (every 100 files processed)
-    if ((i + batch.length) % 100 === 0) {
-      manageCaches();
-      // Also flush logs every 100 files to maintain responsiveness
-      await logBatcher.flush();
+          console.log(`     📁 Uploading ${pathFiles.length} files to: ${folderStructure}`);
+
+          const response = await fetch(`${API_BASE_URL}/api/storage/batch-upload-and-process`, {
+            method: 'POST',
+            headers: {
+              'x-api-key': API_TOKEN,
+            },
+            body: pathFormData,
+          });
+
+          if (!response.ok) {
+            const errorText = await response.text();
+            throw new Error(`HTTP ${response.status}: ${errorText}`);
+          }
+
+          const result = await response.json();
+
+          // Check if upload was successful based on stats rather than success field
+          const isSuccessful = result.stats && result.stats.uploadedCount > 0 && result.stats.errorCount === 0;
+
+          if (isSuccessful) {
+            console.log(`     ✅ Group uploaded: ${result.stats.uploadedCount} files to ${folderStructure}`);
+            totalUploaded += result.stats.uploadedCount;
+
+            if (result.stats.detectedCount > 0) {
+              console.log(`     🔍 Files detected: ${result.stats.detectedCount}`);
+            }
+            if (result.stats.organizedCount > 0) {
+              console.log(`     📁 Files organized: ${result.stats.organizedCount}`);
+            }
+          } else {
+            console.error(`     ❌ Upload failed for ${folderStructure}:`);
+            if (result.errors && result.errors.length > 0) {
+              result.errors.forEach(error => {
+                console.error(`        - ${error.fileName}: ${error.error}`);
+              });
+            }
+            totalErrors += pathFiles.length;
+          }
+
+          // Small delay between path groups
+          await new Promise(resolve => setTimeout(resolve, 100));
+        }
+
+      } catch (error) {
+        console.error(`   ❌ Error uploading batch ${batchNumber}:`, error.message);
+        totalErrors += filesToUpload.length;
+      }
     }
 
-    // Small delay between batches to prevent overwhelming the server
-    if (i + batchSize < files.length) {
-      await delay(100);
+    // Small delay between batches
+    if (i + batchSize < allRelatedFiles.length) {
+      await new Promise(resolve => setTimeout(resolve, BATCH_DELAY));
     }
   }
 
-  progressBar.stop();
+  if (options.showProgress !== false) {
+    progressBar.stop();
+  }
+
+  console.log(`\n${'='.repeat(60)}`);
+  console.log(`🎯 RFC-BASED UPLOAD COMPLETED`);
+  console.log(`${'='.repeat(60)}`);
+  console.log(`   📋 Files processed: ${totalProcessed}`);
+  console.log(`   ✅ Files uploaded: ${totalUploaded}`);
+  console.log(`   ⏭️  Files skipped: ${totalSkipped}`);
+  console.log(`   ❌ Errors: ${totalErrors}`);
+  console.log(`${'='.repeat(60)}\n`);
+
+  return {
+    processedCount: totalProcessed,
+    uploadedCount: totalUploaded,
+    skippedCount: totalSkipped,
+    errorCount: totalErrors,
+  };
+};
+
+/**
+ * Propagate arela_path from pedimento_simplificado records to related files with same base path
+ */
+const propagateArelaPath = async (options = {}) => {
+  if (!supabase) {
+    console.error('❌ Supabase client not initialized');
+    process.exit(1);
+  }
+
+  console.log('🔍 Finding pedimento_simplificado records with arela_path...');
+
+  // Get all pedimento_simplificado records that have arela_path
+  const { data: pedimentoRecords, error: pedimentoError } = await supabase
+    .from('uploader')
+    .select('id, original_path, arela_path, filename')
+    .eq('document_type', 'pedimento_simplificado')
+    .not('arela_path', 'is', null);
+
+  if (pedimentoError) {
+    console.error('❌ Error fetching pedimento records:', pedimentoError.message);
+    return { processedCount: 0, updatedCount: 0, errorCount: 1 };
+  }
+
+  if (!pedimentoRecords || pedimentoRecords.length === 0) {
+    console.log('ℹ️  No pedimento_simplificado records with arela_path found');
+    return { processedCount: 0, updatedCount: 0, errorCount: 0 };
+  }
+
+  console.log(`📋 Found ${pedimentoRecords.length} pedimento records with arela_path`);
+
+  let totalProcessed = 0;
+  let totalUpdated = 0;
+  let totalErrors = 0;
+
+  // Create progress bar
+  const progressBar = new cliProgress.SingleBar({
+    format: '🔄 Propagating paths |{bar}| {percentage}% | {value}/{total} | Updated: {updated} | Errors: {errors}',
+    barCompleteChar: '█',
+    barIncompleteChar: '░',
+    hideCursor: true,
+  });
+
+  if (options.showProgress !== false) {
+    progressBar.start(pedimentoRecords.length, 0, {
+      updated: 0,
+      errors: 0,
+    });
+  }
+
+  // Process each pedimento record
+  for (const pedimento of pedimentoRecords) {
+    try {
+      totalProcessed++;
+
+      // Extract base path from original_path (remove filename)
+      const basePath = path.dirname(pedimento.original_path);
+
+      console.log(`\n🔍 Processing: ${pedimento.filename}`);
+      console.log(`   📁 Base path: ${basePath}`);
+
+      // Extract folder part from existing arela_path by removing the filename
+      const existingPath = pedimento.arela_path;
+      const folderArelaPath = existingPath.includes('/') ?
+        existingPath.substring(0, existingPath.lastIndexOf('/')) + '/' :
+        existingPath.endsWith('/') ? existingPath : existingPath + '/';
+
+      console.log(`   🎯 Original arela path: ${existingPath}`);
+      console.log(`   📁 Folder arela path: ${folderArelaPath}`);
+
+      // Find all files with the same base path that don't have arela_path yet
+      const { data: relatedFiles, error: relatedError } = await supabase
+        .from('uploader')
+        .select('id, filename, original_path')
+        .like('original_path', `${basePath}%`)
+        .is('arela_path', null)
+        .neq('id', pedimento.id); // Exclude the pedimento itself
+
+      if (relatedError) {
+        console.error(`❌ Error finding related files for ${pedimento.filename}:`, relatedError.message);
+        totalErrors++;
+        continue;
+      }
+
+      if (!relatedFiles || relatedFiles.length === 0) {
+        console.log(`   ℹ️  No related files found needing arela_path update`);
+        continue;
+      }
+
+      console.log(`   📄 Found ${relatedFiles.length} related files to update:`);
+
+      // Show first 10 files, then indicate if there are more
+      const filesToShow = relatedFiles.slice(0, 10);
+      filesToShow.forEach(file => {
+        console.log(`      - ${file.filename}`);
+      });
+
+      if (relatedFiles.length > 10) {
+        console.log(`      ... and ${relatedFiles.length - 10} more files`);
+      }
+
+      // Process files in batches to avoid URI length limitations
+      const BATCH_SIZE = 50; // Process 50 files at a time
+      const fileIds = relatedFiles.map(f => f.id);
+      let batchErrors = 0;
+      let batchUpdated = 0;
+
+      console.log(`   🔄 Processing ${relatedFiles.length} files in batches of ${BATCH_SIZE}...`);
+
+      for (let i = 0; i < fileIds.length; i += BATCH_SIZE) {
+        const batchIds = fileIds.slice(i, i + BATCH_SIZE);
+        const batchNumber = Math.floor(i / BATCH_SIZE) + 1;
+        const totalBatches = Math.ceil(fileIds.length / BATCH_SIZE);
+
+        console.log(`   📦 Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`);
+
+        try {
+          const { error: updateError } = await supabase
+            .from('uploader')
+            .update({ arela_path: folderArelaPath })
+            .in('id', batchIds);
+
+          if (updateError) {
+            console.error(`   ❌ Error in batch ${batchNumber}:`, updateError.message);
+            batchErrors++;
+          } else {
+            console.log(`   ✅ Batch ${batchNumber} completed: ${batchIds.length} files updated`);
+            batchUpdated += batchIds.length;
+          }
+        } catch (error) {
+          console.error(`   ❌ Exception in batch ${batchNumber}:`, error.message);
+          batchErrors++;
+        }
+
+        // Small delay between batches to avoid overwhelming the database
+        if (i + BATCH_SIZE < fileIds.length) {
+          await new Promise(resolve => setTimeout(resolve, 100));
+        }
+      }
+
+      if (batchErrors > 0) {
+        console.error(`❌ ${batchErrors} batch(es) failed for ${pedimento.filename}`);
+        totalErrors++;
+      } else {
+        console.log(`   🎯 Successfully updated ${batchUpdated} related files`);
+        totalUpdated += batchUpdated;
+      }
+
+    } catch (error) {
+      console.error(`❌ Error processing ${pedimento.filename}:`, error.message);
+      totalErrors++;
+    }
+
+    if (options.showProgress !== false) {
+      progressBar.update(totalProcessed, {
+        updated: totalUpdated,
+        errors: totalErrors,
+      });
+    }
+  }
+
+  if (options.showProgress !== false) {
+    progressBar.stop();
+  }
+
+  console.log(`\n${'='.repeat(60)}`);
+  console.log(`🎯 ARELA PATH PROPAGATION COMPLETED`);
+  console.log(`${'='.repeat(60)}`);
+  console.log(`   📋 Pedimento records processed: ${totalProcessed}`);
+  console.log(`   ✅ Related files updated: ${totalUpdated}`);
+  console.log(`   ❌ Errors: ${totalErrors}`);
+  console.log(`${'='.repeat(60)}\n`);
 
   return {
-    successCount,
-    failureCount,
-    skippedCount,
+    processedCount: totalProcessed,
+    updatedCount: totalUpdated,
+    errorCount: totalErrors,
   };
 };
 
 program
-  .name('supabase-uploader')
-  .description('CLI to upload folders from a base path to Supabase Storage')
+  .name('arela-uploader')
+  .description(
+    'CLI to upload folders to Arela API or Supabase Storage with automatic processing',
+  )
   .option('-v, --version', 'output the version number')
   .option('-p, --prefix <prefix>', 'Prefix path in bucket', '')
+  .option('-b, --bucket <bucket>', 'Bucket name override')
+  .option('--force-supabase', 'Force direct Supabase upload (skip API)')
   .option(
-    '-r, --rename-files',
-    'Rename files with problematic characters before uploading',
+    '--no-auto-detect',
+    'Disable automatic file detection (API mode only)',
   )
   .option(
-    '--dry-run',
-    'Show what files would be renamed without actually renaming them',
+    '--no-auto-organize',
+    'Disable automatic file organization (API mode only)',
   )
   .option(
     '-c, --concurrency <number>',
-    'Number of files to process concurrently (default: 3)',
-    '3',
+    'Files per batch for processing (default: 10)',
+    '10',
   )
+  .option('--batch-size <number>', 'API batch size (default: 10)', '10')
+  .option('--show-stats', 'Show detailed processing statistics')
   .option(
-    '--show-cache-stats',
-    'Show cache statistics for performance analysis',
+    '--folder-structure <structure>',
+    'Custom folder structure (e.g., "2024/4023260" or "cliente1/pedimentos")',
   )
   .option(
-    '--batch-size <number>',
-    'Number of logs to batch before sending to Supabase (default: 50)',
-    '50',
+    '--auto-detect-structure',
+    'Automatically detect year/pedimento from file paths',
   )
+  .option('--client-path <path>', 'Client path for metadata tracking')
+  .option('--stats-only', 'Phase 1: Only read filesystem stats and insert to database (no file reading or detection)')
+  .option('--no-detect', 'Disable document type detection in stats-only mode')
+  .option('--detect-pdfs', 'Phase 2: Process PDF files in database for pedimento-simplificado detection')
+  .option('--propagate-arela-path', 'Phase 3: Propagate arela_path from pedimento_simplificado records to related files with same base path')
+  .option('--upload-by-rfc', 'Phase 4: Upload files to Arela API based on RFC values from UPLOAD_RFCS environment variable')
+  .option('--run-all-phases', 'Run all 4 phases in sequence: stats → detect → propagate → upload')
   .action(async (options) => {
-    // Handle version option
     if (options.version) {
-      console.log(version);
+      console.log(packageVersion);
       process.exit(0);
     }
 
+    // Handle detect-pdfs option (Phase 2)
+    if (options.detectPdfs) {
+      console.log('🔍 Starting Phase 2: PDF Detection');
+      await checkCredentials(true); // Force Supabase mode
+
+      const result = await detectPedimentosInDatabase({
+        batchSize: parseInt(options.batchSize) || 10,
+      });
+
+      console.log(`✅ Phase 2 Complete: ${result.detectedCount} detected, ${result.errorCount} errors`);
+      return;
+    }
+
+    // Handle run-all-phases option
+    if (options.runAllPhases) {
+      console.log('🚀 Starting all 4 phases in sequence...');
+      await checkCredentials(true); // Force Supabase mode
+
+      // Phase 1: Stats collection
+      console.log('\n📊 === PHASE 1: Filesystem Stats ===');
+      options.statsOnly = true;
+      // Continue with normal processing to run Phase 1
+
+      // The rest will be handled after Phase 1 completes
+    }
+
+    // Handle propagate-arela-path option
+    if (options.propagateArelaPath) {
+      // Initialize Supabase credentials for propagation
+      await checkCredentials(true); // Force Supabase mode
+
+      const result = await propagateArelaPath({
+        showProgress: options.showStats || true,
+      });
+
+      if (result.errorCount > 0) {
+        process.exit(1);
+      }
+      return;
+    }
+
+    // Handle upload-by-rfc option
+    if (options.uploadByRfc) {
+      // RFC upload needs both Supabase (for database queries) and API (for uploads)
+      await checkCredentials(false); // Initialize API mode
+
+      // Also initialize Supabase for database queries
+      if (!supabase) {
+        if (!supabaseUrl || !supabaseKey) {
+          console.error('❌ RFC upload requires Supabase credentials for database queries.');
+          console.error('   Please set SUPABASE_URL and SUPABASE_KEY environment variables.');
+          process.exit(1);
+        }
+
+        supabase = createClient(supabaseUrl, supabaseKey);
+        console.log('✅ Connected to Supabase for database queries');
+      }
+
+      const result = await uploadFilesByRfc({
+        showProgress: options.showStats || true,
+        batchSize: parseInt(options.batchSize) || 10,
+        folderStructure: options.folderStructure,
+      });
+
+      if (result.errorCount > 0) {
+        process.exit(1);
+      }
+      return;
+    }
+
+    // Initialize credentials with force supabase flag (for stats mode, always need Supabase)
+    await checkCredentials(options.forceSupabase || options.statsOnly);
+
     if (!basePath || !sources || sources.length === 0) {
       console.error(
         '⚠️ UPLOAD_BASE_PATH or UPLOAD_SOURCES not defined in environment variables.',
@@ -739,18 +1853,31 @@ program
       process.exit(1);
     }
 
-    const concurrency = parseInt(options.concurrency) || 3;
-    const batchSize = parseInt(options.batchSize) || 50;
-
-    // Configure log batcher with custom batch size
-    logBatcher.batchSize = batchSize;
+    const batchSize = parseInt(options.batchSize) || 10;
+    const concurrency = parseInt(options.concurrency) || 10;
 
-    console.log(`🚀 Using concurrency level: ${concurrency}`);
-    console.log(`📦 Using log batch size: ${batchSize}`);
+    if (options.statsOnly) {
+      console.log('📊 Mode: Stats Only - Reading file stats and inserting to uploader table');
+      console.log('🚫 Files will NOT be uploaded');
+      if (options.detect !== false) {
+        console.log('🔍 Document type detection ENABLED for supported files');
+      } else {
+        console.log('🔍 Document type detection DISABLED');
+      }
+    } else {
+      console.log(
+        `🚀 Mode: ${apiMode ? 'Arela API with auto-processing' : 'Direct Supabase'}`,
+      );
+    }
+    console.log(`📦 Batch size: ${batchSize}`);
+    console.log(`⚡ Concurrency: ${concurrency}`);
 
-    const processedPaths = await getProcessedPaths();
+    const processedPaths = getProcessedPaths();
     let globalSuccess = 0;
+    let globalDetected = 0;
+    let globalOrganized = 0;
     let globalFailure = 0;
+    let globalSkipped = 0;
 
     for (const folder of sources) {
       const sourcePath = path.resolve(basePath, folder).replace(/\\/g, '/');
@@ -764,78 +1891,131 @@ program
 
         console.log(`📊 Found ${files.length} files to process`);
 
-        // Process files in parallel batches
-        const { successCount, failureCount, skippedCount } =
-          await processFilesInBatches(
-            files,
-            concurrency,
-            options,
-            basePath,
-            folder,
-            sourcePath,
-            processedPaths,
-          );
-
-        globalSuccess += successCount;
-        globalFailure += failureCount;
-
-        console.log(`\n📦 Upload Summary for ${folder}:`);
-        console.log(`   ✅ Successfully uploaded files: ${successCount}`);
-        console.log(`   ❌ Files with errors: ${failureCount}`);
-        console.log(`   ⏭️  Files skipped (already exist): ${skippedCount}`);
-        console.log(`   📜 Log file: ${logFilePath} \n`);
+        const result = await processFilesInBatches(
+          files,
+          batchSize,
+          options,
+          basePath,
+          folder,
+          sourcePath,
+          processedPaths,
+        );
 
-        // Show cache statistics if requested
-        if (options.showCacheStats) {
-          console.log(`📊 Cache Statistics:`);
-          console.log(
-            `   🗂️  Filename sanitization cache: ${sanitizationCache.size} entries`,
-          );
-          console.log(
-            `   📁 Path sanitization cache: ${pathSanitizationCache.size} entries`,
-          );
-          console.log(
-            `   📋 Log batch pending: ${logBatcher.batch.length} entries`,
-          );
+        globalSuccess += result.successCount;
+        globalDetected += result.detectedCount || 0;
+        globalOrganized += result.organizedCount || 0;
+        globalFailure += result.failureCount;
+        globalSkipped += result.skippedCount;
 
-          // Calculate cache hit rate (rough estimation)
-          const totalProcessed = successCount + failureCount + skippedCount;
-          const estimatedCacheHitRate =
-            totalProcessed > 0
-              ? Math.round(
-                  ((totalProcessed - sanitizationCache.size) / totalProcessed) *
-                    100,
-                )
-              : 0;
-          console.log(
-            `   🎯 Estimated cache hit rate: ${Math.max(0, estimatedCacheHitRate)}%\n`,
-          );
+        console.log(`\n📦 Summary for ${folder}:`);
+        if (options.statsOnly) {
+          console.log(`   📊 Stats recorded: ${result.successCount}`);
+        } else {
+          console.log(`   ✅ Uploaded: ${result.successCount}`);
+          if (apiMode) {
+            console.log(`   🔍 Detected: ${result.detectedCount || 0}`);
+            console.log(`   📁 Organized: ${result.organizedCount || 0}`);
+          }
+        }
+        console.log(`   ❌ Errors: ${result.failureCount}`);
+        if (options.statsOnly) {
+          console.log(`   ⏭️  Duplicates: ${result.skippedCount}`);
+        } else {
+          console.log(`   ⏭️  Skipped: ${result.skippedCount}`);
         }
 
         writeLog(
-          `📦 Upload Summary for folder ${folder}: Success: ${successCount}, Errors: ${failureCount}, Skipped: ${skippedCount}`,
+          `📦 Summary for ${folder}: Success: ${result.successCount}, Detected: ${result.detectedCount || 0}, Organized: ${result.organizedCount || 0}, Errors: ${result.failureCount}, ${options.statsOnly ? 'Duplicates' : 'Skipped'}: ${result.skippedCount}`,
         );
       } catch (err) {
         console.error(`⚠️ Error processing folder ${folder}:`, err.message);
         writeLog(`⚠️ Error processing folder ${folder}: ${err.message}`);
-        await sendLogToSupabase({
-          file: folder,
-          uploadPath: folder,
-          status: 'error',
-          message: err.message,
-        });
         globalFailure++;
       }
     }
 
-    // Force flush any remaining logs before finishing
-    console.log(`📤 Flushing remaining logs...`);
-    await logBatcher.forceFlush();
-
-    console.log(`🎯 Upload completed.`);
-    console.log(`   ✅ Total uploaded: ${globalSuccess}`);
-    console.log(`   ❌ Total with errors: ${globalFailure}`);
+    console.log(`\n${'='.repeat(60)}`);
+    if (options.statsOnly) {
+      console.log(`📊 STATS COLLECTION COMPLETED`);
+      console.log(`${'='.repeat(60)}`);
+      console.log(`   📊 Total stats recorded: ${globalSuccess}`);
+    } else {
+      console.log(`🎯 ${apiMode ? 'ARELA API' : 'SUPABASE'} UPLOAD COMPLETED`);
+      console.log(`${'='.repeat(60)}`);
+      console.log(`   ✅ Total uploaded: ${globalSuccess}`);
+      if (apiMode) {
+        console.log(`   🔍 Total detected: ${globalDetected}`);
+        console.log(`   📁 Total organized: ${globalOrganized}`);
+      }
+    }
+    if (options.statsOnly) {
+      console.log(`   ⏭️  Total duplicates: ${globalSkipped}`);
+    } else {
+      console.log(`   ⏭️  Total skipped: ${globalSkipped}`);
+    }
+    console.log(`   ❌ Total errors: ${globalFailure}`);
     console.log(`   📜 Log file: ${logFilePath}`);
+    console.log(`${'='.repeat(60)}\n`);
+
+    // Continue with remaining phases if running all phases
+    if (options.runAllPhases && options.statsOnly) {
+      try {
+        // Phase 2: PDF Detection
+        console.log('\n🔍 === PHASE 2: PDF Detection ===');
+        const detectionResult = await detectPedimentosInDatabase({
+          batchSize: parseInt(options.batchSize) || 10,
+        });
+        console.log(`✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`);
+
+        // Phase 3: Propagate arela_path
+        console.log('\n📁 === PHASE 3: Propagate Arela Paths ===');
+        const propagateResult = await propagateArelaPath({
+          showProgress: options.showStats || true,
+        });
+        console.log(`✅ Phase 3 Complete: ${propagateResult.updatedCount || 0} paths propagated`);
+
+        // Phase 4: Upload by RFC
+        if (uploadRfcs && uploadRfcs.length > 0) {
+          console.log('\n🚀 === PHASE 4: Upload by RFC ===');
+
+          // Initialize API mode for uploads
+          await checkCredentials(false);
+
+          const uploadResult = await uploadFilesByRfc({
+            showProgress: options.showStats || true,
+            batchSize: parseInt(options.batchSize) || 10,
+            folderStructure: options.folderStructure,
+          });
+          console.log(`✅ Phase 4 Complete: Upload finished`);
+        } else {
+          console.log('\n⚠️ === PHASE 4: Upload by RFC ===');
+          console.log('⚠️ UPLOAD_RFCS environment variable not configured, skipping Phase 4');
+        }
+
+        console.log('\n🎉 All 4 phases completed successfully!');
+
+      } catch (error) {
+        console.error(`❌ Error in multi-phase execution:`, error.message);
+        process.exit(1);
+      }
+    }
+
+    if (options.showStats && (sanitizationCache.size > 0 || pathDetectionCache.size > 0)) {
+      console.log(`📊 Performance Statistics:`);
+      if (sanitizationCache.size > 0) {
+        console.log(
+          `   🗂️  Sanitization cache entries: ${sanitizationCache.size}`,
+        );
+      }
+      if (pathDetectionCache.size > 0) {
+        console.log(
+          `   📁  Path detection cache entries: ${pathDetectionCache.size}`,
+        );
+      }
+    }
+
+    // OPTIMIZED: Ensure log buffer is flushed before exit
+    flushLogBuffer();
   });
 
 program.parse();