@arela/uploader 0.2.11 → 0.2.13

package/src/index-old.js

@@ -1,2658 +0,0 @@
-#!/usr/bin/env node
-import { createClient } from '@supabase/supabase-js';
-import cliProgress from 'cli-progress';
-import { Command } from 'commander';
-import { config } from 'dotenv';
-import FormData from 'form-data';
-import fs from 'fs';
-import { globby } from 'globby';
-import mime from 'mime-types';
-import fetch from 'node-fetch';
-import path from 'path';
-
-import { FileDetectionService } from './file-detection.js';
-
-config();
-
-const program = new Command();
-
-// Read package.json version at startup
-let packageVersion = '0.2.4'; // fallback
-try {
-  const __filename = new URL(import.meta.url).pathname;
-  const __dirname = path.dirname(__filename);
-  const packageJsonPath = path.resolve(__dirname, '../package.json');
-  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
-  packageVersion = packageJson.version || '1.0.0';
-} catch (error) {
-  console.warn('⚠️ Could not read package.json version, using fallback');
-}
-
-// Configuración de Supabase (original)
-const supabaseUrl = process.env.SUPABASE_URL;
-const supabaseKey = process.env.SUPABASE_KEY;
-const bucket = process.env.SUPABASE_BUCKET;
-
-// Configuración de API (nueva)
-const API_BASE_URL = process.env.ARELA_API_URL;
-const API_TOKEN = process.env.ARELA_API_TOKEN;
-
-// Configuración del uploader mejorado
-const basePath = process.env.UPLOAD_BASE_PATH;
-const sources = process.env.UPLOAD_SOURCES?.split('|')
-  .map((s) => s.trim())
-  .filter(Boolean);
-
-// Configuración de RFCs para upload
-console.log('🔧 Configured RFCs for upload:', process.env.UPLOAD_RFCS);
-const uploadRfcs = process.env.UPLOAD_RFCS?.split('|')
-  .map((s) => s.trim())
-  .filter(Boolean);
-
-let supabase;
-let apiMode = false;
-
-// Pre-compiled regex patterns for better performance (from original complex uploader)
-const SANITIZATION_PATTERNS = [
-  [/[áàâäãåāăą]/gi, 'a'],
-  [/[éèêëēĕėę]/gi, 'e'],
-  [/[íìîïīĭį]/gi, 'i'],
-  [/[óòôöõōŏő]/gi, 'o'],
-  [/[úùûüūŭů]/gi, 'u'],
-  [/[ñň]/gi, 'n'],
-  [/[ç]/gi, 'c'],
-  [/[ý]/gi, 'y'],
-  [/[멕]/g, 'meok'],
-  [/[시]/g, 'si'],
-  [/[코]/g, 'ko'],
-  [/[용]/g, 'yong'],
-  [/[가-힣]/g, 'kr'],
-  [/[\u0300-\u036f]/g, ''],
-  [/[\\?%*:|"<>[\]~`^]/g, '-'],
-  [/[{}]/g, '-'],
-  [/[&]/g, 'and'],
-  [/[()]/g, ''],
-  [/\s+/g, '-'],
-  [/-+/g, '-'],
-  [/^-+|-+$/g, ''],
-  [/^\.+/, ''],
-  [/[^\w.-]/g, ''],
-];
-
-const sanitizationCache = new Map();
-
-const sanitizeFileName = (fileName) => {
-  if (sanitizationCache.has(fileName)) {
-    return sanitizationCache.get(fileName);
-  }
-
-  const ext = path.extname(fileName);
-  const nameWithoutExt = path.basename(fileName, ext);
-
-  if (/^[a-zA-Z0-9._-]+$/.test(nameWithoutExt)) {
-    const result = fileName;
-    sanitizationCache.set(fileName, result);
-    return result;
-  }
-
-  let sanitized = nameWithoutExt.normalize('NFD');
-
-  for (const [pattern, replacement] of SANITIZATION_PATTERNS) {
-    sanitized = sanitized.replace(pattern, replacement);
-  }
-
-  // Additional sanitization for problematic characters
-  sanitized = sanitized
-    .replace(/~/g, '-') // Replace tildes
-    .replace(/\s+/g, '-') // Replace spaces with dashes
-    .replace(/\.+/g, '-') // Replace multiple dots with dashes
-    .replace(/-+/g, '-') // Collapse multiple dashes
-    .replace(/^-+|-+$/g, ''); // Remove leading/trailing dashes
-
-  if (!sanitized) {
-    sanitized = 'unnamed_file';
-  }
-
-  const result = sanitized + ext;
-  sanitizationCache.set(fileName, result);
-  return result;
-};
-
-const checkCredentials = async (forceSupabase = false) => {
-  // Force Supabase mode if explicitly requested
-  if (forceSupabase) {
-    console.log('🔧 Force Supabase mode enabled - skipping API');
-    apiMode = false;
-  } else if (API_BASE_URL && API_TOKEN) {
-    console.log(
-      '🌐 API mode enabled - files will be uploaded to Arela API with automatic processing',
-    );
-    apiMode = true;
-
-    try {
-      const response = await fetch(`${API_BASE_URL}/api/health`, {
-        headers: {
-          'x-api-key': API_TOKEN,
-        },
-      });
-
-      if (!response.ok) {
-        console.warn(
-          '⚠️ API connection failed, falling back to direct Supabase upload',
-        );
-        apiMode = false;
-      } else {
-        console.log('✅ Connected to Arela API');
-        return;
-      }
-    } catch (err) {
-      console.warn(
-        '⚠️ API connection failed, falling back to direct Supabase upload',
-      );
-      apiMode = false;
-    }
-  }
-
-  // Initialize Supabase client if not in API mode or if forced
-  if (!apiMode || forceSupabase) {
-    if (!supabaseUrl || !supabaseKey || !bucket) {
-      console.error(
-        '⚠️ Missing credentials. Please set either:\n' +
-          '   - ARELA_API_URL and ARELA_API_TOKEN for API mode, or\n' +
-          '   - SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET for direct mode',
-      );
-      process.exit(1);
-    }
-
-    supabase = createClient(supabaseUrl, supabaseKey);
-
-    try {
-      const { error } = await supabase.storage.from(bucket).list('');
-      if (error) {
-        console.error('⚠️ Error connecting to Supabase:', error.message);
-        process.exit(1);
-      }
-      console.log('✅ Connected to Supabase (direct mode)');
-    } catch (err) {
-      console.error('⚠️ Error:', err.message);
-      process.exit(1);
-    }
-  }
-};
-
-const logFilePath = path.resolve(process.cwd(), 'arela-upload.log');
-
-/**
- * OPTIMIZED: Log buffer to reduce I/O operations
- */
-let logBuffer = [];
-const LOG_BUFFER_SIZE = 100; // Flush every 100 log entries
-let lastFlushTime = Date.now();
-const LOG_FLUSH_INTERVAL = 5000; // Flush every 5 seconds
-
-const flushLogBuffer = () => {
-  if (logBuffer.length === 0) return;
-
-  try {
-    const logContent = logBuffer.join('\n') + '\n';
-    fs.appendFileSync(logFilePath, logContent);
-    logBuffer = [];
-    lastFlushTime = Date.now();
-  } catch (error) {
-    console.error(`❌ Error writing to log file: ${error.message}`);
-  }
-};
-
-const writeLog = (message) => {
-  try {
-    const timestamp = new Date().toISOString();
-    logBuffer.push(`[${timestamp}] ${message}`);
-
-    // Flush if buffer is full or enough time has passed
-    const now = Date.now();
-    if (
-      logBuffer.length >= LOG_BUFFER_SIZE ||
-      now - lastFlushTime >= LOG_FLUSH_INTERVAL
-    ) {
-      flushLogBuffer();
-    }
-  } catch (error) {
-    console.error(`❌ Error buffering log message: ${error.message}`);
-  }
-};
-
-// Ensure logs are flushed on process exit
-process.on('exit', flushLogBuffer);
-process.on('SIGINT', () => {
-  flushLogBuffer();
-  process.exit(0);
-});
-process.on('SIGTERM', () => {
-  flushLogBuffer();
-  process.exit(0);
-});
-
-/**
- * OPTIMIZED: Conditional logging to reduce console overhead
- */
-const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true';
-const BATCH_DELAY = parseInt(process.env.BATCH_DELAY) || 100; // Configurable delay between batches
-const PROGRESS_UPDATE_INTERVAL =
-  parseInt(process.env.PROGRESS_UPDATE_INTERVAL) || 10; // Update progress every N items
-
-const logVerbose = (message) => {
-  if (VERBOSE_LOGGING) {
-    console.log(message);
-  }
-};
-const batchReadFileStats = (filePaths) => {
-  const results = [];
-
-  for (const filePath of filePaths) {
-    try {
-      const stats = fs.statSync(filePath);
-      results.push({ path: filePath, stats, error: null });
-    } catch (error) {
-      results.push({ path: filePath, stats: null, error: error.message });
-    }
-  }
-
-  return results;
-};
-
-/**
- * OPTIMIZED: Cache for year/pedimento detection results to avoid redundant parsing
- */
-const pathDetectionCache = new Map();
-
-/**
- * OPTIMIZED: Clear the path detection cache (useful for testing or long-running processes)
- */
-const clearPathDetectionCache = () => {
-  pathDetectionCache.clear();
-};
-
-/**
- * OPTIMIZED: Get detection results with caching
- */
-const getCachedPathDetection = (filePath, basePath) => {
-  const cacheKey = `${filePath}|${basePath}`;
-
-  if (pathDetectionCache.has(cacheKey)) {
-    return pathDetectionCache.get(cacheKey);
-  }
-
-  const detection = extractYearAndPedimentoFromPath(filePath, basePath);
-  pathDetectionCache.set(cacheKey, detection);
-
-  return detection;
-};
-
-/**
- * Extracts year and pedimento number from file path
- * Supports patterns like:
- * - /path/to/2024/4023260/file.pdf
- * - /path/to/pedimentos/2024/4023260/file.pdf
- * - /path/to/docs/año2024/ped4023260/file.pdf
- */
-const extractYearAndPedimentoFromPath = (filePath, basePath) => {
-  try {
-    const relativePath = path.relative(basePath, filePath);
-    const pathParts = relativePath.split(path.sep);
-
-    let year = null;
-    let pedimento = null;
-
-    // Pattern 1: Direct year/pedimento structure (2024/4023260)
-    for (let i = 0; i < pathParts.length - 1; i++) {
-      const part = pathParts[i];
-      const nextPart = pathParts[i + 1];
-
-      // Check if current part looks like a year (2020-2030)
-      const yearMatch = part.match(/^(202[0-9])$/);
-      if (yearMatch && nextPart) {
-        year = yearMatch[1];
-
-        // Check if next part looks like a pedimento (4-8 digits)
-        const pedimentoMatch = nextPart.match(/^(\d{4,8})$/);
-        if (pedimentoMatch) {
-          pedimento = pedimentoMatch[1];
-          break;
-        }
-      }
-    }
-
-    // Pattern 2: Named patterns (año2024, ped4023260)
-    if (!year || !pedimento) {
-      for (const part of pathParts) {
-        if (!year) {
-          const namedYearMatch = part.match(/(?:año|year|anio)(\d{4})/i);
-          if (namedYearMatch) {
-            year = namedYearMatch[1];
-          }
-        }
-
-        if (!pedimento) {
-          const namedPedimentoMatch = part.match(
-            /(?:ped|pedimento|pedi)(\d{4,8})/i,
-          );
-          if (namedPedimentoMatch) {
-            pedimento = namedPedimentoMatch[1];
-          }
-        }
-      }
-    }
-
-    // Pattern 3: Loose year detection in any part
-    if (!year) {
-      for (const part of pathParts) {
-        const yearMatch = part.match(/(202[0-9])/);
-        if (yearMatch) {
-          year = yearMatch[1];
-          break;
-        }
-      }
-    }
-
-    // Pattern 4: Loose pedimento detection (4-8 consecutive digits)
-    if (!pedimento) {
-      for (const part of pathParts) {
-        const pedimentoMatch = part.match(/(\d{4,8})/);
-        if (pedimentoMatch && pedimentoMatch[1].length >= 4) {
-          pedimento = pedimentoMatch[1];
-          break;
-        }
-      }
-    }
-
-    return { year, pedimento, detected: !!(year && pedimento) };
-  } catch (error) {
-    return {
-      year: null,
-      pedimento: null,
-      detected: false,
-      error: error.message,
-    };
-  }
-};
-
-/**
- * OPTIMIZED: Get processed paths with caching and buffered log reading
- */
-let processedPathsCache = null;
-let lastLogModTime = 0;
-
-const getProcessedPaths = () => {
-  try {
-    // Check if log file exists
-    if (!fs.existsSync(logFilePath)) {
-      return new Set();
-    }
-
-    // Check if cache is still valid
-    const logStats = fs.statSync(logFilePath);
-    if (processedPathsCache && logStats.mtime.getTime() === lastLogModTime) {
-      return processedPathsCache;
-    }
-
-    // Read and parse log file
-    const processed = new Set();
-    const content = fs.readFileSync(logFilePath, 'utf-8');
-
-    // Use more efficient regex with global flag
-    const regex = /(SUCCESS|SKIPPED): .*? -> (.+)/g;
-    let match;
-
-    while ((match = regex.exec(content)) !== null) {
-      const path = match[2];
-      if (path) {
-        processed.add(path.trim());
-      }
-    }
-
-    // Update cache
-    processedPathsCache = processed;
-    lastLogModTime = logStats.mtime.getTime();
-
-    return processed;
-  } catch (error) {
-    console.error(`⚠️ Error reading processed paths: ${error.message}`);
-    return new Set();
-  }
-};
-
-/**
- * Upload files to Arela API with automatic detection and organization
- */
-const uploadToApi = async (files, options) => {
-  const formData = new FormData();
-
-  files.forEach((file) => {
-    const fileBuffer = fs.readFileSync(file.path);
-    formData.append('files', fileBuffer, {
-      filename: file.name,
-      contentType: file.contentType,
-    });
-  });
-
-  if (bucket) formData.append('bucket', bucket);
-  if (options.prefix) formData.append('prefix', options.prefix);
-
-  // Nueva funcionalidad: estructura de carpetas personalizada
-  let combinedStructure = null;
-  let cachedDetection = null; // Cache detection result to avoid redundant calls
-
-  if (
-    options.folderStructure &&
-    options.autoDetectStructure &&
-    files.length > 0
-  ) {
-    // Combine custom folder structure with auto-detection
-    const firstFile = files[0];
-    cachedDetection = getCachedPathDetection(firstFile.path, process.cwd());
-
-    if (cachedDetection.detected) {
-      const autoStructure = `${cachedDetection.year}/${cachedDetection.pedimento}`;
-      combinedStructure = `${options.folderStructure}/${autoStructure}`;
-      formData.append('folderStructure', combinedStructure);
-      console.log(
-        `📁 Combined folder structure: ${options.folderStructure} + ${autoStructure} = ${combinedStructure}`,
-      );
-    } else {
-      // Fallback to just custom structure if auto-detection fails
-      formData.append('folderStructure', options.folderStructure);
-      console.log(
-        `📁 Using custom folder structure (auto-detection failed): ${options.folderStructure}`,
-      );
-    }
-  } else if (options.folderStructure) {
-    formData.append('folderStructure', options.folderStructure);
-    console.log(`📁 Using custom folder structure: ${options.folderStructure}`);
-  } else if (options.autoDetectStructure && files.length > 0) {
-    // Try to auto-detect from the first file if no explicit structure is provided
-    const firstFile = files[0];
-    cachedDetection = getCachedPathDetection(firstFile.path, process.cwd());
-
-    if (cachedDetection.detected) {
-      const autoStructure = `${cachedDetection.year}/${cachedDetection.pedimento}`;
-      formData.append('folderStructure', autoStructure);
-    }
-  }
-
-  // Si se especifica clientPath para user_metadata
-  if (options.clientPath) {
-    formData.append('clientPath', options.clientPath);
-  }
-
-  formData.append('autoDetect', String(options.autoDetect ?? true));
-  formData.append('autoOrganize', String(options.autoOrganize ?? true));
-  formData.append('batchSize', String(options.batchSize || 10));
-  formData.append('clientVersion', packageVersion);
-
-  const response = await fetch(
-    `${API_BASE_URL}/api/storage/batch-upload-and-process`,
-    {
-      method: 'POST',
-      headers: {
-        'x-api-key': API_TOKEN,
-      },
-      body: formData,
-    },
-  );
-
-  if (!response.ok) {
-    const errorText = await response.text();
-    throw new Error(
-      `API request failed: ${response.status} ${response.statusText} - ${errorText}`,
-    );
-  }
-
-  return response.json();
-};
-
-/**
- * Upload file directly to Supabase (fallback method)
- */
-const uploadToSupabase = async (file, uploadPath) => {
-  const content = fs.readFileSync(file);
-  const contentType = mime.lookup(file) || 'application/octet-stream';
-
-  const { data, error } = await supabase.storage
-    .from(bucket)
-    .upload(uploadPath.replace(/\\/g, '/'), content, {
-      upsert: true,
-      contentType,
-    });
-
-  if (error) {
-    throw new Error(error.message);
-  }
-
-  return data;
-};
-
-/**
- * Insert file stats into uploader table with document detection
- */
-const insertStatsToUploaderTable = async (files, options) => {
-  if (!supabase) {
-    throw new Error(
-      'Supabase client not initialized. Stats mode requires Supabase connection.',
-    );
-  }
-
-  const detectionService = new FileDetectionService();
-  const records = [];
-
-  for (const file of files) {
-    // OPTIMIZED: Use pre-computed stats if available, otherwise call fs.statSync
-    const stats = file.stats || fs.statSync(file.path);
-    const originalPath = options.clientPath || file.path;
-
-    // Check if record already exists
-    const { data: existingRecords, error: checkError } = await supabase
-      .from('uploader')
-      .select('id, original_path')
-      .eq('original_path', originalPath)
-      .limit(1);
-
-    if (checkError) {
-      console.error(
-        `❌ Error checking for existing record: ${checkError.message}`,
-      );
-      continue;
-    }
-
-    if (existingRecords && existingRecords.length > 0) {
-      console.log(`⏭️ Skipping duplicate: ${path.basename(file.path)}`);
-      continue;
-    }
-
-    // Initialize record with basic file stats
-    const record = {
-      document_type: null,
-      size: stats.size,
-      num_pedimento: null,
-      filename: file.originalName || path.basename(file.path),
-      original_path: originalPath,
-      arela_path: null,
-      status: 'stats',
-      rfc: null,
-      message: null,
-    };
-
-    // Try to detect document type for supported files
-    if (detectionService.isSupportedFileType(file.path)) {
-      try {
-        const detection = await detectionService.detectFile(file.path);
-
-        if (detection.detectedType) {
-          record.document_type = detection.detectedType;
-          record.num_pedimento = detection.detectedPedimento;
-          record.status = 'detected';
-
-          // Set arela_path for pedimento_simplificado documents
-          if (detection.arelaPath) {
-            record.arela_path = detection.arelaPath;
-          }
-
-          // Extract RFC from fields if available
-          const rfcField = detection.fields.find(
-            (f) => f.name === 'rfc' && f.found,
-          );
-          if (rfcField) {
-            record.rfc = rfcField.value;
-          }
-        } else {
-          record.status = 'not-detected';
-          if (detection.error) {
-            record.message = detection.error;
-          }
-        }
-      } catch (error) {
-        console.error(`❌ Error detecting ${record.filename}:`, error.message);
-        record.status = 'detection-error';
-        record.message = error.message;
-      }
-    } else {
-      record.status = 'unsupported';
-      record.message = 'File type not supported for detection';
-    }
-
-    records.push(record);
-  }
-
-  if (records.length === 0) {
-    console.log('📝 No new records to insert (all were duplicates or errors)');
-    return [];
-  }
-
-  console.log(
-    `💾 Inserting ${records.length} new records into uploader table...`,
-  );
-
-  const { data, error } = await supabase
-    .from('uploader')
-    .insert(records)
-    .select();
-
-  if (error) {
-    throw new Error(`Failed to insert stats records: ${error.message}`);
-  }
-
-  return data;
-};
-
-/**
- * OPTIMIZED: Insert ONLY file stats into uploader table (Phase 1)
- * No file reading, no detection - just filesystem metadata
- * Returns summary statistics with accurate counts for new vs updated records
- */
-const insertStatsOnlyToUploaderTable = async (files, options) => {
-  if (!supabase) {
-    throw new Error(
-      'Supabase client not initialized. Stats mode requires Supabase connection.',
-    );
-  }
-
-  const batchSize = 1000; // Large batch size for performance
-  const allRecords = [];
-
-  // Prepare all file stats data first - OPTIMIZED to use pre-computed stats
-  console.log('📊 Collecting filesystem stats...');
-  for (const file of files) {
-    try {
-      // Use pre-computed stats if available, otherwise call fs.statSync
-      const stats = file.stats || fs.statSync(file.path);
-      const originalPath = options.clientPath || file.path;
-      const fileExtension = path
-        .extname(file.path)
-        .toLowerCase()
-        .replace('.', '');
-
-      const record = {
-        document_type: null,
-        size: stats.size,
-        num_pedimento: null,
-        filename: file.originalName || path.basename(file.path),
-        original_path: originalPath,
-        arela_path: null,
-        status: 'fs-stats',
-        rfc: null,
-        message: null,
-        file_extension: fileExtension,
-        created_at: new Date().toISOString(),
-        modified_at: stats.mtime.toISOString(),
-      };
-
-      allRecords.push(record);
-    } catch (error) {
-      console.error(`❌ Error reading stats for ${file.path}:`, error.message);
-    }
-  }
-
-  if (allRecords.length === 0) {
-    console.log('📝 No file stats to insert');
-    return { totalInserted: 0, totalSkipped: 0, totalProcessed: 0 };
-  }
-
-  console.log(
-    `💾 Processing ${allRecords.length} file stats in batches of ${batchSize}...`,
-  );
-
-  let totalInserted = 0;
-  let totalUpdated = 0;
-
-  // Process in batches with accurate insert/update tracking
-  for (let i = 0; i < allRecords.length; i += batchSize) {
-    const batch = allRecords.slice(i, i + batchSize);
-
-    try {
-      // First, check which records already exist
-      const originalPaths = batch.map((r) => r.original_path);
-      const { data: existingRecords, error: checkError } = await supabase
-        .from('uploader')
-        .select('original_path')
-        .in('original_path', originalPaths);
-
-      if (checkError) {
-        console.error(
-          `❌ Error checking existing records:`,
-          checkError.message,
-        );
-        continue;
-      }
-
-      const existingPaths = new Set(
-        existingRecords?.map((r) => r.original_path) || [],
-      );
-      const newRecords = batch.filter(
-        (r) => !existingPaths.has(r.original_path),
-      );
-      const updateRecords = batch.filter((r) =>
-        existingPaths.has(r.original_path),
-      );
-
-      console.log(
-        `📦 Batch ${Math.floor(i / batchSize) + 1}: ${newRecords.length} new, ${updateRecords.length} updates`,
-      );
-
-      // Insert new records
-      if (newRecords.length > 0) {
-        const { error: insertError } = await supabase
-          .from('uploader')
-          .insert(newRecords);
-
-        if (insertError) {
-          console.error(`❌ Error inserting new records:`, insertError.message);
-        } else {
-          totalInserted += newRecords.length;
-          console.log(`✅ Inserted ${newRecords.length} new records`);
-        }
-      }
-
-      // Update existing records with current filesystem stats
-      if (updateRecords.length > 0) {
-        let batchUpdated = 0;
-        for (const record of updateRecords) {
-          const { error: updateError } = await supabase
-            .from('uploader')
-            .update({
-              size: record.size,
-              modified_at: record.modified_at,
-              filename: record.filename,
-              file_extension: record.file_extension,
-            })
-            .eq('original_path', record.original_path);
-
-          if (!updateError) {
-            batchUpdated++;
-          }
-        }
-        totalUpdated += batchUpdated;
-        console.log(`🔄 Updated ${batchUpdated} existing records`);
-      }
-    } catch (error) {
-      console.error(
-        `❌ Unexpected error in batch ${Math.floor(i / batchSize) + 1}:`,
-        error.message,
-      );
-    }
-  }
-
-  console.log(
-    `📊 Phase 1 Summary: ${totalInserted} new records inserted, ${totalUpdated} existing records updated`,
-  );
-
-  return {
-    totalInserted,
-    totalSkipped: totalUpdated, // Updated records can be considered "processed but not new"
-    totalProcessed: allRecords.length,
-  };
-};
-
-/**
- * PHASE 2: Process PDF files for pedimento-simplificado detection
- * Only processes files with status 'fs-stats' and file_extension 'pdf'
- * Processes records in chunks of 1000 to avoid loading all records into memory
- */
-const detectPedimentosInDatabase = async (options = {}) => {
-  if (!supabase) {
-    throw new Error('Supabase client not initialized.');
-  }
-
-  console.log(
-    '🔍 Phase 2: Starting PDF detection for pedimento-simplificado documents...',
-  );
-  writeLog(
-    '🔍 Phase 2: Starting PDF detection for pedimento-simplificado documents',
-  );
-
-  const detectionService = new FileDetectionService();
-  const processingBatchSize = parseInt(options.batchSize) || 10; // Smaller batches for file I/O
-  const queryBatchSize = 1000; // Process 1000 records at a time
-
-  let totalDetected = 0;
-  let totalProcessed = 0;
-  let totalErrors = 0;
-  let offset = 0;
-  let chunkNumber = 1;
-
-  console.log('� Processing PDF files in chunks of 1000 records...');
-  writeLog('📝 Starting PDF detection processing in chunks of 1000 records');
-
-  // Process records in chunks of 1000
-  while (true) {
-    console.log(
-      `\n📥 Fetching chunk ${chunkNumber} (records ${offset + 1} to ${offset + queryBatchSize})...`,
-    );
-    writeLog(
-      `📥 Fetching chunk ${chunkNumber} (records ${offset + 1} to ${offset + queryBatchSize})`,
-    );
-
-    // Fetch next chunk of PDF records
-    const { data: pdfRecords, error: queryError } = await supabase
-      .from('uploader')
-      .select('id, original_path, filename, file_extension, status')
-      .eq('status', 'fs-stats')
-      .eq('file_extension', 'pdf')
-      .ilike('filename', '%simp%')
-      .range(offset, offset + queryBatchSize - 1);
-
-    if (queryError) {
-      throw new Error(
-        `Failed to fetch PDF records chunk ${chunkNumber}: ${queryError.message}`,
-      );
-    }
-
-    // If no records found, we're done
-    if (!pdfRecords || pdfRecords.length === 0) {
-      console.log(`📝 No more PDF files found. Processing completed.`);
-      writeLog(
-        `📝 No more PDF files found. Processing completed at chunk ${chunkNumber}`,
-      );
-      break;
-    }
-
-    console.log(
-      `� Processing chunk ${chunkNumber}: ${pdfRecords.length} PDF records`,
-    );
-    writeLog(
-      `📊 Processing chunk ${chunkNumber}: ${pdfRecords.length} PDF records`,
-    );
-
-    // Create progress bar for this chunk
-    const progressBar = new cliProgress.SingleBar({
-      format: `🔍 Chunk ${chunkNumber} |{bar}| {percentage}% | {value}/{total} | Detected: {detected} | Errors: {errors}`,
-      barCompleteChar: '█',
-      barIncompleteChar: '░',
-      hideCursor: true,
-    });
-
-    progressBar.start(pdfRecords.length, 0, { detected: 0, errors: 0 });
-
-    let chunkDetected = 0;
-    let chunkProcessed = 0;
-    let chunkErrors = 0;
-
-    // Process files in smaller batches within this chunk
-    for (let i = 0; i < pdfRecords.length; i += processingBatchSize) {
-      const batch = pdfRecords.slice(i, i + processingBatchSize);
-      const updatePromises = [];
-
-      for (const record of batch) {
-        try {
-          // Check if file still exists
-          if (!fs.existsSync(record.original_path)) {
-            writeLog(
-              `⚠️ FILE NOT FOUND: ${record.filename} at ${record.original_path}`,
-            );
-            updatePromises.push(
-              supabase
-                .from('uploader')
-                .update({
-                  status: 'file-not-found',
-                  message: 'File no longer exists at original path',
-                })
-                .eq('id', record.id),
-            );
-            chunkErrors++;
-            totalErrors++;
-            continue;
-          }
-
-          // Perform detection
-          const detection = await detectionService.detectFile(
-            record.original_path,
-          );
-          chunkProcessed++;
-          totalProcessed++;
-
-          const updateData = {
-            status: detection.detectedType ? 'detected' : 'not-detected',
-            document_type: detection.detectedType,
-            num_pedimento: detection.detectedPedimento,
-            arela_path: detection.arelaPath,
-            message: detection.error || null,
-          };
-
-          // Extract RFC from fields if available
-          if (detection.fields) {
-            const rfcField = detection.fields.find(
-              (f) => f.name === 'rfc' && f.found,
-            );
-            if (rfcField) {
-              updateData.rfc = rfcField.value;
-            }
-          }
-
-          if (detection.detectedType) {
-            chunkDetected++;
-            totalDetected++;
-            writeLog(
-              `✅ DETECTED: ${record.filename} -> ${detection.detectedType} | Pedimento: ${detection.detectedPedimento || 'N/A'} | RFC: ${detection.fields?.rfc || 'N/A'}`,
-            );
-          } else {
-            writeLog(
-              `⏭️ NOT DETECTED: ${record.filename} - No pedimento-simplificado pattern found`,
-            );
-          }
-
-          updatePromises.push(
-            supabase.from('uploader').update(updateData).eq('id', record.id),
-          );
-        } catch (error) {
-          console.error(
-            `❌ Error detecting ${record.filename}:`,
-            error.message,
-          );
-          writeLog(`❌ ERROR detecting ${record.filename}: ${error.message}`);
-          chunkErrors++;
-          totalErrors++;
-
-          updatePromises.push(
-            supabase
-              .from('uploader')
-              .update({
-                status: 'detection-error',
-                message: error.message,
-              })
-              .eq('id', record.id),
-          );
-        }
-      }
-
-      // Execute all updates in parallel for this batch
-      try {
-        await Promise.all(updatePromises);
-      } catch (error) {
-        console.error(
-          `❌ Error updating batch in chunk ${chunkNumber}:`,
-          error.message,
-        );
-      }
-
-      // Update progress for this chunk
-      progressBar.update(Math.min(i + processingBatchSize, pdfRecords.length), {
-        detected: chunkDetected,
-        errors: chunkErrors,
-      });
-    }
-
-    progressBar.stop();
-
-    console.log(
-      `✅ Chunk ${chunkNumber} completed: ${chunkDetected} detected, ${chunkProcessed} processed, ${chunkErrors} errors`,
-    );
-    writeLog(
-      `✅ Chunk ${chunkNumber} completed: ${chunkDetected} detected, ${chunkProcessed} processed, ${chunkErrors} errors`,
-    );
-
-    // Move to next chunk
-    offset += queryBatchSize;
-    chunkNumber++;
-
-    // If we got fewer records than queryBatchSize, we've reached the end
-    if (pdfRecords.length < queryBatchSize) {
-      console.log(
-        `📝 Reached end of records (chunk had ${pdfRecords.length} records).`,
-      );
-      writeLog(
-        `📝 Reached end of records (chunk had ${pdfRecords.length} records)`,
-      );
-      break;
-    }
-
-    // Small delay between chunks to avoid overwhelming the database
-    await new Promise((resolve) => setTimeout(resolve, 500));
-  }
-
-  console.log(
-    `📊 Phase 2 Summary: ${totalDetected} detected, ${totalProcessed} processed, ${totalErrors} errors`,
-  );
-
-  // Write comprehensive log summary
-  writeLog(
-    `📊 PHASE 2 PDF DETECTION COMPLETED - Summary: Detected: ${totalDetected} pedimento-simplificado documents, Processed: ${totalProcessed} PDF files, Errors: ${totalErrors}`,
-  );
-
-  // Ensure logs are flushed
-  flushLogBuffer();
-
-  return {
-    detectedCount: totalDetected,
-    processedCount: totalProcessed,
-    errorCount: totalErrors,
-  };
-};
-
-const processFilesInBatches = async (
-  files,
-  batchSize,
-  options,
-  basePath,
-  folder,
-  sourcePath,
-  processedPaths,
-) => {
-  let totalUploaded = 0;
-  let totalDetected = 0;
-  let totalOrganized = 0;
-  let totalErrors = 0;
-  let totalSkipped = 0;
-
-  const messageBuffer = [];
-
-  const progressBarFormat = options.statsOnly
-    ? '📊 Processing [{bar}] {percentage}% | {value}/{total} files | Stats: {successCount} | Errors: {failureCount} | Duplicates: {skippedCount}'
-    : '📂 Processing [{bar}] {percentage}% | {value}/{total} files | Success: {successCount} | Errors: {failureCount} | Skipped: {skippedCount}';
-
-  const progressBar = new cliProgress.SingleBar({
-    format: progressBarFormat,
-    barCompleteChar: '█',
-    barIncompleteChar: '░',
-    hideCursor: true,
-  });
-
-  progressBar.start(files.length, 0, {
-    successCount: 0,
-    failureCount: 0,
-    skippedCount: 0,
-  });
-
-  if (options.statsOnly) {
-    // OPTIMIZED Stats-only mode - Only read filesystem stats, no file detection
-    console.log(
-      '📊 Phase 1: Processing files in optimized stats-only mode (no detection)...',
-    );
-
-    for (let i = 0; i < files.length; i += batchSize) {
-      const batch = files.slice(i, i + batchSize);
-
-      // OPTIMIZED: Batch read file stats to reduce I/O overhead
-      const fileStatsResults = batchReadFileStats(batch);
-      const statsFiles = fileStatsResults
-        .filter((result) => result.stats !== null) // Only include files with valid stats
-        .map((result) => {
-          const originalFileName = path.basename(result.path);
-
-          return {
-            path: result.path,
-            originalName: originalFileName,
-            stats: result.stats, // Pass pre-computed stats to avoid redundant calls
-          };
-        });
-
-      // Log any files that couldn't be read
-      const failedFiles = fileStatsResults.filter(
-        (result) => result.error !== null,
-      );
-      if (failedFiles.length > 0) {
-        console.log(
-          `⚠️ Could not read stats for ${failedFiles.length} files in batch`,
-        );
-        failedFiles.forEach((failed) => {
-          console.error(`   ❌ ${failed.path}: ${failed.error}`);
-        });
-      }
-
-      try {
-        const result = await insertStatsOnlyToUploaderTable(
-          statsFiles,
-          options,
-        );
-
-        totalUploaded += result.totalInserted;
-        totalSkipped += result.totalSkipped;
-        totalErrors += failedFiles.length; // Count failed file reads as errors
-
-        progressBar.update(Math.min(i + batch.length, files.length), {
-          successCount: totalUploaded,
-          failureCount: totalErrors,
-          skippedCount: totalSkipped,
-        });
-      } catch (error) {
-        console.error(`❌ Error processing stats batch:`, error.message);
-        totalErrors += batch.length;
-
-        progressBar.update(Math.min(i + batch.length, files.length), {
-          successCount: totalUploaded,
-          failureCount: totalErrors,
-          skippedCount: totalSkipped,
-        });
-      }
-    }
-  } else if (apiMode && !options.forceSupabase) {
-    // API Mode - Process in batches
-    for (let i = 0; i < files.length; i += batchSize) {
-      const batch = files.slice(i, i + batchSize);
-      let sanitizedRelativePath;
-
-      const apiFiles = batch
-        .map((file) => {
-          const relativePathRaw = path
-            .relative(basePath, file)
-            .replace(/^[\\/]+/, '')
-            .replace(/\\/g, '/');
-
-          const pathParts = relativePathRaw.split('/');
-          const originalFileName = pathParts[pathParts.length - 1];
-          const sanitizedFileName = sanitizeFileName(originalFileName);
-          pathParts[pathParts.length - 1] = sanitizedFileName;
-          sanitizedRelativePath = pathParts.join('/');
-
-          let uploadPath;
-
-          // Handle combined folder structure + auto-detection
-          if (options.folderStructure && options.autoDetectStructure) {
-            // OPTIMIZED: Use cached detection to avoid redundant parsing
-            const detection = getCachedPathDetection(file, basePath);
-            if (detection.detected) {
-              const autoStructure = `${detection.year}/${detection.pedimento}`;
-              const combinedStructure = `${options.folderStructure}/${autoStructure}`;
-              uploadPath = path.posix.join(
-                combinedStructure,
-                sanitizedFileName,
-              );
-              logVerbose(
-                `📁 Combined structure: ${options.folderStructure}/${autoStructure} for ${originalFileName} -> ${uploadPath}`,
-              );
-            } else {
-              // Fallback to just custom structure if auto-detection fails
-              uploadPath = path.posix.join(
-                options.folderStructure,
-                sanitizedFileName,
-              );
-              logVerbose(
-                `📁 Custom structure (auto-detection failed): ${uploadPath}`,
-              );
-            }
-          } else if (options.folderStructure) {
-            // Use custom folder structure only
-            uploadPath = path.posix.join(
-              options.folderStructure,
-              sanitizedFileName,
-            );
-            logVerbose(`📁 Custom structure: ${uploadPath}`);
-          } else if (options.autoDetectStructure) {
-            // Auto-detect structure from path if enabled - OPTIMIZED: Use cached detection
-            const detection = getCachedPathDetection(file, basePath);
-            if (detection.detected) {
-              const autoStructure = `${detection.year}/${detection.pedimento}`;
-              uploadPath = path.posix.join(autoStructure, sanitizedFileName);
-              console.log(
-                `🔍 Auto-detected: ${autoStructure} for ${originalFileName} -> ${uploadPath}`,
-              );
-            } else {
-              uploadPath = options.prefix
-                ? path.posix.join(options.prefix, sanitizedRelativePath)
-                : sanitizedRelativePath;
-              console.log(`📁 Using relative path: ${uploadPath}`);
-            }
-          } else {
-            uploadPath = options.prefix
-              ? path.posix.join(options.prefix, sanitizedRelativePath)
-              : sanitizedRelativePath;
-            console.log(`📁 Using standard path: ${uploadPath}`);
-          }
-
-          if (processedPaths.has(uploadPath)) {
-            totalSkipped++;
-            writeLog(`SKIPPED: ${file} -> ${uploadPath}`);
-            return null;
-          }
-
-          return {
-            path: file,
-            name: sanitizedFileName,
-            originalName: originalFileName,
-            uploadPath: uploadPath.replace(/\\/g, '/'), // Ensure forward slashes
-            contentType: mime.lookup(file) || 'application/octet-stream',
-          };
-        })
-        .filter(Boolean);
-
-      if (apiFiles.length > 0) {
-        // console.log(`🔄 Processing batch of ${apiFiles.length} files`);
-        // apiFiles.forEach(f => console.log(`   📄 ${f.name} -> ${f.uploadPath}`));
-
-        try {
-          // Use clientPath from options if specified, otherwise construct from detection or folder
-          let clientPath = options.clientPath;
-
-          if (!clientPath && apiFiles.length > 0) {
-            const firstFile = apiFiles[0];
-            // OPTIMIZED: Use cached detection to avoid redundant parsing
-            const detection = getCachedPathDetection(firstFile.path, basePath);
-            if (detection.detected) {
-              // clientPath = `${detection.year}/${detection.pedimento}/`;
-              clientPath = path
-                .resolve(basePath, sanitizedRelativePath)
-                .replace(/\\/g, '/');
-            } else {
-              // Fallback to folder structure if no year/pedimento detected
-              clientPath = path.resolve(basePath, folder).replace(/\\/g, '/');
-            }
-          }
-
-          const result = await uploadToApi(apiFiles, {
-            ...options,
-            clientPath: clientPath,
-          });
-
-          totalUploaded += result.stats.uploadedCount;
-          totalDetected += result.stats.detectedCount;
-          totalOrganized += result.stats.organizedCount;
-          totalErrors += result.stats.errorCount;
-
-          result.uploaded.forEach((upload) => {
-            const apiFile = apiFiles.find(
-              (f) => f.name === upload.originalName,
-            );
-            if (apiFile) {
-              writeLog(`SUCCESS: ${apiFile.path} -> ${apiFile.uploadPath}`);
-              processedPaths.add(apiFile.uploadPath);
-            }
-          });
-
-          // Update status to "file-uploaded" for successfully uploaded files
-          if (result.uploaded && result.uploaded.length > 0 && supabase) {
-            try {
-              const uploadedFilePaths = result.uploaded
-                .map((upload) => {
-                  const apiFile = apiFiles.find(
-                    (f) =>
-                      f.name === upload.originalName ||
-                      f.originalName === upload.originalName,
-                  );
-                  return apiFile ? apiFile.path : null;
-                })
-                .filter(Boolean);
-
-              if (uploadedFilePaths.length > 0) {
-                await supabase
-                  .from('uploader')
-                  .update({ status: 'file-uploaded' })
-                  .in('original_path', uploadedFilePaths);
-
-                console.log(
-                  `   📝 Updated status to "file-uploaded" for ${uploadedFilePaths.length} files`,
-                );
-              }
-            } catch (error) {
-              console.error(
-                `   ⚠️ Error updating status for uploaded files: ${error.message}`,
-              );
-            }
-          }
-
-          result.errors.forEach((error) => {
-            writeLog(
-              `ERROR: ${error.fileName}: ${error.error} (${error.step})`,
-            );
-            messageBuffer.push(
-              `❌ ${error.fileName}: ${error.error} (${error.step})`,
-            );
-          });
-        } catch (error) {
-          totalErrors += apiFiles.length;
-          apiFiles.forEach((file) => {
-            writeLog(`ERROR: ${file.path}: ${error.message}`);
-            messageBuffer.push(`❌ ${file.name}: ${error.message}`);
-          });
-        }
-      }
-
-      progressBar.update(i + batch.length, {
-        successCount: totalUploaded,
-        failureCount: totalErrors,
-        skippedCount: totalSkipped,
-      });
-
-      if (i + batchSize < files.length) {
-        await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY));
-      }
-    }
-  } else {
-    // Direct Supabase mode
-    for (let i = 0; i < files.length; i++) {
-      const file = files[i];
-      try {
-        const relativePath = path.relative(basePath, file);
-        let uploadPath;
-
-        // Handle combined folder structure + auto-detection
-        if (options.folderStructure && options.autoDetectStructure) {
-          const detection = getCachedPathDetection(file, basePath);
-          if (detection.detected) {
-            const autoStructure = `${detection.year}/${detection.pedimento}`;
-            const combinedStructure = `${options.folderStructure}/${autoStructure}`;
-            const fileName = path.basename(file);
-            uploadPath = path.join(combinedStructure, fileName);
-            console.log(
-              `📁 Combined structure: ${options.folderStructure}/${autoStructure} for ${fileName}`,
-            );
-          } else {
-            // Fallback to just custom structure if auto-detection fails
-            const fileName = path.basename(file);
-            uploadPath = path.join(options.folderStructure, fileName);
-            console.log(
-              `📁 Custom structure (auto-detection failed): ${uploadPath}`,
-            );
-          }
-        } else if (options.folderStructure) {
-          // Use custom folder structure only
-          const fileName = path.basename(file);
-          uploadPath = path.join(options.folderStructure, fileName);
-          console.log(`📁 Custom structure: ${uploadPath}`);
-        } else if (options.autoDetectStructure) {
-          // Auto-detect structure from path if enabled - OPTIMIZED: Use cached detection
-          const detection = getCachedPathDetection(file, basePath);
-          if (detection.detected) {
-            const autoStructure = `${detection.year}/${detection.pedimento}`;
-            const fileName = path.basename(file);
-            uploadPath = path.join(autoStructure, fileName);
-          } else {
-            uploadPath = options.prefix
-              ? path.join(options.prefix, relativePath)
-              : relativePath;
-          }
-        } else {
-          uploadPath = options.prefix
-            ? path.join(options.prefix, relativePath)
-            : relativePath;
-        }
-
-        if (processedPaths.has(uploadPath)) {
-          totalSkipped++;
-          writeLog(`SKIPPED: ${file} -> ${uploadPath}`);
-
-          // Update status to "file-uploaded" for skipped files (they already exist)
-          if (supabase) {
-            try {
-              await supabase
-                .from('uploader')
-                .update({ status: 'file-uploaded' })
-                .eq('original_path', file);
-            } catch (error) {
-              console.error(
-                `   ⚠️ Error updating status for skipped file: ${error.message}`,
-              );
-            }
-          }
-        } else {
-          await uploadToSupabase(file, uploadPath);
-          totalUploaded++;
-          writeLog(`SUCCESS: ${file} -> ${uploadPath}`);
-          processedPaths.add(uploadPath);
-
-          // Update status to "file-uploaded" for successfully uploaded files
-          if (supabase) {
-            try {
-              await supabase
-                .from('uploader')
-                .update({ status: 'file-uploaded' })
-                .eq('original_path', file);
-            } catch (error) {
-              console.error(
-                `   ⚠️ Error updating status for uploaded file: ${error.message}`,
-              );
-            }
-          }
-        }
-      } catch (error) {
-        totalErrors++;
-        writeLog(`ERROR: ${file}: ${error.message}`);
-        messageBuffer.push(`❌ ${path.basename(file)}: ${error.message}`);
-      }
-
-      progressBar.update(i + 1, {
-        successCount: totalUploaded,
-        failureCount: totalErrors,
-        skippedCount: totalSkipped,
-      });
-    }
-  }
-
-  progressBar.stop();
-
-  const errorMessages = messageBuffer.filter((msg) => msg.startsWith('❌'));
-  if (errorMessages.length > 0) {
-    console.log('\n🚨 Errors encountered during processing:');
-    errorMessages.forEach((msg) => console.error(msg));
-  }
-
-  return {
-    successCount: totalUploaded,
-    detectedCount: totalDetected,
-    organizedCount: totalOrganized,
-    failureCount: totalErrors,
-    skippedCount: totalSkipped,
-  };
-};
-
-/**
- * Upload files to Arela API based on specific RFC values
- */
-const uploadFilesByRfc = async (options = {}) => {
-  if (!supabase) {
-    console.error('❌ Supabase client not initialized');
-    process.exit(1);
-  }
-
-  if (!API_BASE_URL || !API_TOKEN) {
-    console.error(
-      '❌ Arela API configuration missing. Please set ARELA_API_URL and ARELA_API_TOKEN environment variables.',
-    );
-    process.exit(1);
-  }
-
-  if (!uploadRfcs || uploadRfcs.length === 0) {
-    console.error(
-      '❌ No RFCs specified. Please set UPLOAD_RFCS environment variable with pipe-separated RFC values.',
-    );
-    console.error(
-      '   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"',
-    );
-    process.exit(1);
-  }
-
-  console.log('🎯 RFC-based Upload Mode');
-  console.log(`📋 Target RFCs: ${uploadRfcs.join(', ')}`);
-  console.log('🔍 Searching for files to upload...');
-
-  // First, count total files for the RFCs to show filtering effect
-  const { count: totalRfcFiles, error: countError } = await supabase
-    .from('uploader')
-    .select('*', { count: 'exact', head: true })
-    .in('rfc', uploadRfcs)
-    .not('arela_path', 'is', null);
-
-  if (countError) {
-    console.warn('⚠️ Could not count total RFC files:', countError.message);
-  } else {
-    console.log(`📊 Total files for specified RFCs: ${totalRfcFiles || 0}`);
-  }
-
-  // Step 1: Get all pedimento_simplificado records that match the specified RFCs and have arela_path
-  console.log(
-    '🎯 Finding pedimento_simplificado records for specified RFCs...',
-  );
-  const { data: pedimentoRfcRecords, error: pedimentoRfcError } = await supabase
-    .from('uploader')
-    .select('arela_path')
-    .eq('document_type', 'pedimento_simplificado')
-    .in('rfc', uploadRfcs)
-    .not('arela_path', 'is', null);
-
-  if (pedimentoRfcError) {
-    console.error(
-      '❌ Error fetching pedimento RFC records:',
-      pedimentoRfcError.message,
-    );
-    return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
-  }
-
-  if (!pedimentoRfcRecords || pedimentoRfcRecords.length === 0) {
-    console.log(
-      'ℹ️  No pedimento_simplificado records found for the specified RFCs with arela_path',
-    );
-    return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
-  }
-
-  // Get unique arela_paths from pedimento records
-  const uniqueArelaPaths = [
-    ...new Set(pedimentoRfcRecords.map((r) => r.arela_path)),
-  ];
-  console.log(
-    `📋 Found ${pedimentoRfcRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths for specified RFCs`,
-  );
-
-  // Step 2: Get all files with these arela_paths that haven't been uploaded yet
-  let rfcRecords = [];
-  const chunkSize = 50;
-
-  for (let i = 0; i < uniqueArelaPaths.length; i += chunkSize) {
-    const pathChunk = uniqueArelaPaths.slice(i, i + chunkSize);
-
-    const { data: chunkFiles, error: chunkError } = await supabase
-      .from('uploader')
-      .select('arela_path')
-      .in('arela_path', pathChunk)
-      .neq('status', 'file-uploaded')
-      .not('arela_path', 'is', null);
-
-    if (chunkError) {
-      console.error(
-        '❌ Error fetching files for arela_paths chunk:',
-        chunkError.message,
-      );
-      return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
-    }
-
-    if (chunkFiles && chunkFiles.length > 0) {
-      rfcRecords = rfcRecords.concat(chunkFiles);
-    }
-  }
-
-  if (!rfcRecords || rfcRecords.length === 0) {
-    if (totalRfcFiles && totalRfcFiles > 0) {
-      console.log(
-        `ℹ️  All ${totalRfcFiles} files for the specified RFCs are already uploaded (status: file-uploaded)`,
-      );
-      console.log('   No new files to upload.');
-    } else {
-      console.log('ℹ️  No files found for the specified RFCs with arela_path');
-      console.log(
-        `   Make sure files for RFCs [${uploadRfcs.join(', ')}] have been processed and have arela_path values`,
-      );
-    }
-    return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
-  }
-
-  // Show filtering effect
-  const uploadableArelaPaths = [
-    ...new Set(rfcRecords.map((r) => r.arela_path)),
-  ];
-  const skipped = (totalRfcFiles || 0) - rfcRecords.length;
-  if (skipped > 0) {
-    console.log(
-      `📊 Found ${rfcRecords.length} files ready for upload (${skipped} already uploaded, skipped)`,
-    );
-  } else {
-    console.log(`📊 Found ${rfcRecords.length} files ready for upload`);
-  }
-
-  console.log(
-    `🎯 Found ${uploadableArelaPaths.length} unique arela_path(s) with files ready for upload`,
-  );
-
-  // Step 3: Get ALL files that have these arela_paths (including supporting documents)
-  // Process arela_paths in smaller chunks to avoid URI length limits
-  let allRelatedFiles = [];
-  const arelaPathChunkSize = 50; // Process 50 arela_paths at a time to avoid URI limits
-  const queryBatchSize = 1000;
-
-  console.log(
-    '📥 Fetching all related files (processing arela_paths in chunks to avoid URI limits)...',
-  );
-
-  // Process arela_paths in chunks
-  for (let i = 0; i < uploadableArelaPaths.length; i += arelaPathChunkSize) {
-    const arelaPathChunk = uploadableArelaPaths.slice(
-      i,
-      i + arelaPathChunkSize,
-    );
-    console.log(
-      `   Processing arela_path chunk ${Math.floor(i / arelaPathChunkSize) + 1}/${Math.ceil(uploadableArelaPaths.length / arelaPathChunkSize)} (${arelaPathChunk.length} paths)`,
-    );
-
-    // For each chunk of arela_paths, use pagination to get all related files
-    let hasMore = true;
-    let offset = 0;
-
-    while (hasMore) {
-      const { data: batch, error: queryError } = await supabase
-        .from('uploader')
-        .select('id, original_path, arela_path, filename, rfc, document_type')
-        .in('arela_path', arelaPathChunk)
-        .not('original_path', 'is', null)
-        .neq('status', 'file-uploaded')
-        .range(offset, offset + queryBatchSize - 1);
-
-      if (queryError) {
-        console.error(
-          `❌ Error fetching related files for chunk ${Math.floor(i / arelaPathChunkSize) + 1}:`,
-          queryError.message,
-        );
-        return { processedCount: 0, uploadedCount: 0, errorCount: 1 };
-      }
-
-      if (!batch || batch.length === 0) {
-        hasMore = false;
-      } else {
-        allRelatedFiles = allRelatedFiles.concat(batch);
-        offset += queryBatchSize;
-
-        // If we got less than queryBatchSize, we've reached the end for this chunk
-        if (batch.length < queryBatchSize) {
-          hasMore = false;
-        }
-      }
-    }
-
-    // Small delay between chunks to avoid overwhelming the database
-    if (i + arelaPathChunkSize < uploadableArelaPaths.length) {
-      await new Promise((resolve) => setTimeout(resolve, 100));
-    }
-  }
-
-  if (!allRelatedFiles || allRelatedFiles.length === 0) {
-    console.log('ℹ️  No related files found for the arela_paths');
-    return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
-  }
-
-  console.log(
-    `📁 Found ${allRelatedFiles.length} total files to upload (including supporting documents, excluding already uploaded)`,
-  );
-
-  // Group by RFC and arela_path for better organization
-  const filesByRfc = allRelatedFiles.reduce((acc, record) => {
-    const rfc = record.rfc || 'No RFC';
-    if (!acc[rfc]) {
-      acc[rfc] = [];
-    }
-    acc[rfc].push(record);
-    return acc;
-  }, {});
-
-  console.log('📊 Files by RFC (including supporting documents):');
-  for (const [rfc, files] of Object.entries(filesByRfc)) {
-    const documentTypes = [
-      ...new Set(files.map((f) => f.document_type || 'Unknown')),
-    ];
-    console.log(
-      `   ${rfc}: ${files.length} files (${documentTypes.join(', ')})`,
-    );
-  }
-
-  // Group by arela_path for upload organization
-  const filesByPath = allRelatedFiles.reduce((acc, record) => {
-    const path = record.arela_path;
-    if (!acc[path]) {
-      acc[path] = [];
-    }
-    acc[path].push(record);
-    return acc;
-  }, {});
-
-  console.log('� Files grouped by arela_path:');
-  for (const [path, files] of Object.entries(filesByPath)) {
-    console.log(`   ${path}: ${files.length} files`);
-  }
-
-  let totalProcessed = 0;
-  let totalUploaded = 0;
-  let totalErrors = 0;
-  let totalSkipped = 0;
-
-  // Create progress bar
-  const progressBar = new cliProgress.SingleBar({
-    format:
-      '🚀 Uploading files |{bar}| {percentage}% | {value}/{total} | Uploaded: {uploaded} | Errors: {errors} | Skipped: {skipped}',
-    barCompleteChar: '█',
-    barIncompleteChar: '░',
-    hideCursor: true,
-  });
-
-  if (options.showProgress !== false) {
-    progressBar.start(allRelatedFiles.length, 0, {
-      uploaded: 0,
-      errors: 0,
-      skipped: 0,
-    });
-  }
-
-  const batchSize = parseInt(options.batchSize) || 10;
-  console.log(`📦 Processing in batches of ${batchSize} files`);
-
-  // Process files in batches
-  for (let i = 0; i < allRelatedFiles.length; i += batchSize) {
-    const batch = allRelatedFiles.slice(i, i + batchSize);
-    const batchNumber = Math.floor(i / batchSize) + 1;
-    const totalBatches = Math.ceil(allRelatedFiles.length / batchSize);
-
-    console.log(
-      `\n📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} files)`,
-    );
-
-    // Prepare files for upload
-    const filesToUpload = [];
-
-    for (const record of batch) {
-      totalProcessed++;
-
-      try {
-        const originalPath = record.original_path;
-
-        // Check if file exists
-        if (!fs.existsSync(originalPath)) {
-          console.log(`   ⚠️  File not found: ${originalPath}`);
-          totalSkipped++;
-          continue;
-        }
-
-        // OPTIMIZED: Read file and get size from buffer instead of separate fs.statSync call
-        const fileBuffer = fs.readFileSync(originalPath);
-
-        filesToUpload.push({
-          path: originalPath,
-          buffer: fileBuffer,
-          size: fileBuffer.length, // Get size from buffer instead of fs.statSync
-          name: record.filename,
-          arelaPath: record.arela_path,
-          rfc: record.rfc,
-          documentType: record.document_type,
-        });
-      } catch (error) {
-        console.error(
-          `   ❌ Error reading file ${record.original_path}:`,
-          error.message,
-        );
-        totalErrors++;
-      }
-
-      if (options.showProgress !== false) {
-        progressBar.update(totalProcessed, {
-          uploaded: totalUploaded,
-          errors: totalErrors,
-          skipped: totalSkipped,
-        });
-      }
-    }
-
-    // Upload the batch if we have files
-    if (filesToUpload.length > 0) {
-      try {
-        console.log(
-          `   🚀 Uploading ${filesToUpload.length} files to Arela API...`,
-        );
-
-        const formData = new FormData();
-
-        // Add files to form data
-        filesToUpload.forEach((file, index) => {
-          formData.append(`files`, file.buffer, {
-            filename: file.name,
-            contentType: mime.lookup(file.name) || 'application/octet-stream',
-          });
-        });
-
-        // Instead of using per-file folder structures, we'll group by arela_path and upload separately
-        // Group files by their arela_path to upload them in correct structure
-        const filesByPath = filesToUpload.reduce((acc, file) => {
-          const path = file.arelaPath.replace(/\/$/, '');
-          if (!acc[path]) {
-            acc[path] = [];
-          }
-          acc[path].push(file);
-          return acc;
-        }, {});
-
-        // Upload each group separately with its folder structure
-        for (const [arelaPath, pathFiles] of Object.entries(filesByPath)) {
-          const pathFormData = new FormData();
-
-          pathFiles.forEach((file) => {
-            pathFormData.append('files', file.buffer, {
-              filename: file.name,
-              contentType: mime.lookup(file.name) || 'application/octet-stream',
-            });
-          });
-
-          // Set folder structure for this group - concatenate custom prefix with arela_path
-          const folderStructure = options.folderStructure
-            ? `${options.folderStructure}/${arelaPath}`
-                .replace(/\/+/g, '/')
-                .replace(/\/$/, '')
-            : arelaPath;
-          pathFormData.append('folderStructure', folderStructure);
-          pathFormData.append('autoDetect', 'true');
-          pathFormData.append('autoOrganize', 'false');
-          pathFormData.append('batchSize', String(pathFiles.length));
-          pathFormData.append('clientVersion', packageVersion);
-          if (bucket) {
-            pathFormData.append('bucket', bucket);
-          }
-
-          console.log(
-            `     📁 Uploading ${pathFiles.length} files to: ${folderStructure}`,
-          );
-
-          const response = await fetch(
-            `${API_BASE_URL}/api/storage/batch-upload-and-process`,
-            {
-              method: 'POST',
-              headers: {
-                'x-api-key': API_TOKEN,
-              },
-              body: pathFormData,
-            },
-          );
-
-          if (!response.ok) {
-            const errorText = await response.text();
-            throw new Error(`HTTP ${response.status}: ${errorText}`);
-          }
-
-          const result = await response.json();
-
-          // Check if upload was successful based on stats rather than success field
-          const isSuccessful =
-            result.stats &&
-            result.stats.uploadedCount > 0 &&
-            result.stats.errorCount === 0;
-
-          if (isSuccessful) {
-            console.log(
-              `     ✅ Group uploaded: ${result.stats.uploadedCount} files to ${folderStructure}`,
-            );
-            totalUploaded += result.stats.uploadedCount;
-
-            if (result.stats.detectedCount > 0) {
-              console.log(
-                `     🔍 Files detected: ${result.stats.detectedCount}`,
-              );
-            }
-            if (result.stats.organizedCount > 0) {
-              console.log(
-                `     📁 Files organized: ${result.stats.organizedCount}`,
-              );
-            }
-
-            // Update status to "file-uploaded" for successfully uploaded files
-            try {
-              const uploadedFilePaths = pathFiles.map((file) => file.path);
-              await supabase
-                .from('uploader')
-                .update({ status: 'file-uploaded' })
-                .in('original_path', uploadedFilePaths);
-
-              console.log(
-                `     📝 Updated status to "file-uploaded" for ${uploadedFilePaths.length} files`,
-              );
-            } catch (error) {
-              console.error(
-                `     ⚠️ Error updating status for uploaded files: ${error.message}`,
-              );
-            }
-          } else {
-            console.error(`     ❌ Upload failed for ${folderStructure}:`);
-            if (result.errors && result.errors.length > 0) {
-              result.errors.forEach((error) => {
-                console.error(`        - ${error.fileName}: ${error.error}`);
-              });
-            }
-            totalErrors += pathFiles.length;
-          }
-
-          // Handle files that already exist (usually indicated in result.uploaded or result.skipped)
-          if (result.uploaded && result.uploaded.length > 0) {
-            try {
-              const alreadyUploadedPaths = result.uploaded
-                .filter(
-                  (upload) =>
-                    upload.status === 'already_exists' || upload.alreadyExists,
-                )
-                .map((upload) => {
-                  // Find the corresponding file path from pathFiles
-                  const matchingFile = pathFiles.find(
-                    (f) =>
-                      f.name === upload.fileName ||
-                      f.name === upload.originalName,
-                  );
-                  return matchingFile ? matchingFile.path : null;
-                })
-                .filter(Boolean);
-
-              if (alreadyUploadedPaths.length > 0) {
-                await supabase
-                  .from('uploader')
-                  .update({ status: 'file-uploaded' })
-                  .in('original_path', alreadyUploadedPaths);
-
-                console.log(
-                  `     📝 Updated status to "file-uploaded" for ${alreadyUploadedPaths.length} already existing files`,
-                );
-              }
-            } catch (error) {
-              console.error(
-                `     ⚠️ Error updating status for already existing files: ${error.message}`,
-              );
-            }
-          }
-
-          // Small delay between path groups
-          await new Promise((resolve) => setTimeout(resolve, 100));
-        }
-      } catch (error) {
-        console.error(
-          `   ❌ Error uploading batch ${batchNumber}:`,
-          error.message,
-        );
-        totalErrors += filesToUpload.length;
-      }
-    }
-
-    // Small delay between batches
-    if (i + batchSize < allRelatedFiles.length) {
-      await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY));
-    }
-  }
-
-  if (options.showProgress !== false) {
-    progressBar.stop();
-  }
-
-  console.log(`\n${'='.repeat(60)}`);
-  console.log(`🎯 RFC-BASED UPLOAD COMPLETED`);
-  console.log(`${'='.repeat(60)}`);
-  console.log(`   📋 Files processed: ${totalProcessed}`);
-  console.log(`   ✅ Files uploaded: ${totalUploaded}`);
-  console.log(`   ⏭️  Files skipped: ${totalSkipped}`);
-  console.log(`   ❌ Errors: ${totalErrors}`);
-  console.log(`${'='.repeat(60)}\n`);
-
-  return {
-    processedCount: totalProcessed,
-    uploadedCount: totalUploaded,
-    skippedCount: totalSkipped,
-    errorCount: totalErrors,
-  };
-};
-
-/**
- * Propagate arela_path from pedimento_simplificado records to related files with same base path
- */
-const propagateArelaPath = async (options = {}) => {
-  if (!supabase) {
-    console.error('❌ Supabase client not initialized');
-    process.exit(1);
-  }
-
-  console.log('🔍 Finding pedimento_simplificado records with arela_path...');
-  writeLog('🔍 Starting arela_path propagation process');
-
-  // Get all pedimento_simplificado records that have arela_path
-  const { data: pedimentoRecords, error: pedimentoError } = await supabase
-    .from('uploader')
-    .select('id, original_path, arela_path, filename')
-    .eq('document_type', 'pedimento_simplificado')
-    .not('arela_path', 'is', null);
-
-  if (pedimentoError) {
-    console.error(
-      '❌ Error fetching pedimento records:',
-      pedimentoError.message,
-    );
-    return { processedCount: 0, updatedCount: 0, errorCount: 1 };
-  }
-
-  if (!pedimentoRecords || pedimentoRecords.length === 0) {
-    console.log('ℹ️  No pedimento_simplificado records with arela_path found');
-    writeLog('ℹ️  No pedimento_simplificado records with arela_path found');
-    return { processedCount: 0, updatedCount: 0, errorCount: 0 };
-  }
-
-  console.log(
-    `📋 Found ${pedimentoRecords.length} pedimento records with arela_path`,
-  );
-  writeLog(
-    `📋 Found ${pedimentoRecords.length} pedimento records with arela_path to process`,
-  );
-
-  let totalProcessed = 0;
-  let totalUpdated = 0;
-  let totalErrors = 0;
-
-  // Create progress bar
-  const progressBar = new cliProgress.SingleBar({
-    format:
-      '🔄 Propagating paths |{bar}| {percentage}% | {value}/{total} | Updated: {updated} | Errors: {errors}',
-    barCompleteChar: '█',
-    barIncompleteChar: '░',
-    hideCursor: true,
-  });
-
-  if (options.showProgress !== false) {
-    progressBar.start(pedimentoRecords.length, 0, {
-      updated: 0,
-      errors: 0,
-    });
-  }
-
-  // Process each pedimento record
-  for (const pedimento of pedimentoRecords) {
-    try {
-      totalProcessed++;
-
-      // Extract base path from original_path (remove filename)
-      const basePath = path.dirname(pedimento.original_path);
-
-      console.log(`\n🔍 Processing: ${pedimento.filename}`);
-      console.log(`   📁 Base path: ${basePath}`);
-      writeLog(
-        `🔍 Processing pedimento: ${pedimento.filename} | Base path: ${basePath}`,
-      );
-
-      // Extract folder part from existing arela_path by removing the filename
-      const existingPath = pedimento.arela_path;
-      const folderArelaPath = existingPath.includes('/')
-        ? existingPath.substring(0, existingPath.lastIndexOf('/')) + '/'
-        : existingPath.endsWith('/')
-          ? existingPath
-          : existingPath + '/';
-
-      console.log(`   🎯 Original arela path: ${existingPath}`);
-      console.log(`   📁 Folder arela path: ${folderArelaPath}`);
-
-      // Find all files with the same base path that don't have arela_path yet
-      const { data: relatedFiles, error: relatedError } = await supabase
-        .from('uploader')
-        .select('id, filename, original_path')
-        .like('original_path', `${basePath}%`)
-        .is('arela_path', null)
-        .neq('id', pedimento.id); // Exclude the pedimento itself
-
-      if (relatedError) {
-        console.error(
-          `❌ Error finding related files for ${pedimento.filename}:`,
-          relatedError.message,
-        );
-        totalErrors++;
-        continue;
-      }
-
-      if (!relatedFiles || relatedFiles.length === 0) {
-        console.log(`   ℹ️  No related files found needing arela_path update`);
-        writeLog(`ℹ️  No related files found for ${pedimento.filename}`);
-        continue;
-      }
-
-      console.log(
-        `   📄 Found ${relatedFiles.length} related files to update:`,
-      );
-      writeLog(
-        `📄 Found ${relatedFiles.length} related files to update for ${pedimento.filename}`,
-      );
-
-      // Show first 10 files, then indicate if there are more
-      const filesToShow = relatedFiles.slice(0, 10);
-      filesToShow.forEach((file) => {
-        console.log(`      - ${file.filename}`);
-      });
-
-      if (relatedFiles.length > 10) {
-        console.log(`      ... and ${relatedFiles.length - 10} more files`);
-      }
-
-      // Process files in batches to avoid URI length limitations
-      const BATCH_SIZE = 50; // Process 50 files at a time
-      const fileIds = relatedFiles.map((f) => f.id);
-      let batchErrors = 0;
-      let batchUpdated = 0;
-
-      console.log(
-        `   🔄 Processing ${relatedFiles.length} files in batches of ${BATCH_SIZE}...`,
-      );
-
-      for (let i = 0; i < fileIds.length; i += BATCH_SIZE) {
-        const batchIds = fileIds.slice(i, i + BATCH_SIZE);
-        const batchNumber = Math.floor(i / BATCH_SIZE) + 1;
-        const totalBatches = Math.ceil(fileIds.length / BATCH_SIZE);
-
-        console.log(
-          `   📦 Batch ${batchNumber}/${totalBatches}: Updating ${batchIds.length} files...`,
-        );
-
-        try {
-          const { error: updateError } = await supabase
-            .from('uploader')
-            .update({ arela_path: folderArelaPath })
-            .in('id', batchIds);
-
-          if (updateError) {
-            console.error(
-              `   ❌ Error in batch ${batchNumber}:`,
-              updateError.message,
-            );
-            batchErrors++;
-          } else {
-            console.log(
-              `   ✅ Batch ${batchNumber} completed: ${batchIds.length} files updated`,
-            );
-            batchUpdated += batchIds.length;
-          }
-        } catch (error) {
-          console.error(
-            `   ❌ Exception in batch ${batchNumber}:`,
-            error.message,
-          );
-          batchErrors++;
-        }
-
-        // Small delay between batches to avoid overwhelming the database
-        if (i + BATCH_SIZE < fileIds.length) {
-          await new Promise((resolve) => setTimeout(resolve, 100));
-        }
-      }
-
-      if (batchErrors > 0) {
-        console.error(
-          `❌ ${batchErrors} batch(es) failed for ${pedimento.filename}`,
-        );
-        writeLog(
-          `❌ ${batchErrors} batch(es) failed for ${pedimento.filename}`,
-        );
-        totalErrors++;
-      } else {
-        console.log(`   🎯 Successfully updated ${batchUpdated} related files`);
-        writeLog(
-          `✅ Successfully updated ${batchUpdated} related files for ${pedimento.filename} -> ${folderArelaPath}`,
-        );
-        totalUpdated += batchUpdated;
-      }
-    } catch (error) {
-      console.error(
-        `❌ Error processing ${pedimento.filename}:`,
-        error.message,
-      );
-      writeLog(`❌ Error processing ${pedimento.filename}: ${error.message}`);
-      totalErrors++;
-    }
-
-    if (options.showProgress !== false) {
-      progressBar.update(totalProcessed, {
-        updated: totalUpdated,
-        errors: totalErrors,
-      });
-    }
-  }
-
-  if (options.showProgress !== false) {
-    progressBar.stop();
-  }
-
-  console.log(`\n${'='.repeat(60)}`);
-  console.log(`🎯 ARELA PATH PROPAGATION COMPLETED`);
-  console.log(`${'='.repeat(60)}`);
-  console.log(`   📋 Pedimento records processed: ${totalProcessed}`);
-  console.log(`   ✅ Related files updated: ${totalUpdated}`);
-  console.log(`   ❌ Errors: ${totalErrors}`);
-  console.log(`${'='.repeat(60)}\n`);
-
-  // Write comprehensive log summary
-  writeLog(
-    `🎯 ARELA PATH PROPAGATION COMPLETED - Summary: Processed: ${totalProcessed} pedimento records, Updated: ${totalUpdated} related files, Errors: ${totalErrors}`,
-  );
-
-  // Ensure logs are flushed
-  flushLogBuffer();
-
-  return {
-    processedCount: totalProcessed,
-    updatedCount: totalUpdated,
-    errorCount: totalErrors,
-  };
-};
-
-/**
- * Helper function to query files that need to be uploaded
- * These are files that have been detected but not yet uploaded
- * Uses the same RFC filtering logic as uploadFilesByRfc for consistency
- */
-const getFilesReadyForUpload = async (options = {}) => {
-  if (!supabase) {
-    throw new Error('Supabase client not initialized');
-  }
-
-  console.log('🔍 Querying files ready for upload...');
-
-  // Check if UPLOAD_RFCS is configured
-  if (!uploadRfcs || uploadRfcs.length === 0) {
-    console.log(
-      'ℹ️  No UPLOAD_RFCS configured. Please set UPLOAD_RFCS environment variable to see files ready for upload.',
-    );
-    console.log(
-      '   Example: UPLOAD_RFCS="RFC123456789|RFC987654321|RFC555444333"',
-    );
-    return [];
-  }
-
-  console.log(`🎯 Using RFC filter: ${uploadRfcs.join(', ')}`);
-
-  // Step 1: Find pedimento_simplificado documents for the specified RFCs that have arela_path
-  console.log(
-    '🎯 Finding pedimento_simplificado documents for specified RFCs with arela_path...',
-  );
-  const { data: pedimentoRecords, error: pedimentoError } = await supabase
-    .from('uploader')
-    .select('arela_path')
-    .eq('document_type', 'pedimento_simplificado')
-    .in('rfc', uploadRfcs)
-    .not('arela_path', 'is', null);
-
-  if (pedimentoError) {
-    throw new Error(
-      `Error querying pedimento_simplificado records: ${pedimentoError.message}`,
-    );
-  }
-
-  if (!pedimentoRecords || pedimentoRecords.length === 0) {
-    console.log('ℹ️  No pedimento_simplificado records with arela_path found');
-    return [];
-  }
-
-  // Get unique arela_paths
-  const uniqueArelaPaths = [
-    ...new Set(pedimentoRecords.map((r) => r.arela_path)),
-  ];
-  console.log(
-    `📋 Found ${pedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths`,
-  );
-
-  // Step 2: Find all related files with these arela_paths that haven't been uploaded yet
-  console.log('🔍 Finding all related files that need to be uploaded...');
-
-  // Process arela_paths in chunks to avoid URI length limits
-  let allReadyFiles = [];
-  const chunkSize = 50;
-
-  for (let i = 0; i < uniqueArelaPaths.length; i += chunkSize) {
-    const pathChunk = uniqueArelaPaths.slice(i, i + chunkSize);
-
-    const { data: chunkFiles, error: chunkError } = await supabase
-      .from('uploader')
-      .select(
-        'id, original_path, arela_path, filename, rfc, document_type, status',
-      )
-      .in('arela_path', pathChunk)
-      .neq('status', 'file-uploaded')
-      .not('original_path', 'is', null);
-
-    if (chunkError) {
-      throw new Error(
-        `Error querying files for arela_paths chunk: ${chunkError.message}`,
-      );
-    }
-
-    if (chunkFiles && chunkFiles.length > 0) {
-      allReadyFiles = allReadyFiles.concat(chunkFiles);
-    }
-  }
-
-  const readyFiles = allReadyFiles;
-
-  console.log(`📋 Found ${readyFiles?.length || 0} files ready for upload`);
-
-  if (readyFiles && readyFiles.length > 0) {
-    // Group by document type for summary
-    const byDocType = readyFiles.reduce((acc, file) => {
-      const docType = file.document_type || 'Unknown';
-      acc[docType] = (acc[docType] || 0) + 1;
-      return acc;
-    }, {});
-
-    console.log('📊 Files by document type:');
-    for (const [docType, count] of Object.entries(byDocType)) {
-      console.log(`   ${docType}: ${count} files`);
-    }
-
-    // Group by RFC
-    const byRfc = readyFiles.reduce((acc, file) => {
-      const rfc = file.rfc || 'No RFC';
-      acc[rfc] = (acc[rfc] || 0) + 1;
-      return acc;
-    }, {});
-
-    console.log('📊 Files by RFC:');
-    for (const [rfc, count] of Object.entries(byRfc)) {
-      console.log(`   ${rfc}: ${count} files`);
-    }
-  }
-
-  return readyFiles || [];
-};
-
-program
-  .name('arela-uploader')
-  .description(
-    'CLI to upload folders to Arela API or Supabase Storage with automatic processing\n\n' +
-      'Status workflow:\n' +
-      '  fs-stats → detected → file-uploaded\n' +
-      '  ├─ Phase 1: --stats-only (collects filesystem stats, status: fs-stats)\n' +
-      '  ├─ Phase 2: --detect-pdfs (detects document types, status: detected)\n' +
-      '  ├─ Phase 3: --propagate-arela-path (organizes files by pedimento)\n' +
-      '  └─ Phase 4: --upload-by-rfc (uploads files, status: file-uploaded)\n\n' +
-      'Use --query-ready-files to see files ready for upload (status: detected with arela_path)',
-  )
-  .option('-v, --version', 'output the version number')
-  .option('-p, --prefix <prefix>', 'Prefix path in bucket', '')
-  .option('-b, --bucket <bucket>', 'Bucket name override')
-  .option('--force-supabase', 'Force direct Supabase upload (skip API)')
-  .option(
-    '--no-auto-detect',
-    'Disable automatic file detection (API mode only)',
-  )
-  .option(
-    '--no-auto-organize',
-    'Disable automatic file organization (API mode only)',
-  )
-  .option(
-    '-c, --concurrency <number>',
-    'Files per batch for processing (default: 10)',
-    '10',
-  )
-  .option('--batch-size <number>', 'API batch size (default: 10)', '10')
-  .option('--show-stats', 'Show detailed processing statistics')
-  .option(
-    '--folder-structure <structure>',
-    'Custom folder structure (e.g., "2024/4023260" or "cliente1/pedimentos")',
-  )
-  .option(
-    '--auto-detect-structure',
-    'Automatically detect year/pedimento from file paths',
-  )
-  .option('--client-path <path>', 'Client path for metadata tracking')
-  .option(
-    '--stats-only',
-    'Phase 1: Only read filesystem stats and insert to database (no file reading or detection)',
-  )
-  .option('--no-detect', 'Disable document type detection in stats-only mode')
-  .option(
-    '--detect-pdfs',
-    'Phase 2: Process PDF files in database for pedimento-simplificado detection',
-  )
-  .option(
-    '--propagate-arela-path',
-    'Phase 3: Propagate arela_path from pedimento_simplificado records to related files with same base path',
-  )
-  .option(
-    '--upload-by-rfc',
-    'Phase 4: Upload files to Arela API based on RFC values from UPLOAD_RFCS environment variable',
-  )
-  .option(
-    '--run-all-phases',
-    'Run all 4 phases in sequence: stats → detect → propagate → upload',
-  )
-  .option(
-    '--query-ready-files',
-    'Query and display files that are ready for upload (have been detected but not uploaded)',
-  )
-  .action(async (options) => {
-    if (options.version) {
-      console.log(packageVersion);
-      process.exit(0);
-    }
-
-    // Handle detect-pdfs option (Phase 2)
-    if (options.detectPdfs) {
-      console.log('🔍 Starting Phase 2: PDF Detection');
-      await checkCredentials(true); // Force Supabase mode
-
-      const result = await detectPedimentosInDatabase({
-        batchSize: parseInt(options.batchSize) || 10,
-      });
-
-      console.log(
-        `✅ Phase 2 Complete: ${result.detectedCount} detected, ${result.errorCount} errors`,
-      );
-      return;
-    }
-
-    // Handle query-ready-files option
-    if (options.queryReadyFiles) {
-      await checkCredentials(true); // Force Supabase mode
-
-      const readyFiles = await getFilesReadyForUpload();
-
-      if (readyFiles.length === 0) {
-        console.log('ℹ️  No files are currently ready for upload');
-        console.log(
-          '   Tip: Run --detect-pdfs and --propagate-arela-path first to prepare files for upload',
-        );
-      } else {
-        console.log(`\n📋 ${readyFiles.length} files are ready for upload!`);
-        console.log('   Use --upload-by-rfc to upload them to Arela API');
-      }
-
-      return;
-    }
-
-    // Handle run-all-phases option
-    if (options.runAllPhases) {
-      console.log('🚀 Starting all 4 phases in sequence...');
-      await checkCredentials(true); // Force Supabase mode
-
-      // Phase 1: Stats collection
-      console.log('\n📊 === PHASE 1: Filesystem Stats ===');
-      options.statsOnly = true;
-      // Continue with normal processing to run Phase 1
-
-      // The rest will be handled after Phase 1 completes
-    }
-
-    // Handle propagate-arela-path option
-    if (options.propagateArelaPath) {
-      // Initialize Supabase credentials for propagation
-      await checkCredentials(true); // Force Supabase mode
-
-      const result = await propagateArelaPath({
-        showProgress: options.showStats || true,
-      });
-
-      if (result.errorCount > 0) {
-        process.exit(1);
-      }
-      return;
-    }
-
-    // Handle upload-by-rfc option
-    if (options.uploadByRfc) {
-      // RFC upload needs both Supabase (for database queries) and API (for uploads)
-      await checkCredentials(false); // Initialize API mode
-
-      // Also initialize Supabase for database queries
-      if (!supabase) {
-        if (!supabaseUrl || !supabaseKey) {
-          console.error(
-            '❌ RFC upload requires Supabase credentials for database queries.',
-          );
-          console.error(
-            '   Please set SUPABASE_URL and SUPABASE_KEY environment variables.',
-          );
-          process.exit(1);
-        }
-
-        supabase = createClient(supabaseUrl, supabaseKey);
-        console.log('✅ Connected to Supabase for database queries');
-      }
-
-      const result = await uploadFilesByRfc({
-        showProgress: options.showStats || true,
-        batchSize: parseInt(options.batchSize) || 10,
-        folderStructure: options.folderStructure,
-      });
-
-      if (result.errorCount > 0) {
-        process.exit(1);
-      }
-      return;
-    }
-
-    // Initialize credentials with force supabase flag (for stats mode, always need Supabase)
-    await checkCredentials(options.forceSupabase || options.statsOnly);
-
-    if (!basePath || !sources || sources.length === 0) {
-      console.error(
-        '⚠️ UPLOAD_BASE_PATH or UPLOAD_SOURCES not defined in environment variables.',
-      );
-      process.exit(1);
-    }
-
-    const batchSize = parseInt(options.batchSize) || 10;
-    const concurrency = parseInt(options.concurrency) || 10;
-
-    if (options.statsOnly) {
-      console.log(
-        '📊 Mode: Stats Only - Reading file stats and inserting to uploader table',
-      );
-      console.log('🚫 Files will NOT be uploaded');
-      if (options.detect !== false) {
-        console.log('🔍 Document type detection ENABLED for supported files');
-      } else {
-        console.log('🔍 Document type detection DISABLED');
-      }
-    } else {
-      console.log(
-        `🚀 Mode: ${apiMode ? 'Arela API with auto-processing' : 'Direct Supabase'}`,
-      );
-    }
-    console.log(`📦 Batch size: ${batchSize}`);
-    console.log(`⚡ Concurrency: ${concurrency}`);
-
-    const processedPaths = getProcessedPaths();
-    let globalSuccess = 0;
-    let globalDetected = 0;
-    let globalOrganized = 0;
-    let globalFailure = 0;
-    let globalSkipped = 0;
-
-    for (const folder of sources) {
-      const sourcePath = path.resolve(basePath, folder).replace(/\\/g, '/');
-      console.log(`📂 Processing folder: ${sourcePath}`);
-
-      try {
-        const stats = fs.statSync(sourcePath);
-        const files = stats.isDirectory()
-          ? await globby([`${sourcePath}/**/*`], { onlyFiles: true })
-          : [sourcePath];
-
-        console.log(`📊 Found ${files.length} files to process`);
-
-        const result = await processFilesInBatches(
-          files,
-          batchSize,
-          options,
-          basePath,
-          folder,
-          sourcePath,
-          processedPaths,
-        );
-
-        globalSuccess += result.successCount;
-        globalDetected += result.detectedCount || 0;
-        globalOrganized += result.organizedCount || 0;
-        globalFailure += result.failureCount;
-        globalSkipped += result.skippedCount;
-
-        console.log(`\n📦 Summary for ${folder}:`);
-        if (options.statsOnly) {
-          console.log(`   📊 Stats recorded: ${result.successCount}`);
-        } else {
-          console.log(`   ✅ Uploaded: ${result.successCount}`);
-          if (apiMode) {
-            console.log(`   🔍 Detected: ${result.detectedCount || 0}`);
-            console.log(`   📁 Organized: ${result.organizedCount || 0}`);
-          }
-        }
-        console.log(`   ❌ Errors: ${result.failureCount}`);
-        if (options.statsOnly) {
-          console.log(`   ⏭️  Duplicates: ${result.skippedCount}`);
-        } else {
-          console.log(`   ⏭️  Skipped: ${result.skippedCount}`);
-        }
-
-        writeLog(
-          `📦 Summary for ${folder}: Success: ${result.successCount}, Detected: ${result.detectedCount || 0}, Organized: ${result.organizedCount || 0}, Errors: ${result.failureCount}, ${options.statsOnly ? 'Duplicates' : 'Skipped'}: ${result.skippedCount}`,
-        );
-      } catch (err) {
-        console.error(`⚠️ Error processing folder ${folder}:`, err.message);
-        writeLog(`⚠️ Error processing folder ${folder}: ${err.message}`);
-        globalFailure++;
-      }
-    }
-
-    console.log(`\n${'='.repeat(60)}`);
-    if (options.statsOnly) {
-      console.log(`📊 STATS COLLECTION COMPLETED`);
-      console.log(`${'='.repeat(60)}`);
-      console.log(`   📊 Total stats recorded: ${globalSuccess}`);
-    } else {
-      console.log(`🎯 ${apiMode ? 'ARELA API' : 'SUPABASE'} UPLOAD COMPLETED`);
-      console.log(`${'='.repeat(60)}`);
-      console.log(`   ✅ Total uploaded: ${globalSuccess}`);
-      if (apiMode) {
-        console.log(`   🔍 Total detected: ${globalDetected}`);
-        console.log(`   📁 Total organized: ${globalOrganized}`);
-      }
-    }
-    if (options.statsOnly) {
-      console.log(`   ⏭️  Total duplicates: ${globalSkipped}`);
-    } else {
-      console.log(`   ⏭️  Total skipped: ${globalSkipped}`);
-    }
-    console.log(`   ❌ Total errors: ${globalFailure}`);
-    console.log(`   📜 Log file: ${logFilePath}`);
-    console.log(`${'='.repeat(60)}\n`);
-
-    // Continue with remaining phases if running all phases
-    if (options.runAllPhases && options.statsOnly) {
-      try {
-        // Phase 2: PDF Detection
-        console.log('\n🔍 === PHASE 2: PDF Detection ===');
-        const detectionResult = await detectPedimentosInDatabase({
-          batchSize: parseInt(options.batchSize) || 10,
-        });
-        console.log(
-          `✅ Phase 2 Complete: ${detectionResult.detectedCount} detected, ${detectionResult.errorCount} errors`,
-        );
-
-        // Phase 3: Propagate arela_path
-        console.log('\n📁 === PHASE 3: Propagate Arela Paths ===');
-        const propagateResult = await propagateArelaPath({
-          showProgress: options.showStats || true,
-        });
-        console.log(
-          `✅ Phase 3 Complete: ${propagateResult.updatedCount || 0} paths propagated`,
-        );
-
-        // Phase 4: Upload by RFC
-        if (uploadRfcs && uploadRfcs.length > 0) {
-          console.log('\n🚀 === PHASE 4: Upload by RFC ===');
-
-          // Initialize API mode for uploads
-          await checkCredentials(false);
-
-          const uploadResult = await uploadFilesByRfc({
-            showProgress: options.showStats || true,
-            batchSize: parseInt(options.batchSize) || 10,
-            folderStructure: options.folderStructure,
-          });
-          console.log(`✅ Phase 4 Complete: Upload finished`);
-        } else {
-          console.log('\n⚠️ === PHASE 4: Upload by RFC ===');
-          console.log(
-            '⚠️ UPLOAD_RFCS environment variable not configured, skipping Phase 4',
-          );
-        }
-
-        console.log('\n🎉 All 4 phases completed successfully!');
-      } catch (error) {
-        console.error(`❌ Error in multi-phase execution:`, error.message);
-        process.exit(1);
-      }
-    }
-
-    if (
-      options.showStats &&
-      (sanitizationCache.size > 0 || pathDetectionCache.size > 0)
-    ) {
-      console.log(`📊 Performance Statistics:`);
-      if (sanitizationCache.size > 0) {
-        console.log(
-          `   🗂️  Sanitization cache entries: ${sanitizationCache.size}`,
-        );
-      }
-      if (pathDetectionCache.size > 0) {
-        console.log(
-          `   📁  Path detection cache entries: ${pathDetectionCache.size}`,
-        );
-      }
-    }
-
-    // OPTIMIZED: Ensure log buffer is flushed before exit
-    flushLogBuffer();
-  });
-
-program.parse();