@arela/uploader 0.0.10 → 0.0.11

package/src/index.js

@@ -7,7 +7,6 @@ import fs from 'fs';
 import { globby } from 'globby';
 import mime from 'mime-types';
 import { createRequire } from 'module';
-import ora from 'ora';
 import path from 'path';
 
 const require = createRequire(import.meta.url);
@@ -27,60 +26,236 @@ const sources = process.env.UPLOAD_SOURCES?.split('|')
 
 const supabase = createClient(supabaseUrl, supabaseKey);
 
-// Enhanced sanitization function specifically for file names
+// Pre-compiled regex patterns for better performance
+const SANITIZATION_PATTERNS = [
+  // Character replacements (grouped for efficiency)
+  [/[áàâäãåāăą]/gi, 'a'],
+  [/[éèêëēĕėę]/gi, 'e'],
+  [/[íìîïīĭį]/gi, 'i'],
+  [/[óòôöõōŏő]/gi, 'o'],
+  [/[úùûüūŭů]/gi, 'u'],
+  [/[ñň]/gi, 'n'],
+  [/[ç]/gi, 'c'],
+  [/[ý]/gi, 'y'],
+  // Korean characters (compiled once)
+  [/[멕]/g, 'meok'],
+  [/[시]/g, 'si'],
+  [/[코]/g, 'ko'],
+  [/[용]/g, 'yong'],
+  [/[가-힣]/g, 'kr'],
+  // Unicode diacritics (after normalize)
+  [/[\u0300-\u036f]/g, ''],
+  // Problematic symbols
+  [/[\\?%*:|"<>[\]~`^]/g, '-'],
+  [/[{}]/g, '-'],
+  [/[&]/g, 'and'],
+  [/[()]/g, ''], // Remove parentheses
+  // Cleanup patterns
+  [/\s+/g, '-'], // Replace spaces with dashes
+  [/-+/g, '-'], // Replace multiple dashes with single dash
+  [/^-+|-+$/g, ''], // Remove leading/trailing dashes
+  [/^\.+/, ''], // Remove leading dots
+  [/[^\w.-]/g, ''], // Remove any remaining non-alphanumeric chars
+];
+
+// Cache for sanitized filenames to avoid repeated processing
+const sanitizationCache = new Map();
+
+// Enhanced sanitization function with caching and pre-compiled regex
 const sanitizeFileName = (fileName) => {
+  // Check cache first
+  if (sanitizationCache.has(fileName)) {
+    return sanitizationCache.get(fileName);
+  }
+
   // Get file extension
   const ext = path.extname(fileName);
   const nameWithoutExt = path.basename(fileName, ext);
 
-  // Replace problematic characters with safe alternatives
-  let sanitized = nameWithoutExt
-    // First, handle common character replacements
-    .replace(/[áàâäãåāăą]/gi, 'a')
-    .replace(/[éèêëēĕėę]/gi, 'e')
-    .replace(/[íìîïīĭį]/gi, 'i')
-    .replace(/[óòôöõōŏő]/gi, 'o')
-    .replace(/[úùûüūŭů]/gi, 'u')
-    .replace(/[ñň]/gi, 'n')
-    .replace(/[ç]/gi, 'c')
-    .replace(/[ý]/gi, 'y')
-    // Handle Korean characters more comprehensively
-    .replace(/[멕]/g, 'meok')
-    .replace(/[시]/g, 'si')
-    .replace(/[코]/g, 'ko')
-    .replace(/[용]/g, 'yong')
-    .replace(/[가-힣]/g, 'kr') // Replace any remaining Korean chars with 'kr'
-    // Normalize unicode and remove diacritics
-    .normalize('NFD')
-    .replace(/[\u0300-\u036f]/g, '') // Remove diacritics
-    // Replace problematic symbols
-    .replace(/[\\?%*:|"<>[\]~`^]/g, '-')
-    .replace(/[{}]/g, '-')
-    .replace(/[&]/g, 'and')
-    .replace(/[()]/g, '') // Remove parentheses
-    // Clean up spaces and dashes
-    .replace(/\s+/g, '-') // Replace spaces with dashes
-    .replace(/-+/g, '-') // Replace multiple dashes with single dash
-    .replace(/^-+|-+$/g, '') // Remove leading/trailing dashes
-    .replace(/^\.+/, '') // Remove leading dots
-    .replace(/[^\w.-]/g, ''); // Remove any remaining non-alphanumeric chars except dots and dashes
+  // Fast path for already clean filenames
+  if (/^[a-zA-Z0-9._-]+$/.test(nameWithoutExt)) {
+    const result = fileName;
+    sanitizationCache.set(fileName, result);
+    return result;
+  }
+
+  // Normalize unicode first (more efficient to do once)
+  let sanitized = nameWithoutExt.normalize('NFD');
+
+  // Apply all sanitization patterns
+  for (const [pattern, replacement] of SANITIZATION_PATTERNS) {
+    sanitized = sanitized.replace(pattern, replacement);
+  }
 
   // Ensure the filename is not empty
   if (!sanitized) {
     sanitized = 'unnamed_file';
   }
 
-  return sanitized + ext;
+  const result = sanitized + ext;
+
+  // Cache the result for future use
+  sanitizationCache.set(fileName, result);
+
+  return result;
+};
+
+// Pre-compiled regex patterns for path sanitization
+const PATH_SANITIZATION_PATTERNS = [
+  [/[\\?%*:|"<>[\]~]/g, '-'],
+  [/ +/g, ' '],
+  [/^\.+/, ''],
+  [/\/+/g, '/'],
+];
+
+// Cache for sanitized paths
+const pathSanitizationCache = new Map();
+
+// Batch logging system for performance
+class LogBatcher {
+  constructor(batchSize = 50, flushInterval = 5000) {
+    this.batch = [];
+    this.batchSize = batchSize;
+    this.flushInterval = flushInterval;
+    this.lastFlush = Date.now();
+    this.flushTimer = null;
+  }
+
+  add(logEntry) {
+    this.batch.push({
+      filename: path.basename(logEntry.file),
+      path: logEntry.uploadPath,
+      status: logEntry.status,
+      message: logEntry.message,
+    });
+
+    // Auto-flush if batch is full or enough time has passed
+    if (
+      this.batch.length >= this.batchSize ||
+      Date.now() - this.lastFlush > this.flushInterval
+    ) {
+      this.flush();
+    }
+  }
+
+  async flush() {
+    if (this.batch.length === 0) return;
+
+    const logsToSend = [...this.batch];
+    this.batch = [];
+    this.lastFlush = Date.now();
+
+    // Clear any pending timer
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+
+    try {
+      const { error } = await supabase.from('upload_logs').insert(logsToSend);
+      if (error) {
+        console.error(
+          `⚠️ Error saving batch of ${logsToSend.length} logs to Supabase: ${error.message}`,
+        );
+        // Re-add failed logs to batch for retry (optional)
+        this.batch.unshift(...logsToSend);
+      } else {
+        // Only show verbose output if requested
+        if (process.env.LOG_BATCH_VERBOSE === 'true') {
+          console.log(`📊 Flushed ${logsToSend.length} logs to Supabase`);
+        }
+      }
+    } catch (err) {
+      console.error(`⚠️ Error during batch flush: ${err.message}`);
+      // Re-add failed logs to batch for retry (optional)
+      this.batch.unshift(...logsToSend);
+    }
+  }
+
+  // Schedule auto-flush if not already scheduled
+  scheduleFlush() {
+    if (!this.flushTimer && this.batch.length > 0) {
+      this.flushTimer = setTimeout(() => {
+        this.flush();
+      }, this.flushInterval);
+    }
+  }
+
+  // Force flush all pending logs (called at end of process)
+  async forceFlush() {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+    await this.flush();
+  }
+}
+
+// Global log batcher instance
+const logBatcher = new LogBatcher();
+
+// Function to manage cache size (prevent memory issues in long sessions)
+const manageCaches = () => {
+  const MAX_CACHE_SIZE = 1000;
+
+  if (sanitizationCache.size > MAX_CACHE_SIZE) {
+    // Keep only the most recent 500 entries
+    const entries = Array.from(sanitizationCache.entries());
+    sanitizationCache.clear();
+    entries.slice(-500).forEach(([key, value]) => {
+      sanitizationCache.set(key, value);
+    });
+  }
+
+  if (pathSanitizationCache.size > MAX_CACHE_SIZE) {
+    const entries = Array.from(pathSanitizationCache.entries());
+    pathSanitizationCache.clear();
+    entries.slice(-500).forEach(([key, value]) => {
+      pathSanitizationCache.set(key, value);
+    });
+  }
 };
 
-const sanitizePath = (path) =>
-  path
-    .replace(/[\\?%*:|"<>[\]~]/g, '-')
-    .replace(/ +/g, ' ')
-    .replace(/^\.+/, '')
-    .replace(/\/+/g, '/');
+const sanitizePath = (inputPath) => {
+  // Check cache first
+  if (pathSanitizationCache.has(inputPath)) {
+    return pathSanitizationCache.get(inputPath);
+  }
+
+  // Fast path for already clean paths
+  if (!/[\\?%*:|"<>[\]~]|^ +|^\.+|\/\/+/.test(inputPath)) {
+    pathSanitizationCache.set(inputPath, inputPath);
+    return inputPath;
+  }
+
+  let sanitized = inputPath;
+
+  // Apply path sanitization patterns
+  for (const [pattern, replacement] of PATH_SANITIZATION_PATTERNS) {
+    sanitized = sanitized.replace(pattern, replacement);
+  }
+
+  // Cache the result
+  pathSanitizationCache.set(inputPath, sanitized);
+
+  return sanitized;
+};
 
 const sendLogToSupabase = async ({ file, uploadPath, status, message }) => {
+  // Add to batch instead of sending immediately
+  logBatcher.add({ file, uploadPath, status, message });
+
+  // Schedule auto-flush if needed
+  logBatcher.scheduleFlush();
+};
+
+// Enhanced version for immediate sending (used for critical errors)
+const sendLogToSupabaseImmediate = async ({
+  file,
+  uploadPath,
+  status,
+  message,
+}) => {
   const { error } = await supabase.from('upload_logs').insert([
     {
       filename: path.basename(file),
@@ -91,7 +266,9 @@ const sendLogToSupabase = async ({ file, uploadPath, status, message }) => {
   ]);
 
   if (error) {
-    console.error(`⚠️ Error saving the log to Supabase: ${error.message}`);
+    console.error(
+      `⚠️ Error saving immediate log to Supabase: ${error.message}`,
+    );
   }
 };
 
@@ -101,7 +278,7 @@ const checkCredentials = async () => {
       '⚠️ Missing Supabase credentials. Please set SUPABASE_URL, SUPABASE_KEY, and SUPABASE_BUCKET in your environment variables.',
     );
     writeLog('⚠️ Missing Supabase credentials.');
-    await sendLogToSupabase({
+    await sendLogToSupabaseImmediate({
       file: 'Error',
       uploadPath: 'Error',
       status: 'error',
@@ -115,7 +292,7 @@ const checkCredentials = async () => {
     if (error) {
       console.error('⚠️ Error connecting to Supabase:', error.message);
       writeLog(`⚠️ Error connecting to Supabase: ${error.message}`);
-      await sendLogToSupabase({
+      await sendLogToSupabaseImmediate({
         file: 'Error',
         uploadPath: 'Error',
         status: 'error',
@@ -126,7 +303,7 @@ const checkCredentials = async () => {
   } catch (err) {
     console.error('⚠️ Error:', err.message);
     writeLog(`⚠️ Error: ${err.message}`);
-    await sendLogToSupabase({
+    await sendLogToSupabaseImmediate({
       file: 'Error',
       uploadPath: 'Error',
       status: 'error',
@@ -147,7 +324,7 @@ const fileExistsInBucket = async (pathInBucket) => {
   if (error) {
     console.error(`⚠️ Could not verify duplicate: ${error.message}`);
     writeLog(`⚠️ Could not verify duplicate: ${error.message}`);
-    await sendLogToSupabase({
+    await sendLogToSupabaseImmediate({
       file: 'Error',
       uploadPath: 'Error',
       status: 'error',
@@ -277,6 +454,279 @@ const uploadWithRetry = async (uploadFn, maxRetries = 5, delayMs = 2000) => {
   };
 };
 
+// Function to process a single file
+const processFile = async (
+  file,
+  options,
+  basePath,
+  folder,
+  sourcePath,
+  processedPaths,
+) => {
+  let currentFile = file;
+  let result = {
+    success: false,
+    skipped: false,
+    error: null,
+    message: '',
+  };
+
+  try {
+    // Check if we need to rename the file
+    if (options.renameFiles) {
+      const originalName = path.basename(file);
+      const sanitizedName = sanitizeFileName(originalName);
+
+      if (originalName !== sanitizedName) {
+        const newFilePath = path.join(path.dirname(file), sanitizedName);
+
+        if (options.dryRun) {
+          result.message = `Would rename: ${originalName} → ${sanitizedName}`;
+          result.skipped = true;
+          return result;
+        } else {
+          try {
+            fs.renameSync(file, newFilePath);
+            currentFile = newFilePath;
+            writeLog(`RENAMED: ${originalName} → ${sanitizedName}`);
+            await sendLogToSupabase({
+              file: originalName,
+              uploadPath: sanitizedName,
+              status: 'renamed',
+              message: `Renamed from ${originalName}`,
+            });
+          } catch (renameError) {
+            result.error = `Failed to rename ${originalName}: ${renameError.message}`;
+            writeLog(`RENAME_ERROR: ${originalName} | ${renameError.message}`);
+            return result;
+          }
+        }
+      }
+    }
+
+    const content = fs.readFileSync(currentFile);
+    const relativePathRaw = path
+      .relative(basePath, currentFile)
+      .replace(/^[\\/]+/, '')
+      .replace(/\\/g, '/');
+
+    // Always sanitize the filename for upload path
+    const pathParts = relativePathRaw.split('/');
+    const originalFileName = pathParts[pathParts.length - 1];
+    const sanitizedFileName = sanitizeFileName(originalFileName);
+    pathParts[pathParts.length - 1] = sanitizedFileName;
+    const sanitizedRelativePath = pathParts.join('/');
+
+    const uploadPathRaw = options.prefix
+      ? path.posix.join(options.prefix, sanitizedRelativePath)
+      : sanitizedRelativePath;
+    const uploadPath = sanitizePath(uploadPathRaw);
+
+    if (
+      uploadPath !== uploadPathRaw ||
+      originalFileName !== sanitizedFileName
+    ) {
+      writeLog(`SANITIZED: ${relativePathRaw} → ${uploadPath}`);
+      await sendLogToSupabase({
+        file: currentFile,
+        uploadPath: relativePathRaw,
+        status: 'sanitized',
+        message: `Sanitized to ${uploadPath} (Arela Version: ${version})`,
+      });
+    }
+
+    if (processedPaths.has(uploadPath)) {
+      result.skipped = true;
+      result.message = `Already processed (log): ${currentFile}`;
+      return result;
+    }
+
+    const contentType = mime.lookup(currentFile) || 'application/octet-stream';
+
+    const exists = await fileExistsInBucket(uploadPath);
+
+    if (exists) {
+      result.skipped = true;
+      result.message = `Skipped (already exists): ${currentFile}`;
+      writeLog(`SKIPPED: ${currentFile} -> ${uploadPath}`);
+      await sendLogToSupabase({
+        file: currentFile,
+        uploadPath,
+        status: 'skipped',
+        message: 'Already exists in bucket',
+      });
+      return result;
+    }
+
+    const { error } = await uploadWithRetry(() =>
+      supabase.storage.from(bucket).upload(uploadPath, content, {
+        upsert: true,
+        contentType,
+        metadata: {
+          originalName: path.basename(currentFile),
+          sanitizedName: path.basename(uploadPath),
+          clientPath: path.posix.join(
+            basePath,
+            folder,
+            path.relative(sourcePath, currentFile).replace(/\\/g, '/'),
+          ),
+          arelaVersion: version,
+        },
+      }),
+    );
+
+    if (error) {
+      result.error = error.message || JSON.stringify(error);
+      writeLog(`ERROR: ${currentFile} -> ${uploadPath} | ${result.error}`);
+      await sendLogToSupabase({
+        file: currentFile,
+        uploadPath,
+        status: 'error',
+        message: result.error,
+      });
+    } else {
+      result.success = true;
+      result.message = `Uploaded ${currentFile} -> ${uploadPath}`;
+      writeLog(`SUCCESS: ${currentFile} -> ${uploadPath}`);
+      await sendLogToSupabase({
+        file: currentFile,
+        uploadPath,
+        status: 'success',
+        message: 'Uploaded successfully',
+      });
+    }
+  } catch (err) {
+    result.error = err.message || JSON.stringify(err);
+    writeLog(`ERROR: ${currentFile} | ${result.error}`);
+    await sendLogToSupabase({
+      file: currentFile,
+      uploadPath: currentFile,
+      status: 'error',
+      message: result.error,
+    });
+  }
+
+  return result;
+};
+
+// Function to process files in parallel batches
+const processFilesInBatches = async (
+  files,
+  batchSize,
+  options,
+  basePath,
+  folder,
+  sourcePath,
+  processedPaths,
+) => {
+  let successCount = 0;
+  let failureCount = 0;
+  let skippedCount = 0;
+
+  // Buffer for messages to show after progress bar completes
+  const messageBuffer = [];
+
+  const progressBar = new cliProgress.SingleBar({
+    format:
+      '📂 Processing [{bar}] {percentage}% | {value}/{total} files | Success: {successCount} | Errors: {failureCount} | Skipped: {skippedCount}',
+    barCompleteChar: '█',
+    barIncompleteChar: '░',
+    hideCursor: true,
+  });
+
+  progressBar.start(files.length, 0, {
+    successCount: 0,
+    failureCount: 0,
+    skippedCount: 0,
+  });
+
+  for (let i = 0; i < files.length; i += batchSize) {
+    const batch = files.slice(i, i + batchSize);
+
+    // Process batch in parallel
+    const batchResults = await Promise.all(
+      batch.map((file) =>
+        processFile(
+          file,
+          options,
+          basePath,
+          folder,
+          sourcePath,
+          processedPaths,
+        ),
+      ),
+    );
+
+    // Update counters and buffer messages (don't print them yet)
+    for (const result of batchResults) {
+      if (result.success) {
+        successCount++;
+        // Only buffer verbose success messages if needed
+        if (
+          process.env.UPLOAD_VERBOSE === 'true' &&
+          result.message &&
+          !result.error
+        ) {
+          messageBuffer.push(`✅ ${result.message}`);
+        }
+      } else if (result.skipped) {
+        skippedCount++;
+        // Only buffer verbose skip messages if needed
+        if (process.env.UPLOAD_VERBOSE === 'true' && result.message) {
+          messageBuffer.push(`⏭️  ${result.message}`);
+        }
+      } else if (result.error) {
+        failureCount++;
+        // Always buffer error messages to show later
+        messageBuffer.push(`❌ ${result.error}`);
+      }
+    }
+
+    progressBar.update(i + batch.length, {
+      successCount,
+      failureCount,
+      skippedCount,
+    });
+
+    // Manage cache size periodically (every 100 files processed)
+    if ((i + batch.length) % 100 === 0) {
+      manageCaches();
+      // Also flush logs every 100 files to maintain responsiveness
+      await logBatcher.flush();
+    }
+
+    // Small delay between batches to prevent overwhelming the server
+    if (i + batchSize < files.length) {
+      await delay(100);
+    }
+  }
+
+  // Stop progress bar cleanly before showing any messages
+  progressBar.stop();
+
+  // Now show buffered messages if there are any important ones to show
+  const errorMessages = messageBuffer.filter((msg) => msg.startsWith('❌'));
+  if (errorMessages.length > 0) {
+    console.log('\n🚨 Errors encountered during processing:');
+    errorMessages.forEach((msg) => console.error(msg));
+  }
+
+  // Show verbose messages only if requested
+  if (process.env.UPLOAD_VERBOSE === 'true') {
+    const otherMessages = messageBuffer.filter((msg) => !msg.startsWith('❌'));
+    if (otherMessages.length > 0) {
+      console.log('\n📝 Detailed processing log:');
+      otherMessages.forEach((msg) => console.log(msg));
+    }
+  }
+
+  return {
+    successCount,
+    failureCount,
+    skippedCount,
+  };
+};
+
 program
   .name('supabase-uploader')
   .description('CLI to upload folders from a base path to Supabase Storage')
@@ -290,6 +740,20 @@ program
     '--dry-run',
     'Show what files would be renamed without actually renaming them',
   )
+  .option(
+    '-c, --concurrency <number>',
+    'Number of files to process concurrently (default: 3)',
+    '3',
+  )
+  .option(
+    '--show-cache-stats',
+    'Show cache statistics for performance analysis',
+  )
+  .option(
+    '--batch-size <number>',
+    'Number of logs to batch before sending to Supabase (default: 50)',
+    '50',
+  )
   .action(async (options) => {
     // Handle version option
     if (options.version) {
@@ -304,6 +768,15 @@ program
       process.exit(1);
     }
 
+    const concurrency = parseInt(options.concurrency) || 3;
+    const batchSize = parseInt(options.batchSize) || 50;
+
+    // Configure log batcher with custom batch size
+    logBatcher.batchSize = batchSize;
+
+    console.log(`🚀 Using concurrency level: ${concurrency}`);
+    console.log(`📦 Using log batch size: ${batchSize}`);
+
     const processedPaths = await getProcessedPaths();
     let globalSuccess = 0;
     let globalFailure = 0;
@@ -318,185 +791,61 @@ program
           ? await globby([`${sourcePath}/**/*`], { onlyFiles: true })
           : [sourcePath];
 
-        const progressBar = new cliProgress.SingleBar({
-          format: '📂 Reading [{bar}] {percentage}% | {value}/{total} files',
-          barCompleteChar: '█',
-          barIncompleteChar: '░',
-          hideCursor: true,
-        });
-        progressBar.start(files.length, 0);
-
-        let successCount = 0;
-        let failureCount = 0;
-
-        for (const file of files) {
-          progressBar.increment();
-
-          let currentFile = file;
-
-          // Check if we need to rename the file
-          if (options.renameFiles) {
-            const originalName = path.basename(file);
-            const sanitizedName = sanitizeFileName(originalName);
-
-            if (originalName !== sanitizedName) {
-              const newFilePath = path.join(path.dirname(file), sanitizedName);
-
-              if (options.dryRun) {
-                console.log(`Would rename: ${originalName} → ${sanitizedName}`);
-                continue;
-              } else {
-                try {
-                  fs.renameSync(file, newFilePath);
-                  currentFile = newFilePath;
-                  console.log(`✅ Renamed: ${originalName} → ${sanitizedName}`);
-                  writeLog(`RENAMED: ${originalName} → ${sanitizedName}`);
-                  await sendLogToSupabase({
-                    file: originalName,
-                    uploadPath: sanitizedName,
-                    status: 'renamed',
-                    message: `Renamed from ${originalName}`,
-                  });
-                } catch (renameError) {
-                  console.error(
-                    `❌ Failed to rename ${originalName}: ${renameError.message}`,
-                  );
-                  writeLog(
-                    `RENAME_ERROR: ${originalName} | ${renameError.message}`,
-                  );
-                  continue;
-                }
-              }
-            }
-          }
+        console.log(`📊 Found ${files.length} files to process`);
 
-          const content = fs.readFileSync(currentFile);
-          const relativePathRaw = path
-            .relative(basePath, currentFile)
-            .replace(/^[\\/]+/, '')
-            .replace(/\\/g, '/');
-
-          // Always sanitize the filename for upload path
-          const pathParts = relativePathRaw.split('/');
-          const originalFileName = pathParts[pathParts.length - 1];
-          const sanitizedFileName = sanitizeFileName(originalFileName);
-          pathParts[pathParts.length - 1] = sanitizedFileName;
-          const sanitizedRelativePath = pathParts.join('/');
-
-          const uploadPathRaw = options.prefix
-            ? path.posix.join(options.prefix, sanitizedRelativePath)
-            : sanitizedRelativePath;
-          const uploadPath = sanitizePath(uploadPathRaw);
-
-          if (
-            uploadPath !== uploadPathRaw ||
-            originalFileName !== sanitizedFileName
-          ) {
-            writeLog(`SANITIZED: ${relativePathRaw} → ${uploadPath}`);
-            await sendLogToSupabase({
-              file: currentFile,
-              uploadPath: relativePathRaw,
-              status: 'sanitized',
-              message: `Sanitized to ${uploadPath} (Arela Version: ${version})`,
-            });
-          }
-
-          if (processedPaths.has(uploadPath)) {
-            ora().info(`⏭️  Already processed (log): ${currentFile}`);
-            continue;
-          }
+        // Process files in parallel batches
+        const { successCount, failureCount, skippedCount } =
+          await processFilesInBatches(
+            files,
+            concurrency,
+            options,
+            basePath,
+            folder,
+            sourcePath,
+            processedPaths,
+          );
 
-          const contentType =
-            mime.lookup(currentFile) || 'application/octet-stream';
+        globalSuccess += successCount;
+        globalFailure += failureCount;
 
-          const spinner = ora(`Checking ${currentFile}...`).start();
-          const exists = await fileExistsInBucket(uploadPath);
-
-          if (exists) {
-            spinner.info(`⏭️  Skipped (already exists): ${currentFile}`);
-            writeLog(`SKIPPED: ${currentFile} -> ${uploadPath}`);
-            await sendLogToSupabase({
-              file: currentFile,
-              uploadPath,
-              status: 'skipped',
-              message: 'Already exists in bucket',
-            });
-            continue;
-          }
+        // Small delay to ensure progress bar is fully cleared
+        await delay(100);
 
-          try {
-            // await delay(5000); // TODO: Remove this delay before production
-
-            spinner.text = `Uploading ${currentFile}...`;
-
-            const { error } = await uploadWithRetry(() =>
-              supabase.storage.from(bucket).upload(uploadPath, content, {
-                upsert: true,
-                contentType,
-                metadata: {
-                  originalName: path.basename(currentFile),
-                  sanitizedName: path.basename(uploadPath),
-                  clientPath: path.posix.join(
-                    basePath,
-                    folder,
-                    path.relative(sourcePath, currentFile).replace(/\\/g, '/'),
-                  ),
-                  arelaVersion: version,
-                },
-              }),
-            );
-
-            if (error) {
-              failureCount++;
-              globalFailure++;
-              const errorMsg = error.message || JSON.stringify(error);
-              spinner.fail(`❌ Failed to upload ${currentFile}: ${errorMsg}`);
-              writeLog(`ERROR: ${currentFile} -> ${uploadPath} | ${errorMsg}`);
-              await sendLogToSupabase({
-                file: currentFile,
-                uploadPath,
-                status: 'error',
-                message: errorMsg,
-              });
-            } else {
-              successCount++;
-              globalSuccess++;
-              spinner.succeed(`✅ Uploaded ${currentFile} -> ${uploadPath}`);
-              writeLog(`SUCCESS: ${currentFile} -> ${uploadPath}`);
-              await sendLogToSupabase({
-                file: currentFile,
-                uploadPath,
-                status: 'success',
-                message: 'Uploaded successfully',
-              });
-            }
-          } catch (err) {
-            failureCount++;
-            globalFailure++;
-            const errorMsg = err.message || JSON.stringify(err);
-            spinner.fail(`❌ Error uploading ${currentFile}: ${errorMsg}`);
-            writeLog(`ERROR: ${currentFile} -> ${uploadPath} | ${errorMsg}`);
-            await sendLogToSupabase({
-              file: currentFile,
-              uploadPath,
-              status: 'error',
-              message: errorMsg,
-            });
-          }
-        }
-
-        progressBar.stop();
-
-        console.log(`\n📦 Upload Summary:`);
+        console.log(`\n📦 Upload Summary for ${folder}:`);
         console.log(`   ✅ Successfully uploaded files: ${successCount}`);
         console.log(`   ❌ Files with errors: ${failureCount}`);
-        console.log(
-          `   ⏭️  Files skipped (already exist): ${files.length - successCount - failureCount}`,
-        );
+        console.log(`   ⏭️  Files skipped (already exist): ${skippedCount}`);
         console.log(`   📜 Log file: ${logFilePath} \n`);
 
+        // Show cache statistics if requested
+        if (options.showCacheStats) {
+          console.log(`📊 Cache Statistics:`);
+          console.log(
+            `   🗂️  Filename sanitization cache: ${sanitizationCache.size} entries`,
+          );
+          console.log(
+            `   📁 Path sanitization cache: ${pathSanitizationCache.size} entries`,
+          );
+          console.log(
+            `   📋 Log batch pending: ${logBatcher.batch.length} entries`,
+          );
+
+          // Calculate cache hit rate (rough estimation)
+          const totalProcessed = successCount + failureCount + skippedCount;
+          const estimatedCacheHitRate =
+            totalProcessed > 0
+              ? Math.round(
+                  ((totalProcessed - sanitizationCache.size) / totalProcessed) *
+                    100,
+                )
+              : 0;
+          console.log(
+            `   🎯 Estimated cache hit rate: ${Math.max(0, estimatedCacheHitRate)}%\n`,
+          );
+        }
+
         writeLog(
-          `📦 Upload Summary for folder ${folder}: Success: ${successCount}, Errors: ${failureCount}, Skipped: ${files.length - successCount - failureCount}`,
+          `📦 Upload Summary for folder ${folder}: Success: ${successCount}, Errors: ${failureCount}, Skipped: ${skippedCount}`,
         );
       } catch (err) {
         console.error(`⚠️ Error processing folder ${folder}:`, err.message);
@@ -507,13 +856,22 @@ program
           status: 'error',
           message: err.message,
         });
+        globalFailure++;
       }
     }
 
-    console.log(`🎯 Upload completed.`);
+    // Force flush any remaining logs before finishing
+    console.log(`\n📤 Flushing remaining logs...`);
+    await logBatcher.forceFlush();
+
+    // Final summary with clear separation
+    console.log(`\n${'='.repeat(50)}`);
+    console.log(`🎯 UPLOAD COMPLETED`);
+    console.log(`${'='.repeat(50)}`);
     console.log(`   ✅ Total uploaded: ${globalSuccess}`);
     console.log(`   ❌ Total with errors: ${globalFailure}`);
     console.log(`   📜 Log file: ${logFilePath}`);
+    console.log(`${'='.repeat(50)}\n`);
   });
 
 program.parse();