@arela/uploader 0.0.8 → 0.0.10

package/README.md

@@ -1 +1,84 @@
 # arela-uploader
+
+CLI tool to upload files and directories to Supabase Storage with automatic file renaming and sanitization.
+
+## Features
+
+- 📁 Upload entire directories or individual files
+- 🔄 Automatic file renaming to handle problematic characters
+- 📝 Comprehensive logging (local and remote)
+- ⚡ Retry mechanism for failed uploads
+- 🎯 Skip duplicate files automatically
+- 📊 Progress bars and detailed summaries
+
+## Installation
+
+```bash
+npm install -g @arela/uploader
+```
+
+## Usage
+
+### Basic Upload
+```bash
+arela -p "my-folder"
+```
+
+### Upload with File Renaming
+For files with accents, special characters, or problematic names:
+
+```bash
+# Preview what files would be renamed (dry run)
+arela --rename-files --dry-run
+
+# Actually rename and upload files
+arela --rename-files -p "documents"
+```
+
+### Options
+
+- `-p, --prefix <prefix>`: Prefix path in bucket (default: "")
+- `-r, --rename-files`: Rename files with problematic characters before uploading
+- `--dry-run`: Show what files would be renamed without actually renaming them
+- `-h, --help`: Display help information
+- `-v, --version`: Display version number
+
+## Environment Variables
+
+Create a `.env` file in your project root:
+
+```env
+SUPABASE_URL=your_supabase_url
+SUPABASE_KEY=your_supabase_anon_key
+SUPABASE_BUCKET=your_bucket_name
+UPLOAD_BASE_PATH=/path/to/your/files
+UPLOAD_SOURCES=folder1|folder2|file.pdf
+```
+
+## File Renaming
+
+The tool automatically handles problematic characters by:
+
+- Removing accents (á → a, ñ → n, etc.)
+- Replacing special characters with safe alternatives
+- Converting spaces to dashes
+- Removing or replacing symbols like `{}[]~^`|"<>?*:`
+- Handling Korean characters and other Unicode symbols
+
+### Examples
+
+| Original | Renamed |
+|----------|---------|
+| `Facturas Importación.pdf` | `Facturas-Importacion.pdf` |
+| `File{with}brackets.pdf` | `File-with-brackets.pdf` |
+| `Document ^& symbols.pdf` | `Document-and-symbols.pdf` |
+| `CI & PL-20221212(멕시코용).xls` | `CI-and-PL-20221212.xls` |
+
+## Logging
+
+The tool maintains logs both locally (`upload.log`) and remotely in your Supabase database. Logs include:
+
+- Upload status (success/error/skipped)
+- File paths and sanitization changes
+- Error messages and timestamps
+- Rename operations

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arela/uploader",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "description": "CLI to upload files/directories to Arela",
   "bin": {
     "arela": "./src/index.js"

package/src/index.js

@@ -27,6 +27,52 @@ const sources = process.env.UPLOAD_SOURCES?.split('|')
 
 const supabase = createClient(supabaseUrl, supabaseKey);
 
+// Enhanced sanitization function specifically for file names
+const sanitizeFileName = (fileName) => {
+  // Get file extension
+  const ext = path.extname(fileName);
+  const nameWithoutExt = path.basename(fileName, ext);
+
+  // Replace problematic characters with safe alternatives
+  let sanitized = nameWithoutExt
+    // First, handle common character replacements
+    .replace(/[áàâäãåāăą]/gi, 'a')
+    .replace(/[éèêëēĕėę]/gi, 'e')
+    .replace(/[íìîïīĭį]/gi, 'i')
+    .replace(/[óòôöõōŏő]/gi, 'o')
+    .replace(/[úùûüūŭů]/gi, 'u')
+    .replace(/[ñň]/gi, 'n')
+    .replace(/[ç]/gi, 'c')
+    .replace(/[ý]/gi, 'y')
+    // Handle Korean characters more comprehensively
+    .replace(/[멕]/g, 'meok')
+    .replace(/[시]/g, 'si')
+    .replace(/[코]/g, 'ko')
+    .replace(/[용]/g, 'yong')
+    .replace(/[가-힣]/g, 'kr') // Replace any remaining Korean chars with 'kr'
+    // Normalize unicode and remove diacritics
+    .normalize('NFD')
+    .replace(/[\u0300-\u036f]/g, '') // Remove diacritics
+    // Replace problematic symbols
+    .replace(/[\\?%*:|"<>[\]~`^]/g, '-')
+    .replace(/[{}]/g, '-')
+    .replace(/[&]/g, 'and')
+    .replace(/[()]/g, '') // Remove parentheses
+    // Clean up spaces and dashes
+    .replace(/\s+/g, '-') // Replace spaces with dashes
+    .replace(/-+/g, '-') // Replace multiple dashes with single dash
+    .replace(/^-+|-+$/g, '') // Remove leading/trailing dashes
+    .replace(/^\.+/, '') // Remove leading dots
+    .replace(/[^\w.-]/g, ''); // Remove any remaining non-alphanumeric chars except dots and dashes
+
+  // Ensure the filename is not empty
+  if (!sanitized) {
+    sanitized = 'unnamed_file';
+  }
+
+  return sanitized + ext;
+};
+
 const sanitizePath = (path) =>
   path
     .replace(/[\\?%*:|"<>[\]~]/g, '-')
@@ -122,16 +168,66 @@ const writeLog = (message) => {
   }
 };
 
-const getProcessedPaths = () => {
-  const lines = fs.existsSync(logFilePath)
-    ? fs.readFileSync(logFilePath, 'utf-8').split('\n')
-    : [];
+// Modified to fetch from Supabase first, then fallback to local log
+const getProcessedPaths = async () => {
   const processed = new Set();
-  for (const line of lines) {
-    const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
-    if (match) {
-      const [, , path] = match;
-      processed.add(path.trim());
+
+  // Try to fetch from Supabase first
+  try {
+    const { data, error } = await supabase
+      .from('upload_logs')
+      .select('path')
+      .in('status', ['success', 'skipped']);
+
+    if (error) {
+      console.warn(
+        `⚠️ Could not fetch processed paths from Supabase: ${error.message}. Falling back to local log.`,
+      );
+      // Fallback to local log if Supabase fetch fails
+      const lines = fs.existsSync(logFilePath)
+        ? fs.readFileSync(logFilePath, 'utf-8').split('\\n')
+        : [];
+      for (const line of lines) {
+        const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
+        if (match) {
+          const [, , path] = match;
+          processed.add(path.trim());
+        }
+      }
+    } else if (data) {
+      data.forEach((log) => {
+        if (log.path) {
+          processed.add(log.path.trim());
+        }
+      });
+      // Also read from local log to ensure any paths logged before this change or during a Supabase outage are included
+      const lines = fs.existsSync(logFilePath)
+        ? fs.readFileSync(logFilePath, 'utf-8').split('\\n')
+        : [];
+      for (const line of lines) {
+        const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
+        if (match) {
+          const [, , pathValue] = match;
+          if (pathValue) {
+            processed.add(pathValue.trim());
+          }
+        }
+      }
+    }
+  } catch (e) {
+    console.warn(
+      `⚠️ Error fetching from Supabase or reading local log: ${e.message}. Proceeding with an empty set of processed paths initially.`,
+    );
+    // Ensure local log is still attempted if Supabase connection itself fails
+    const lines = fs.existsSync(logFilePath)
+      ? fs.readFileSync(logFilePath, 'utf-8').split('\\n')
+      : [];
+    for (const line of lines) {
+      const match = line.match(/(SUCCESS|SKIPPED): .*? -> (.+)/);
+      if (match) {
+        const [, , path] = match;
+        processed.add(path.trim());
+      }
     }
   }
   return processed;
@@ -139,23 +235,68 @@ const getProcessedPaths = () => {
 
 const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
-const uploadWithRetry = async (uploadFn, maxRetries = 3, delayMs = 1000) => {
+const uploadWithRetry = async (uploadFn, maxRetries = 5, delayMs = 2000) => {
   let attempt = 0;
+  let lastError;
+
   while (attempt < maxRetries) {
-    const result = await uploadFn();
-    if (!result.error) return result;
-    attempt++;
-    if (attempt < maxRetries) await delay(delayMs);
+    try {
+      const result = await uploadFn();
+      if (!result.error) return result;
+      lastError = result.error;
+      attempt++;
+
+      // Exponential backoff with jitter
+      if (attempt < maxRetries) {
+        const backoffDelay =
+          delayMs * Math.pow(2, attempt - 1) + Math.random() * 1000;
+        console.log(
+          `Retry ${attempt}/${maxRetries} after ${Math.round(backoffDelay)}ms...`,
+        );
+        await delay(backoffDelay);
+      }
+    } catch (error) {
+      lastError = error;
+      attempt++;
+
+      if (attempt < maxRetries) {
+        const backoffDelay =
+          delayMs * Math.pow(2, attempt - 1) + Math.random() * 1000;
+        console.log(
+          `Retry ${attempt}/${maxRetries} after ${Math.round(backoffDelay)}ms due to exception...`,
+        );
+        await delay(backoffDelay);
+      }
+    }
   }
-  return { error: new Error('Max retries exceeded') };
+
+  return {
+    error: new Error(
+      `Max retries exceeded. Last error: ${lastError?.message || 'Unknown error'}`,
+    ),
+  };
 };
 
 program
   .name('supabase-uploader')
   .description('CLI to upload folders from a base path to Supabase Storage')
-  .version(version)
+  .option('-v, --version', 'output the version number')
   .option('-p, --prefix <prefix>', 'Prefix path in bucket', '')
+  .option(
+    '-r, --rename-files',
+    'Rename files with problematic characters before uploading',
+  )
+  .option(
+    '--dry-run',
+    'Show what files would be renamed without actually renaming them',
+  )
   .action(async (options) => {
+    // Handle version option
+    if (options.version) {
+      console.log(version);
+      process.exit(0);
+    }
+
     if (!basePath || !sources || sources.length === 0) {
       console.error(
         '⚠️ UPLOAD_BASE_PATH or UPLOAD_SOURCES not defined in environment variables.',
@@ -163,7 +304,7 @@ program
       process.exit(1);
     }
 
-    const processedPaths = getProcessedPaths();
+    const processedPaths = await getProcessedPaths();
     let globalSuccess = 0;
     let globalFailure = 0;
 
@@ -190,41 +331,92 @@ program
 
         for (const file of files) {
           progressBar.increment();
-          const content = fs.readFileSync(file);
+
+          let currentFile = file;
+
+          // Check if we need to rename the file
+          if (options.renameFiles) {
+            const originalName = path.basename(file);
+            const sanitizedName = sanitizeFileName(originalName);
+
+            if (originalName !== sanitizedName) {
+              const newFilePath = path.join(path.dirname(file), sanitizedName);
+
+              if (options.dryRun) {
+                console.log(`Would rename: ${originalName} → ${sanitizedName}`);
+                continue;
+              } else {
+                try {
+                  fs.renameSync(file, newFilePath);
+                  currentFile = newFilePath;
+                  console.log(`✅ Renamed: ${originalName} → ${sanitizedName}`);
+                  writeLog(`RENAMED: ${originalName} → ${sanitizedName}`);
+                  await sendLogToSupabase({
+                    file: originalName,
+                    uploadPath: sanitizedName,
+                    status: 'renamed',
+                    message: `Renamed from ${originalName}`,
+                  });
+                } catch (renameError) {
+                  console.error(
+                    `❌ Failed to rename ${originalName}: ${renameError.message}`,
+                  );
+                  writeLog(
+                    `RENAME_ERROR: ${originalName} | ${renameError.message}`,
+                  );
+                  continue;
+                }
+              }
+            }
+          }
+
+          const content = fs.readFileSync(currentFile);
           const relativePathRaw = path
-            .relative(basePath, file)
+            .relative(basePath, currentFile)
             .replace(/^[\\/]+/, '')
             .replace(/\\/g, '/');
+
+          // Always sanitize the filename for upload path
+          const pathParts = relativePathRaw.split('/');
+          const originalFileName = pathParts[pathParts.length - 1];
+          const sanitizedFileName = sanitizeFileName(originalFileName);
+          pathParts[pathParts.length - 1] = sanitizedFileName;
+          const sanitizedRelativePath = pathParts.join('/');
+
           const uploadPathRaw = options.prefix
-            ? path.posix.join(options.prefix, relativePathRaw)
-            : relativePathRaw;
+            ? path.posix.join(options.prefix, sanitizedRelativePath)
+            : sanitizedRelativePath;
           const uploadPath = sanitizePath(uploadPathRaw);
 
-          if (uploadPath !== uploadPathRaw) {
-            writeLog(`SANITIZED: ${uploadPathRaw} → ${uploadPath}`);
+          if (
+            uploadPath !== uploadPathRaw ||
+            originalFileName !== sanitizedFileName
+          ) {
+            writeLog(`SANITIZED: ${relativePathRaw} → ${uploadPath}`);
             await sendLogToSupabase({
-              file,
-              uploadPath: uploadPathRaw,
+              file: currentFile,
+              uploadPath: relativePathRaw,
               status: 'sanitized',
-              message: `Sanitized to ${uploadPath}`,
+              message: `Sanitized to ${uploadPath} (Arela Version: ${version})`,
             });
           }
 
           if (processedPaths.has(uploadPath)) {
-            ora().info(`⏭️  Already processed (log): ${file}`);
+            ora().info(`⏭️  Already processed (log): ${currentFile}`);
             continue;
           }
 
-          const contentType = mime.lookup(file) || 'application/octet-stream';
+          const contentType =
+            mime.lookup(currentFile) || 'application/octet-stream';
 
-          const spinner = ora(`Checking ${file}...`).start();
+          const spinner = ora(`Checking ${currentFile}...`).start();
           const exists = await fileExistsInBucket(uploadPath);
 
           if (exists) {
-            spinner.info(`⏭️  Skipped (already exists): ${file}`);
-            writeLog(`SKIPPED: ${file} -> ${uploadPath}`);
+            spinner.info(`⏭️  Skipped (already exists): ${currentFile}`);
+            writeLog(`SKIPPED: ${currentFile} -> ${uploadPath}`);
             await sendLogToSupabase({
-              file,
+              file: currentFile,
               uploadPath,
               status: 'skipped',
               message: 'Already exists in bucket',
@@ -235,12 +427,21 @@ program
           try {
             // await delay(5000); // TODO: Remove this delay before production
 
+            spinner.text = `Uploading ${currentFile}...`;
+
             const { error } = await uploadWithRetry(() =>
               supabase.storage.from(bucket).upload(uploadPath, content, {
                 upsert: true,
                 contentType,
                 metadata: {
-                  originalName: path.basename(file),
+                  originalName: path.basename(currentFile),
+                  sanitizedName: path.basename(uploadPath),
+                  clientPath: path.posix.join(
+                    basePath,
+                    folder,
+                    path.relative(sourcePath, currentFile).replace(/\\/g, '/'),
+                  ),
+                  arelaVersion: version,
                 },
               }),
             );
@@ -248,31 +449,39 @@ program
             if (error) {
               failureCount++;
               globalFailure++;
-              spinner.fail(
-                `❌ Failed to upload ${file}: ${JSON.stringify(error, null, 2)}`,
-              );
-              writeLog(`ERROR: ${file} -> ${uploadPath} | ${error.message}`);
+              const errorMsg = error.message || JSON.stringify(error);
+              spinner.fail(`❌ Failed to upload ${currentFile}: ${errorMsg}`);
+              writeLog(`ERROR: ${currentFile} -> ${uploadPath} | ${errorMsg}`);
               await sendLogToSupabase({
-                file,
+                file: currentFile,
                 uploadPath,
                 status: 'error',
-                message: error.message,
+                message: errorMsg,
               });
             } else {
               successCount++;
               globalSuccess++;
-              spinner.succeed(`✅ Uploaded ${file} -> ${uploadPath}`);
-              writeLog(`SUCCESS: ${file} -> ${uploadPath}`);
+              spinner.succeed(`✅ Uploaded ${currentFile} -> ${uploadPath}`);
+              writeLog(`SUCCESS: ${currentFile} -> ${uploadPath}`);
               await sendLogToSupabase({
-                file,
+                file: currentFile,
                 uploadPath,
                 status: 'success',
                 message: 'Uploaded successfully',
               });
             }
           } catch (err) {
-            spinner.fail(`❌ Error uploading ${file}: ${err.message}`);
-            writeLog(`❌ Error uploading ${file}: ${err.message}`);
+            failureCount++;
+            globalFailure++;
+            const errorMsg = err.message || JSON.stringify(err);
+            spinner.fail(`❌ Error uploading ${currentFile}: ${errorMsg}`);
+            writeLog(`ERROR: ${currentFile} -> ${uploadPath} | ${errorMsg}`);
+            await sendLogToSupabase({
+              file: currentFile,
+              uploadPath,
+              status: 'error',
+              message: errorMsg,
+            });
           }
         }