@arela/uploader 0.2.12 → 0.3.0

package/src/services/DatabaseService.js

@@ -25,6 +25,15 @@ export class DatabaseService {
     return await supabaseService.getClient();
   }
 
+  /**
+   * Get Supabase client (public wrapper for dependency injection)
+   * Used by specialized services like SmartWatchDatabaseService
+   * @returns {Promise<Object>} Supabase client
+   */
+  async getSupabaseClient() {
+    return this.#getSupabaseClient();
+  }
+
   /**
    * Execute database query with retry logic and exponential backoff
    * @private
@@ -51,6 +60,7 @@ export class DatabaseService {
           error.message?.includes('timeout') ||
           error.message?.includes('canceling statement') ||
           error.message?.includes('connection') ||
+          error.message?.includes('fetch failed') ||
           error.code === 'PGRST301'; // PostgREST timeout
 
         if (!isRetriableError || attempt === maxRetries) {
@@ -70,6 +80,18 @@ export class DatabaseService {
     throw lastError;
   }
 
+  /**
+   * Execute database query with retry logic (public wrapper for dependency injection)
+   * Used by specialized services like SmartWatchDatabaseService
+   * @param {Function} queryFn - Query function to execute
+   * @param {string} operation - Description of the operation for logging
+   * @param {number} maxRetries - Maximum number of retry attempts (default: 3)
+   * @returns {Promise<Object>} Query result
+   */
+  async queryWithRetry(queryFn, operation, maxRetries = 3) {
+    return this.#queryWithRetry(queryFn, operation, maxRetries);
+  }
+
   /**
    * Insert file stats with document detection into uploader table
    * @param {Array} files - Array of file objects
@@ -111,6 +133,7 @@ export class DatabaseService {
       const filename = file.originalName || path.basename(file.path);
 
       const record = {
+        name: filename,
         document_type: null,
         size: stats.size,
         num_pedimento: null,
@@ -121,11 +144,14 @@ export class DatabaseService {
         rfc: null,
         message: null,
         file_extension: fileExtension,
-        is_like_simplificado:
-          fileExtension === 'pdf' && filename.toLowerCase().includes('simp'),
+        is_like_simplificado: filename.toLowerCase().includes('simp'),
         year: null,
         created_at: new Date().toISOString(),
+        updated_at: new Date().toISOString(),
         modified_at: stats.mtime.toISOString(),
+        // Queue/Processing columns (for arela-api)
+        processing_status: 'PENDING',
+        upload_attempts: 0,
       };
 
       // Try to detect document type for supported files
@@ -180,18 +206,153 @@ export class DatabaseService {
       `Inserting ${records.length} new records into uploader table...`,
     );
 
+    // Use upsert to handle duplicates gracefully
+    // This will insert new records or update existing ones (by original_path)
     const { data, error } = await supabase
       .from('uploader')
-      .insert(records)
+      .upsert(records, { onConflict: 'original_path' })
       .select();
 
     if (error) {
       throw new Error(`Failed to insert stats records: ${error.message}`);
     }
 
+    // Propagate arela_path to related files in same folder
+    if (data && data.length > 0) {
+      await this.#propagateArelaPathToRelatedFiles(data, supabase);
+    }
+
     return data;
   }
 
+  /**
+   * Propagate arela_path from pedimento files to related files in the same directory
+   * When a pedimento_simplificado is detected, all files in its directory get the same arela_path
+   * Also checks database for existing files in the same directory that need the arela_path
+   * @private
+   * @param {Array} insertedRecords - Records that were just inserted
+   * @param {Object} supabase - Supabase client
+   * @returns {Promise<void>}
+   */
+  async #propagateArelaPathToRelatedFiles(insertedRecords, supabase) {
+    try {
+      // Group records by directory
+      const recordsByDir = {};
+
+      for (const record of insertedRecords) {
+        if (!record.original_path) continue;
+
+        const dirPath = path.dirname(record.original_path);
+        if (!recordsByDir[dirPath]) {
+          recordsByDir[dirPath] = {
+            pedimentos: [],
+            allFiles: [],
+          };
+        }
+
+        recordsByDir[dirPath].allFiles.push(record);
+
+        // Identify pedimento files
+        if (
+          record.document_type === 'pedimento_simplificado' &&
+          record.arela_path
+        ) {
+          recordsByDir[dirPath].pedimentos.push(record);
+        }
+      }
+
+      // For each directory with a pedimento, propagate its arela_path to related files
+      for (const [dirPath, dirData] of Object.entries(recordsByDir)) {
+        if (dirData.pedimentos.length === 0) continue;
+
+        // Use the first pedimento's arela_path (should be only one per directory typically)
+        const pedimentoRecord = dirData.pedimentos[0];
+        const arelaPath = pedimentoRecord.arela_path;
+
+        logger.info(
+          `📁 Propagating arela_path from pedimento to files in ${path.basename(dirPath)}/`,
+        );
+
+        // Step 1: Update newly inserted files in this directory
+        const fileIds = dirData.allFiles
+          .filter(
+            (f) =>
+              f.id &&
+              f.arela_path !== arelaPath &&
+              f.document_type !== 'pedimento_simplificado',
+          )
+          .map((f) => f.id);
+
+        if (fileIds.length > 0) {
+          const { error: updateError } = await supabase
+            .from('uploader')
+            .update({ arela_path: arelaPath })
+            .in('id', fileIds);
+
+          if (updateError) {
+            logger.warn(
+              `Could not propagate arela_path: ${updateError.message}`,
+            );
+          } else {
+            logger.info(
+              `✅ Updated ${fileIds.length} related files with arela_path: ${arelaPath}`,
+            );
+          }
+        }
+
+        // Step 2: Also find and update any existing files in the same directory that don't have arela_path
+        // This handles the case where files were detected earlier but the pedimento is detected now
+        try {
+          const dirPattern = dirPath.replace(/\\/g, '/'); // Normalize for SQL LIKE
+          const { data: existingFiles, error: fetchError } = await supabase
+            .from('uploader')
+            .select('id, original_path, document_type')
+            .like('original_path', `${dirPattern}/%`)
+            .is('arela_path', null)
+            .limit(1000); // Reasonable limit to avoid huge queries
+
+          if (fetchError) {
+            logger.warn(
+              `Could not fetch existing files in ${path.basename(dirPath)}: ${fetchError.message}`,
+            );
+          } else if (existingFiles && existingFiles.length > 0) {
+            const existingFileIds = existingFiles
+              .filter(
+                (f) => f.id && f.document_type !== 'pedimento_simplificado',
+              )
+              .map((f) => f.id);
+
+            if (existingFileIds.length > 0) {
+              const { error: existingError } = await supabase
+                .from('uploader')
+                .update({ arela_path: arelaPath })
+                .in('id', existingFileIds);
+
+              if (existingError) {
+                logger.warn(
+                  `Could not update existing files with arela_path: ${existingError.message}`,
+                );
+              } else {
+                logger.info(
+                  `✅ Updated ${existingFileIds.length} existing files in directory with arela_path: ${arelaPath}`,
+                );
+              }
+            }
+          }
+        } catch (existingFilesError) {
+          logger.warn(
+            `Error checking for existing files in ${path.basename(dirPath)}: ${existingFilesError.message}`,
+          );
+        }
+      }
+    } catch (error) {
+      logger.warn(
+        `Error propagating arela_path to related files: ${error.message}`,
+      );
+      // Don't throw - this is a non-critical operation
+    }
+  }
+
   /**
    * Insert file stats only (no detection) into uploader table
    * @param {Array} files - Array of file objects
@@ -201,11 +362,29 @@ export class DatabaseService {
   async insertStatsOnlyToUploaderTable(files, options) {
     const supabase = await this.#getSupabaseClient();
     const batchSize = 1000;
+    const quietMode = options?.quietMode || false;
     const allRecords = [];
 
-    logger.info('Collecting filesystem stats...');
+    if (!quietMode) {
+      logger.info('Collecting filesystem stats...');
+    }
+
+    // Filter out system files and hidden files (macOS, Windows, Python, editors)
+    const systemFilePattern =
+      /^\.|__pycache__|\.pyc|\.swp|\.swo|Thumbs\.db|desktop\.ini|DS_Store|\$RECYCLE\.BIN|System Volume Information|~\$|\.tmp/i;
+
     for (const file of files) {
       try {
+        const fileName = file.originalName || path.basename(file.path);
+
+        // Skip system and hidden files
+        if (systemFilePattern.test(fileName)) {
+          if (!quietMode) {
+            logger.debug(`Skipping system file: ${fileName}`);
+          }
+          continue;
+        }
+
         const stats = file.stats || fs.statSync(file.path);
         const originalPath = options.clientPath || file.path;
         const fileExtension = path
@@ -214,6 +393,7 @@ export class DatabaseService {
           .replace('.', '');
 
         const record = {
+          name: file.originalName || path.basename(file.path),
           document_type: null,
           size: stats.size,
           num_pedimento: null,
@@ -225,6 +405,7 @@ export class DatabaseService {
           message: null,
           file_extension: fileExtension,
           created_at: new Date().toISOString(),
+          updated_at: new Date().toISOString(),
           modified_at: stats.mtime.toISOString(),
           is_like_simplificado:
             fileExtension === 'pdf' &&
@@ -232,6 +413,9 @@ export class DatabaseService {
               .toLowerCase()
               .includes('simp'),
           year: null,
+          // Queue/Processing columns (for arela-api)
+          processing_status: 'PENDING',
+          upload_attempts: 0,
         };
 
         allRecords.push(record);
@@ -241,13 +425,17 @@ export class DatabaseService {
     }
 
     if (allRecords.length === 0) {
-      logger.info('No file stats to insert');
+      if (!quietMode) {
+        logger.info('No file stats to insert');
+      }
       return { totalInserted: 0, totalSkipped: 0, totalProcessed: 0 };
     }
 
-    logger.info(
-      `Processing ${allRecords.length} file stats in batches of ${batchSize}...`,
-    );
+    if (!quietMode) {
+      logger.info(
+        `Processing ${allRecords.length} file stats in batches of ${batchSize}...`,
+      );
+    }
 
     let totalInserted = 0;
     let totalUpdated = 0;
@@ -280,10 +468,11 @@ export class DatabaseService {
           existingPaths.has(r.original_path),
         );
 
-        // Only log every 10th batch to reduce noise
+        // Only log every 10th batch to reduce noise (skip in quiet mode)
         if (
-          (Math.floor(i / batchSize) + 1) % 10 === 0 ||
-          Math.floor(i / batchSize) + 1 === 1
+          !quietMode &&
+          ((Math.floor(i / batchSize) + 1) % 10 === 0 ||
+            Math.floor(i / batchSize) + 1 === 1)
         ) {
           logger.info(
             `Batch ${Math.floor(i / batchSize) + 1}: ${newRecords.length} new, ${updateRecords.length} updates`,
@@ -311,6 +500,7 @@ export class DatabaseService {
             const { error: updateError } = await supabase
               .from('uploader')
               .update({
+                name: record.filename,
                 size: record.size,
                 modified_at: record.modified_at,
                 filename: record.filename,
@@ -324,8 +514,8 @@ export class DatabaseService {
             }
           }
           totalUpdated += batchUpdated;
-          // Reduce logging noise - only log when there are updates
-          if (batchUpdated > 0) {
+          // Reduce logging noise - only log when there are updates (skip in quiet mode)
+          if (!quietMode && batchUpdated > 0) {
             logger.info(`Updated ${batchUpdated} existing records`);
           }
         }
@@ -336,9 +526,11 @@ export class DatabaseService {
       }
     }
 
-    logger.success(
-      `Phase 1 Summary: ${totalInserted} new records inserted, ${totalUpdated} existing records updated`,
-    );
+    if (!quietMode) {
+      logger.success(
+        `Phase 1 Summary: ${totalInserted} new records inserted, ${totalUpdated} existing records updated`,
+      );
+    }
 
     return {
       totalInserted,
@@ -650,16 +842,18 @@ export class DatabaseService {
 
           while (hasMoreRelatedFiles) {
             const { data: relatedFilesPage, error: relatedError } =
-              await supabase
-                .from('uploader')
-                .select('id, filename, original_path')
-                .like('original_path', `${basePath}%`)
-                .is('arela_path', null)
-                .neq('id', pedimento.id) // Exclude the pedimento itself
-                .range(
-                  relatedFilesFrom,
-                  relatedFilesFrom + relatedFilesPageSize - 1,
-                );
+              await this.#queryWithRetry(async () => {
+                return await supabase
+                  .from('uploader')
+                  .select('id, filename, original_path')
+                  .like('original_path', `${basePath}%`)
+                  .is('arela_path', null)
+                  .neq('id', pedimento.id) // Exclude the pedimento itself
+                  .range(
+                    relatedFilesFrom,
+                    relatedFilesFrom + relatedFilesPageSize - 1,
+                  );
+              }, `query related files for ${pedimento.filename} (page ${relatedFilesPageNumber})`);
 
             if (relatedError) {
               logger.error(
@@ -782,11 +976,13 @@ export class DatabaseService {
         'ℹ️  No pedimento_simplificado records with arela_path found',
       );
     } else {
+      // pageNumber represents the current page (starts at 1)
+      const totalPages = pageNumber;
       console.log(
-        `📋 Processed ${totalProcessed} pedimento records across ${pageNumber - 1} pages`,
+        `📋 Processed ${totalProcessed} pedimento records across ${totalPages} page${totalPages !== 1 ? 's' : ''}`,
       );
       logger.info(
-        `Processed ${totalProcessed} pedimento records across ${pageNumber - 1} pages`,
+        `Processed ${totalProcessed} pedimento records across ${totalPages} page${totalPages !== 1 ? 's' : ''}`,
       );
     }
 
@@ -923,200 +1119,129 @@ export class DatabaseService {
     const uniqueArelaPaths = [
       ...new Set(allPedimentoRecords.map((r) => r.arela_path)),
     ];
+    // pageNumber represents the current page (starts at 1)
+    const totalPages = pageNumber;
     console.log(
-      `📋 Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths for specified RFCs across ${pageNumber - 1} pages`,
+      `📋 Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths for specified RFCs across ${totalPages} page${totalPages !== 1 ? 's' : ''}`,
     );
     logger.info(
-      `Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths across ${pageNumber - 1} pages`,
+      `Found ${allPedimentoRecords.length} pedimento records with ${uniqueArelaPaths.length} unique arela_paths across ${totalPages} page${totalPages !== 1 ? 's' : ''}`,
     );
 
-    // Step 2: Get all files with these arela_paths that haven't been uploaded yet
-    let rfcRecords = [];
-    const chunkSize = 50;
-
-    for (let i = 0; i < uniqueArelaPaths.length; i += chunkSize) {
-      const pathChunk = uniqueArelaPaths.slice(i, i + chunkSize);
-
-      const { data: chunkFiles, error: chunkError } = await supabase
-        .from('uploader')
-        .select('arela_path')
-        .in('arela_path', pathChunk)
-        .neq('status', 'file-uploaded')
-        .not('arela_path', 'is', null);
-
-      if (chunkError) {
-        const errorMsg = `Error fetching files for arela_paths chunk: ${chunkError.message}`;
-        logger.error(errorMsg);
-        throw new Error(errorMsg);
-      }
-
-      if (chunkFiles && chunkFiles.length > 0) {
-        rfcRecords = rfcRecords.concat(chunkFiles);
-      }
-    }
-
-    if (!rfcRecords || rfcRecords.length === 0) {
-      if (totalRfcFiles && totalRfcFiles > 0) {
-        console.log(
-          `ℹ️  All ${totalRfcFiles} files for the specified RFCs are already uploaded (status: file-uploaded)`,
-        );
-        console.log('   No new files to upload.');
-        logger.info(
-          `All ${totalRfcFiles} files for specified RFCs already uploaded`,
-        );
-      } else {
-        console.log(
-          'ℹ️  No files found for the specified RFCs with arela_path',
-        );
-        console.log(
-          `   Make sure files for RFCs [${appConfig.upload.rfcs.join(', ')}] have been processed and have arela_path values`,
-        );
-        logger.info('No files found for specified RFCs with arela_path');
-      }
-      return { processedCount: 0, uploadedCount: 0, errorCount: 0 };
-    }
-
-    // Show filtering effect
-    const uploadableArelaPaths = [
-      ...new Set(rfcRecords.map((r) => r.arela_path)),
-    ];
-    const skipped = (totalRfcFiles || 0) - rfcRecords.length;
-    if (skipped > 0) {
-      console.log(
-        `📊 Found ${rfcRecords.length} files ready for upload (${skipped} already uploaded, skipped)`,
-      );
-    } else {
-      console.log(`📊 Found ${rfcRecords.length} files ready for upload`);
-    }
-    logger.info(
-      `Found ${rfcRecords.length} files ready for upload, ${skipped} skipped`,
-    );
-
-    // Step 3: Process files with streaming pagination to avoid memory overload
+    // Step 2: Process files with optimized single query per chunk
     let totalProcessed = 0;
     let totalUploaded = 0;
     let totalErrors = 0;
     let globalFileCount = 0;
     const arelaPathChunkSize = 50;
-    const queryBatchSize = 500; // Reduced batch size for better memory management
     const batchSize = parseInt(options.batchSize) || 10;
+    const filePageSize = 1000; // Supabase limit per request
 
     // Import performance configuration
     const { performance: perfConfig } = appConfig;
     const maxConcurrency = perfConfig?.maxApiConnections || 3;
 
-    console.log(
-      '📥 Processing files with streaming pagination (processing arela_paths in chunks to avoid URI limits and memory overload)...',
-    );
+    console.log('📥 Processing files in chunks to avoid URI limits...');
 
     // Process arela_paths in chunks and upload files as we fetch them
-    for (let i = 0; i < uploadableArelaPaths.length; i += arelaPathChunkSize) {
-      const arelaPathChunk = uploadableArelaPaths.slice(
-        i,
-        i + arelaPathChunkSize,
-      );
+    for (let i = 0; i < uniqueArelaPaths.length; i += arelaPathChunkSize) {
+      const arelaPathChunk = uniqueArelaPaths.slice(i, i + arelaPathChunkSize);
       const chunkNumber = Math.floor(i / arelaPathChunkSize) + 1;
       const totalChunks = Math.ceil(
-        uploadableArelaPaths.length / arelaPathChunkSize,
+        uniqueArelaPaths.length / arelaPathChunkSize,
       );
 
       console.log(
         `   Processing arela_path chunk ${chunkNumber}/${totalChunks} (${arelaPathChunk.length} paths)`,
       );
 
-      // For each chunk of arela_paths, use pagination to get related files and process them immediately
-      let hasMore = true;
-      let offset = 0;
-      let chunkFileCount = 0;
+      // Fetch all files for this chunk with pagination to handle >1000 records
+      let allChunkFiles = [];
+      let fileOffset = 0;
+      let hasMoreFiles = true;
+      let filePageNum = 1;
 
-      while (hasMore) {
+      while (hasMoreFiles) {
         const { data: batch, error: queryError } = await supabase
           .from('uploader')
           .select('id, original_path, arela_path, filename, rfc, document_type')
           .in('arela_path', arelaPathChunk)
-          .not('original_path', 'is', null)
           .neq('status', 'file-uploaded')
-          .range(offset, offset + queryBatchSize - 1)
-          .order('created_at');
+          .order('created_at')
+          .range(fileOffset, fileOffset + filePageSize - 1);
 
         if (queryError) {
-          const errorMsg = `Error fetching related files for chunk ${chunkNumber}: ${queryError.message}`;
+          const errorMsg = `Error fetching files for chunk ${chunkNumber} page ${filePageNum}: ${queryError.message}`;
           logger.error(errorMsg);
           throw new Error(errorMsg);
         }
 
         if (!batch || batch.length === 0) {
-          hasMore = false;
+          hasMoreFiles = false;
+          if (filePageNum === 1) {
+            // No files found at all for this chunk
+            console.log(
+              `   ℹ️  Chunk ${chunkNumber}/${totalChunks}: No files to upload`,
+            );
+          }
           break;
         }
 
-        chunkFileCount += batch.length;
-        globalFileCount += batch.length;
-
-        console.log(
-          `   📦 Processing batch within chunk ${chunkNumber}: ${batch.length} files (total processed so far: ${globalFileCount})`,
+        logger.debug(
+          `Chunk ${chunkNumber} page ${filePageNum}: fetched ${batch.length} files`,
         );
+        allChunkFiles = allChunkFiles.concat(batch);
 
-        // Track if any uploads occurred in this batch
-        let uploadsOccurred = false;
+        // Check if we need more pages
+        if (batch.length < filePageSize) {
+          hasMoreFiles = false;
+          logger.debug(
+            `Chunk ${chunkNumber}: Completed pagination with ${allChunkFiles.length} total files`,
+          );
+        } else {
+          fileOffset += filePageSize;
+          filePageNum++;
+        }
+      }
 
-        // Process this batch of files immediately using concurrent processing
-        // Split batch into upload batches
-        for (let j = 0; j < batch.length; j += batchSize) {
-          const uploadBatch = batch.slice(j, j + batchSize);
-          const batchNum = Math.floor(globalFileCount / batchSize) + 1;
+      if (allChunkFiles.length === 0) {
+        continue;
+      }
 
-          console.log(
-            `📦 Processing upload batch ${batchNum} (${uploadBatch.length} files)`,
-          );
+      const chunkFileCount = allChunkFiles.length;
+      globalFileCount += chunkFileCount;
 
-          // Process batch using concurrent processing similar to UploadCommand
-          const batchResults = await this.#processRfcBatch(
-            uploadBatch,
-            uploadService,
-            supabase,
-            options,
-            maxConcurrency,
-          );
+      console.log(
+        `   📦 Chunk ${chunkNumber}/${totalChunks}: Processing ${chunkFileCount} files`,
+      );
 
-          totalProcessed += batchResults.processed;
-          totalUploaded += batchResults.uploaded;
-          totalErrors += batchResults.errors;
+      // Process this batch of files immediately using concurrent processing
+      // Split batch into upload batches
+      for (let j = 0; j < allChunkFiles.length; j += batchSize) {
+        const uploadBatch = allChunkFiles.slice(j, j + batchSize);
+        const batchNum = Math.floor(j / batchSize) + 1;
+        const totalBatches = Math.ceil(allChunkFiles.length / batchSize);
 
-          // Track if uploads occurred (status changes from non-uploaded to uploaded)
-          if (batchResults.uploaded > 0) {
-            uploadsOccurred = true;
-          }
+        console.log(
+          `   📦 Processing upload batch ${batchNum}/${totalBatches} within chunk ${chunkNumber} (${uploadBatch.length} files)`,
+        );
 
-          console.log(
-            `📊 Upload batch complete - Total progress: ${totalUploaded} uploaded, ${totalErrors} errors`,
-          );
-        }
+        // Process batch using concurrent processing similar to UploadCommand
+        const batchResults = await this.#processRfcBatch(
+          uploadBatch,
+          uploadService,
+          supabase,
+          options,
+          maxConcurrency,
+        );
 
-        // Check if we need more data from this chunk
-        if (batch.length < queryBatchSize) {
-          hasMore = false;
-        } else {
-          // If uploads occurred, reset pagination to start from beginning
-          // since records that matched the query may no longer match after upload
-          if (uploadsOccurred) {
-            offset = 0;
-            console.log(
-              `   📄 Batch complete with uploads: ${batch.length} files processed, restarting pagination from beginning due to query condition changes...`,
-            );
-          } else {
-            offset += queryBatchSize;
-            console.log(
-              `   📄 Batch complete: ${batch.length} files processed, continuing to next page (offset: ${offset})...`,
-            );
-          }
-        }
+        totalProcessed += batchResults.processed;
+        totalUploaded += batchResults.uploaded;
+        totalErrors += batchResults.errors;
 
-        if (hasMore) {
-          console.log(
-            `   📄 Fetching more files for chunk ${chunkNumber}... (offset: ${offset})`,
-          );
-        }
+        console.log(
+          `   📊 Batch complete - Progress: ${totalUploaded} uploaded, ${totalErrors} errors`,
+        );
       }
 
       console.log(
@@ -1467,6 +1592,7 @@ export class DatabaseService {
           .update({
             status: 'file-uploaded',
             message: 'Successfully uploaded to Supabase',
+            processing_status: 'UPLOADED',
           })
           .eq('id', file.id);
 
@@ -1648,6 +1774,7 @@ export class DatabaseService {
                 .update({
                   status: 'file-uploaded',
                   message: 'Successfully uploaded to Arela API (batch)',
+                  processing_status: 'UPLOADED',
                 })
                 .in('id', successfulFileIds);
 
@@ -1787,6 +1914,432 @@ export class DatabaseService {
 
     return { processed, uploaded, errors };
   }
+
+  /**
+   * Insert upload session event into watch_uploads table
+   * @param {Object} uploadEvent - Upload event from LoggingService
+   * @param {string} sessionId - Session ID for tracking
+   * @returns {Promise<Object>} Inserted record
+   */
+  async insertUploadEvent(uploadEvent, sessionId) {
+    const supabase = await this.#getSupabaseClient();
+
+    const record = {
+      session_id: sessionId,
+      timestamp: uploadEvent.timestamp || new Date().toISOString(),
+      strategy: uploadEvent.strategy, // 'individual', 'batch', 'full-structure'
+      file_count: uploadEvent.fileCount || 0,
+      success_count: uploadEvent.successCount || 0,
+      failure_count: uploadEvent.failureCount || 0,
+      retry_count: uploadEvent.retryCount || 0,
+      duration_ms: uploadEvent.duration || 0,
+      status: uploadEvent.status || 'completed',
+      metadata: uploadEvent.metadata || null,
+    };
+
+    try {
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await supabase.from('watch_uploads').insert([record]).select();
+      }, `insert upload event for session ${sessionId}`);
+
+      if (error) {
+        logger.error(`Failed to insert upload event: ${error.message}`);
+        throw error;
+      }
+
+      return data[0];
+    } catch (error) {
+      logger.error(`Error inserting upload event: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Insert retry event into watch_events table
+   * @param {string} uploadEventId - ID of the parent upload event
+   * @param {string} sessionId - Session ID for tracking
+   * @param {Object} retryEvent - Retry event from LoggingService
+   * @returns {Promise<Object>} Inserted record
+   */
+  async insertRetryEvent(uploadEventId, sessionId, retryEvent) {
+    const supabase = await this.#getSupabaseClient();
+
+    const record = {
+      upload_event_id: uploadEventId,
+      session_id: sessionId,
+      timestamp: retryEvent.timestamp || new Date().toISOString(),
+      attempt_number: retryEvent.attemptNumber || 0,
+      error_message: retryEvent.error || null,
+      backoff_ms: retryEvent.backoffMs || 0,
+      type: 'retry',
+    };
+
+    try {
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await supabase.from('watch_events').insert([record]).select();
+      }, `insert retry event for upload ${uploadEventId}`);
+
+      if (error) {
+        logger.error(`Failed to insert retry event: ${error.message}`);
+        throw error;
+      }
+
+      return data[0];
+    } catch (error) {
+      logger.error(`Error inserting retry event: ${error.message}`);
+      throw error;
+    }
+  }
+
+  /**
+   * Get upload history for a session
+   * @param {string} sessionId - Session ID to query
+   * @param {Object} options - Query options (limit, offset, strategy filter)
+   * @returns {Promise<Array>} Array of upload events
+   */
+  async getSessionUploadHistory(sessionId, options = {}) {
+    const supabase = await this.#getSupabaseClient();
+    const limit = options.limit || 100;
+    const offset = options.offset || 0;
+
+    try {
+      let query = supabase
+        .from('watch_uploads')
+        .select('*')
+        .eq('session_id', sessionId)
+        .order('timestamp', { ascending: false })
+        .range(offset, offset + limit - 1);
+
+      // Filter by strategy if provided
+      if (options.strategy) {
+        query = query.eq('strategy', options.strategy);
+      }
+
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await query;
+      }, `fetch upload history for session ${sessionId}`);
+
+      if (error) {
+        logger.error(`Failed to fetch upload history: ${error.message}`);
+        throw error;
+      }
+
+      return data || [];
+    } catch (error) {
+      logger.error(`Error fetching upload history: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get retry history for an upload event
+   * @param {string} uploadEventId - Upload event ID to query
+   * @param {Object} options - Query options (limit, offset)
+   * @returns {Promise<Array>} Array of retry events
+   */
+  async getUploadRetryHistory(uploadEventId, options = {}) {
+    const supabase = await this.#getSupabaseClient();
+    const limit = options.limit || 100;
+    const offset = options.offset || 0;
+
+    try {
+      const { data, error } = await this.#queryWithRetry(async () => {
+        return await supabase
+          .from('watch_events')
+          .select('*')
+          .eq('upload_event_id', uploadEventId)
+          .eq('type', 'retry')
+          .order('timestamp', { ascending: true })
+          .range(offset, offset + limit - 1);
+      }, `fetch retry history for upload ${uploadEventId}`);
+
+      if (error) {
+        logger.error(`Failed to fetch retry history: ${error.message}`);
+        throw error;
+      }
+
+      return data || [];
+    } catch (error) {
+      logger.error(`Error fetching retry history: ${error.message}`);
+      return [];
+    }
+  }
+
+  /**
+   * Get session statistics
+   * @param {string} sessionId - Session ID to analyze
+   * @returns {Promise<Object>} Session statistics
+   */
+  async getSessionStatistics(sessionId) {
+    const supabase = await this.#getSupabaseClient();
+
+    try {
+      // Fetch all upload events for the session
+      const { data: uploads, error: uploadError } = await this.#queryWithRetry(
+        async () => {
+          return await supabase
+            .from('watch_uploads')
+            .select('*')
+            .eq('session_id', sessionId);
+        },
+        `fetch statistics for session ${sessionId}`,
+      );
+
+      if (uploadError) {
+        throw uploadError;
+      }
+
+      // Fetch all retry events for the session
+      const { data: retries, error: retryError } = await this.#queryWithRetry(
+        async () => {
+          return await supabase
+            .from('watch_events')
+            .select('*')
+            .eq('session_id', sessionId)
+            .eq('type', 'retry');
+        },
+        `fetch retry statistics for session ${sessionId}`,
+      );
+
+      if (retryError) {
+        throw retryError;
+      }
+
+      // Calculate statistics
+      const stats = {
+        sessionId,
+        totalUploadEvents: uploads?.length || 0,
+        totalRetryEvents: retries?.length || 0,
+        totalFileCount: 0,
+        totalSuccessCount: 0,
+        totalFailureCount: 0,
+        totalRetryCount: 0,
+        totalDuration: 0,
+        byStrategy: {
+          individual: {
+            uploadCount: 0,
+            totalFiles: 0,
+            totalSuccess: 0,
+            totalFailure: 0,
+            successRate: 0,
+            totalDuration: 0,
+          },
+          batch: {
+            uploadCount: 0,
+            totalFiles: 0,
+            totalSuccess: 0,
+            totalFailure: 0,
+            successRate: 0,
+            totalDuration: 0,
+          },
+          'full-structure': {
+            uploadCount: 0,
+            totalFiles: 0,
+            totalSuccess: 0,
+            totalFailure: 0,
+            successRate: 0,
+            totalDuration: 0,
+          },
+        },
+        retryStats: {
+          totalRetries: retries?.length || 0,
+          uniqueUploadsWithRetries: new Set(
+            retries?.map((r) => r.upload_event_id) || [],
+          ).size,
+          totalRetryDuration:
+            retries?.reduce((sum, r) => sum + (r.backoff_ms || 0), 0) || 0,
+        },
+      };
+
+      // Process upload events
+      if (uploads && uploads.length > 0) {
+        uploads.forEach((upload) => {
+          stats.totalFileCount += upload.file_count || 0;
+          stats.totalSuccessCount += upload.success_count || 0;
+          stats.totalFailureCount += upload.failure_count || 0;
+          stats.totalRetryCount += upload.retry_count || 0;
+          stats.totalDuration += upload.duration_ms || 0;
+
+          const strategyKey = upload.strategy || 'individual';
+          if (stats.byStrategy[strategyKey]) {
+            stats.byStrategy[strategyKey].uploadCount += 1;
+            stats.byStrategy[strategyKey].totalFiles += upload.file_count || 0;
+            stats.byStrategy[strategyKey].totalSuccess +=
+              upload.success_count || 0;
+            stats.byStrategy[strategyKey].totalFailure +=
+              upload.failure_count || 0;
+            stats.byStrategy[strategyKey].totalDuration +=
+              upload.duration_ms || 0;
+
+            // Calculate success rate
+            const totalFiles =
+              stats.byStrategy[strategyKey].totalSuccess +
+              stats.byStrategy[strategyKey].totalFailure;
+            if (totalFiles > 0) {
+              stats.byStrategy[strategyKey].successRate = (
+                (stats.byStrategy[strategyKey].totalSuccess / totalFiles) *
+                100
+              ).toFixed(2);
+            }
+          }
+        });
+      }
+
+      return stats;
+    } catch (error) {
+      logger.error(`Error calculating session statistics: ${error.message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Delete old session data (cleanup)
+   * @param {number} daysOld - Delete sessions older than this many days
+   * @returns {Promise<Object>} Deletion results
+   */
+  async cleanupOldSessions(daysOld = 30) {
+    const supabase = await this.#getSupabaseClient();
+
+    try {
+      const cutoffDate = new Date();
+      cutoffDate.setDate(cutoffDate.getDate() - daysOld);
+
+      // Get sessions to delete
+      const { data: sessionsToDelete, error: fetchError } =
+        await this.#queryWithRetry(async () => {
+          return await supabase
+            .from('watch_uploads')
+            .select('session_id')
+            .lt('timestamp', cutoffDate.toISOString())
+            .distinct();
+        }, `fetch sessions older than ${daysOld} days`);
+
+      if (fetchError) {
+        throw fetchError;
+      }
+
+      let deletedUploads = 0;
+      let deletedEvents = 0;
+
+      if (sessionsToDelete && sessionsToDelete.length > 0) {
+        const sessionIds = sessionsToDelete.map((s) => s.session_id);
+
+        // Delete events
+        const { count: eventCount, error: eventError } =
+          await this.#queryWithRetry(async () => {
+            return await supabase
+              .from('watch_events')
+              .delete()
+              .in('session_id', sessionIds);
+          }, `delete events for old sessions`);
+
+        if (!eventError) {
+          deletedEvents = eventCount || 0;
+        }
+
+        // Delete uploads
+        const { count: uploadCount, error: uploadError } =
+          await this.#queryWithRetry(async () => {
+            return await supabase
+              .from('watch_uploads')
+              .delete()
+              .in('session_id', sessionIds);
+          }, `delete old session uploads`);
+
+        if (!uploadError) {
+          deletedUploads = uploadCount || 0;
+        }
+      }
+
+      return {
+        deletedUploads,
+        deletedEvents,
+        sessionsDeleted: sessionsToDelete?.length || 0,
+      };
+    } catch (error) {
+      logger.error(`Error cleaning up old sessions: ${error.message}`);
+      return { deletedUploads: 0, deletedEvents: 0, sessionsDeleted: 0 };
+    }
+  }
+
+  /**
+   * Cleanup database connections and resources
+   * Called during graceful shutdown
+   * @returns {Promise<Object>} Cleanup results
+   */
+  async cleanup() {
+    try {
+      logger.info('DatabaseService: Starting cleanup...');
+
+      // Commit any pending transactions
+      const transactionResult = await this.commitPendingTransactions();
+
+      // Close database connections
+      const closeResult = await this.closeConnections();
+
+      logger.info('DatabaseService: Cleanup complete');
+
+      return {
+        success: true,
+        transactionsCommitted: transactionResult.count,
+        connectionsClosedResult: closeResult,
+      };
+    } catch (error) {
+      logger.error(`DatabaseService: Error during cleanup: ${error.message}`);
+      return {
+        success: false,
+        error: error.message,
+      };
+    }
+  }
+
+  /**
+   * Commit any pending transactions before shutdown
+   * @private
+   * @returns {Promise<Object>} Results
+   */
+  async commitPendingTransactions() {
+    try {
+      logger.debug('DatabaseService: Committing pending transactions...');
+
+      // Note: This is a placeholder for actual transaction handling
+      // In a real implementation, you would track active transactions
+      // and ensure they are properly committed before shutdown
+
+      logger.debug('DatabaseService: Pending transactions committed');
+      return { count: 0, success: true };
+    } catch (error) {
+      logger.error(
+        `DatabaseService: Error committing transactions: ${error.message}`,
+      );
+      return { count: 0, success: false, error: error.message };
+    }
+  }
+
+  /**
+   * Close all database connections
+   * @private
+   * @returns {Promise<Object>} Results
+   */
+  async closeConnections() {
+    try {
+      logger.debug('DatabaseService: Closing database connections...');
+
+      // Close Supabase connection if available
+      if (this.supabase) {
+        // Supabase client will handle connection cleanup automatically
+        logger.debug('DatabaseService: Supabase connection cleanup initiated');
+      }
+
+      logger.info('DatabaseService: All database connections closed');
+      return { success: true };
+    } catch (error) {
+      logger.error(
+        `DatabaseService: Error closing connections: ${error.message}`,
+      );
+      return { success: false, error: error.message };
+    }
+  }
 }
 
 // Export singleton instance