ragdoll-rails 0.1.9 → 0.1.11

data/app/jobs/ragdoll/bulk_document_processing_job.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class BulkDocumentProcessingJob < ApplicationJob
+    queue_as :default
+    
+    private
+    
+    def broadcast_status_update(session_id, data)
+      ActionCable.server.broadcast("bulk_upload_status_#{session_id}", data)
+    rescue => e
+      logger.error "Failed to broadcast status update: #{e.message}"
+    end
+    
+    def safe_log_operation(operation, details = {})
+      return unless defined?(RagdollLogging)
+      RagdollLogging.log_operation(operation, details)
+    rescue => e
+      logger.debug "Failed to log operation #{operation}: #{e.message}"
+    end
+    
+    def safe_log_error(operation, error, details = {})
+      return unless defined?(RagdollLogging)
+      RagdollLogging.log_error(operation, error, details)
+    rescue => e
+      logger.debug "Failed to log error #{operation}: #{e.message}"
+    end
+    
+    def safe_log_performance(operation, duration, details = {})
+      return unless defined?(RagdollLogging)
+      RagdollLogging.log_performance(operation, duration, details)
+    rescue => e
+      logger.debug "Failed to log performance #{operation}: #{e.message}"
+    end
+    
+    def perform(session_id, file_paths_data, force_duplicate = false)
+      start_time = Time.current
+      
+      # Initialize variables early to avoid nil errors in rescue block
+      total_files = file_paths_data&.size || 0
+      processed_count = 0
+      failed_files = []
+      
+      safe_log_operation("bulk_processing_start", {
+        session_id: session_id,
+        file_count: total_files,
+        force_duplicate: force_duplicate,
+        job_id: job_id
+      })
+      
+      logger.info "🚀 Starting bulk document processing job for session #{session_id}"
+      logger.info "📁 Processing #{total_files} files"
+      
+      # Early return if no files to process
+      if file_paths_data.nil? || file_paths_data.empty?
+        logger.warn "⚠️ No files provided for processing in session #{session_id}"
+        broadcast_status_update(session_id, {
+          type: 'upload_error',
+          error: 'No files provided for processing',
+          status: 'failed'
+        })
+        return
+      end
+      
+      # Broadcast upload start
+      broadcast_status_update(session_id, {
+        type: 'upload_start',
+        total_files: total_files,
+        status: 'processing',
+        started_at: Time.current.iso8601
+      })
+      
+      batch_size = 10  # Process 10 files at a time for async jobs
+      
+      file_paths_data.each_slice(batch_size).with_index do |file_batch, batch_index|
+        logger.info "📦 Processing batch #{batch_index + 1} of #{(total_files.to_f / batch_size).ceil}"
+        
+        file_batch.each do |file_data|
+          file_start_time = Time.current
+          
+          begin
+            temp_path = file_data[:temp_path]
+            original_filename = file_data[:original_filename]
+            
+            safe_log_operation("file_processing_start", {
+              session_id: session_id,
+              filename: original_filename,
+              temp_path: temp_path,
+              file_exists: File.exist?(temp_path),
+              file_size: File.exist?(temp_path) ? File.size(temp_path) : 0
+            })
+            
+            unless File.exist?(temp_path)
+              error_msg = "Temporary file not found: #{temp_path}"
+              safe_log_error("file_processing", StandardError.new(error_msg), {
+                session_id: session_id,
+                filename: original_filename,
+                temp_path: temp_path
+              })
+              next
+            end
+            
+            logger.info "🔄 Processing file: #{original_filename}"
+            
+            # Broadcast file start
+            progress_percentage = ((processed_count.to_f / total_files) * 100).round(1)
+            broadcast_status_update(session_id, {
+              type: 'file_start',
+              filename: original_filename,
+              processed: processed_count,
+              total: total_files,
+              percentage: progress_percentage,
+              status: 'processing',
+              batch_index: batch_index + 1,
+              total_batches: (total_files.to_f / batch_size).ceil
+            })
+            
+            # Process the document
+            ragdoll_start_time = Time.current
+            result = ::Ragdoll.add_document(path: temp_path, force: force_duplicate)
+            ragdoll_duration = Time.current - ragdoll_start_time
+            
+            safe_log_performance("ragdoll_add_document", ragdoll_duration, {
+              session_id: session_id,
+              filename: original_filename,
+              result_success: result && result[:success],
+              force_duplicate: force_duplicate
+            })
+            
+            if result && result[:success]
+              processed_count += 1
+              file_duration = Time.current - file_start_time
+              
+              safe_log_operation("file_processing_success", {
+                session_id: session_id,
+                filename: original_filename,
+                document_id: result[:document_id],
+                processing_duration: file_duration.round(3),
+                processed_count: processed_count,
+                total_files: total_files
+              })
+              
+              logger.info "✅ Successfully processed: #{original_filename}"
+              
+              # Broadcast success
+              broadcast_status_update(session_id, {
+                type: 'file_complete',
+                filename: original_filename,
+                processed: processed_count,
+                total: total_files,
+                percentage: ((processed_count.to_f / total_files) * 100).round(1),
+                status: 'completed',
+                document_id: result[:document_id],
+                processing_time: file_duration.round(3)
+              })
+            else
+              failed_files << original_filename
+              error_message = result ? result[:error] : 'Unknown error'
+              file_duration = Time.current - file_start_time
+              
+              safe_log_error("file_processing", StandardError.new(error_message), {
+                session_id: session_id,
+                filename: original_filename,
+                processing_duration: file_duration.round(3),
+                ragdoll_result: result,
+                temp_path: temp_path,
+                file_size: File.size(temp_path)
+              })
+              
+              logger.error "❌ Failed to process: #{original_filename} - #{error_message}"
+              
+              # Broadcast error
+              broadcast_status_update(session_id, {
+                type: 'file_error',
+                filename: original_filename,
+                processed: processed_count,
+                total: total_files,
+                percentage: ((processed_count.to_f / total_files) * 100).round(1),
+                status: 'failed',
+                error: error_message,
+                processing_time: file_duration.round(3)
+              })
+            end
+            
+            # Clean up temp file
+            File.delete(temp_path) if File.exist?(temp_path)
+            
+          rescue => e
+            failed_files << (file_data[:original_filename] || 'unknown file')
+            file_duration = Time.current - file_start_time
+            
+            safe_log_error("file_processing_exception", e, {
+              session_id: session_id,
+              filename: file_data[:original_filename],
+              temp_path: file_data[:temp_path],
+              processing_duration: file_duration.round(3),
+              file_data: file_data,
+              processed_count: processed_count,
+              total_files: total_files
+            })
+            
+            logger.error "💥 Exception processing file #{file_data[:original_filename]}: #{e.message}"
+            logger.error e.backtrace.join("\n")
+            
+            # Broadcast error
+            ActionCable.server.broadcast("ragdoll_file_processing_#{session_id}", {
+              type: 'file_error',
+              filename: file_data[:original_filename],
+              processed: processed_count,
+              total: total_files,
+              percentage: ((processed_count.to_f / total_files) * 100).round(1),
+              status: 'failed',
+              error: e.message
+            })
+          end
+        end
+        
+        # Force garbage collection after each batch
+        GC.start
+        
+        # Small delay between batches to prevent overwhelming the system
+        sleep(0.1)
+      end
+      
+      # Broadcast final completion
+      total_duration = Time.current - start_time
+      final_percentage = 100.0
+      broadcast_status_update(session_id, {
+        type: 'upload_complete',
+        processed: processed_count,
+        total: total_files,
+        failed: failed_files.size,
+        failed_files: failed_files,
+        percentage: final_percentage,
+        status: 'completed',
+        total_duration: total_duration.round(3),
+        completed_at: Time.current.iso8601
+      })
+      
+      safe_log_operation("bulk_processing_complete", {
+        session_id: session_id,
+        total_files: total_files,
+        processed_count: processed_count,
+        failed_count: failed_files.size,
+        failed_files: failed_files,
+        total_duration: total_duration.round(3),
+        avg_file_duration: total_files > 0 ? (total_duration / total_files).round(3) : 0
+      })
+      
+      logger.info "🎉 Bulk processing completed for session #{session_id}"
+      logger.info "📊 Results: #{processed_count}/#{total_files} successful, #{failed_files.size} failed"
+      
+    rescue => e
+      total_duration = Time.current - start_time
+      
+      safe_log_error("bulk_processing_job_failure", e, {
+        session_id: session_id,
+        total_files: total_files,
+        processed_count: processed_count,
+        failed_count: failed_files.size,
+        total_duration: total_duration.round(3),
+        job_id: job_id
+      })
+      
+      logger.error "💀 Bulk processing job failed for session #{session_id}: #{e.message}"
+      logger.error e.backtrace.join("\n")
+      
+      # Broadcast job failure
+      broadcast_status_update(session_id, {
+        type: 'upload_error',
+        error: e.message,
+        status: 'failed',
+        processed: processed_count,
+        total: total_files,
+        failed_at: Time.current.iso8601,
+        total_duration: total_duration.round(3)
+      })
+    end
+  end
+end

data/app/jobs/ragdoll/process_file_job.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class ProcessFileJob < ApplicationJob
+    queue_as :default
+
+    def perform(file_id, session_id, filename, temp_path)
+      ::Rails.logger.info "🚀 Ragdoll::ProcessFileJob starting: file_id=#{file_id}, session_id=#{session_id}, filename=#{filename}"
+      ::Rails.logger.info "📁 Temp file path: #{temp_path}"
+      ::Rails.logger.info "📊 Temp file exists: #{File.exist?(temp_path)}"
+      ::Rails.logger.info "📏 Temp file size: #{File.exist?(temp_path) ? File.size(temp_path) : 'N/A'} bytes"
+      
+      begin
+        # Verify temp file exists before processing
+        unless File.exist?(temp_path)
+          raise "Temporary file not found: #{temp_path}"
+        end
+        
+        # Broadcast start
+        broadcast_data = {
+          file_id: file_id,
+          filename: filename,
+          status: 'started',
+          progress: 0,
+          message: 'Starting file processing...'
+        }
+        
+        ::Rails.logger.info "📡 Broadcasting start: #{broadcast_data}"
+        begin
+          ActionCable.server.broadcast("ragdoll_file_processing_#{session_id}", broadcast_data)
+          ::Rails.logger.info "✅ ActionCable broadcast sent successfully"
+          
+          # Track job start in monitoring system
+          track_job_progress(session_id, file_id, filename, 0, 'started')
+        rescue => e
+          ::Rails.logger.error "❌ ActionCable broadcast failed: #{e.message}"
+          ::Rails.logger.error e.backtrace.first(3)
+        end
+
+        # Simulate progress updates during processing
+        broadcast_progress(session_id, file_id, filename, 25, 'Reading file...')
+        track_job_progress(session_id, file_id, filename, 25, 'processing')
+        
+        # Use Ragdoll to add document
+        result = ::Ragdoll.add_document(path: temp_path)
+        
+        broadcast_progress(session_id, file_id, filename, 75, 'Generating embeddings...')
+        track_job_progress(session_id, file_id, filename, 75, 'processing')
+        
+        if result[:success] && result[:document_id]
+          document = ::Ragdoll::Document.find(result[:document_id])
+          
+          # Broadcast completion
+          completion_data = {
+            file_id: file_id,
+            filename: filename,
+            status: 'completed',
+            progress: 100,
+            message: 'Processing completed successfully',
+            document_id: document.id
+          }
+          
+          ::Rails.logger.info "🎉 Broadcasting completion: #{completion_data}"
+          begin
+            ActionCable.server.broadcast("ragdoll_file_processing_#{session_id}", completion_data)
+            ::Rails.logger.info "✅ Completion broadcast sent successfully"
+            
+            # Mark job as completed in monitoring system
+            mark_job_completed(session_id, file_id)
+          rescue => e
+            ::Rails.logger.error "❌ Completion broadcast failed: #{e.message}"
+          end
+        else
+          raise "Processing failed: #{result[:error] || 'Unknown error'}"
+        end
+        
+      rescue => e
+        ::Rails.logger.error "💥 Ragdoll::ProcessFileJob error: #{e.message}"
+        ::Rails.logger.error e.backtrace.first(5)
+        
+        # Broadcast error
+        error_data = {
+          file_id: file_id,
+          filename: filename,
+          status: 'error',
+          progress: 0,
+          message: "Error: #{e.message}"
+        }
+        
+        ::Rails.logger.info "📡 Broadcasting error: #{error_data}"
+        begin
+          ActionCable.server.broadcast("ragdoll_file_processing_#{session_id}", error_data)
+          ::Rails.logger.info "✅ Error broadcast sent successfully"
+          
+          # Mark job as failed in monitoring system
+          mark_job_failed(session_id, file_id)
+        rescue => e
+          ::Rails.logger.error "❌ Error broadcast failed: #{e.message}"
+        end
+        
+        # Re-raise the error to mark job as failed
+        raise e
+      ensure
+        # ALWAYS clean up temp file in ensure block
+        if temp_path && File.exist?(temp_path)
+          ::Rails.logger.info "🧹 Cleaning up temp file: #{temp_path}"
+          begin
+            File.delete(temp_path)
+            ::Rails.logger.info "✅ Temp file deleted successfully"
+          rescue => e
+            ::Rails.logger.error "❌ Failed to delete temp file: #{e.message}"
+          end
+        else
+          ::Rails.logger.info "📝 Temp file already cleaned up or doesn't exist: #{temp_path}"
+        end
+      end
+    end
+
+    private
+
+    def broadcast_progress(session_id, file_id, filename, progress, message)
+      broadcast_data = {
+        file_id: file_id,
+        filename: filename,
+        status: 'processing',
+        progress: progress,
+        message: message
+      }
+      
+      ::Rails.logger.info "📡 Broadcasting progress: #{broadcast_data}"
+      begin
+        ActionCable.server.broadcast("ragdoll_file_processing_#{session_id}", broadcast_data)
+        ::Rails.logger.info "✅ Progress broadcast sent successfully"
+      rescue => e
+        ::Rails.logger.error "❌ Progress broadcast failed: #{e.message}"
+      end
+      
+      # Small delay to simulate processing time
+      sleep(0.5)
+    end
+    
+    def track_job_progress(session_id, file_id, filename, progress, status)
+      if defined?(JobFailureMonitorService)
+        JobFailureMonitorService.track_job_progress(session_id, file_id, filename, progress, status)
+      end
+    rescue => e
+      ::Rails.logger.error "❌ Failed to track job progress: #{e.message}"
+    end
+    
+    def mark_job_completed(session_id, file_id)
+      if defined?(JobFailureMonitorService)
+        JobFailureMonitorService.mark_job_completed(session_id, file_id)
+      end
+    rescue => e
+      ::Rails.logger.error "❌ Failed to mark job as completed: #{e.message}"
+    end
+    
+    def mark_job_failed(session_id, file_id)
+      if defined?(JobFailureMonitorService)
+        JobFailureMonitorService.mark_job_failed(session_id, file_id)
+      end
+    rescue => e
+      ::Rails.logger.error "❌ Failed to mark job as failed: #{e.message}"
+    end
+  end
+end

data/app/services/ragdoll/worker_health_service.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class WorkerHealthService
+    class << self
+      def check_worker_health
+        {
+          status: 'healthy',
+          workers: worker_status,
+          queues: queue_status,
+          timestamp: Time.current
+        }
+      rescue => e
+        {
+          status: 'error',
+          error: e.message,
+          timestamp: Time.current
+        }
+      end
+
+      def needs_restart?
+        # Check if there are stuck jobs or workers
+        stuck_jobs_count > 5 || !workers_running?
+      rescue
+        false
+      end
+
+      def process_stuck_jobs!(limit = 10)
+        processed = 0
+        
+        # Find jobs that have been processing for too long (e.g., > 1 hour)
+        if defined?(SolidQueue::Job)
+          stuck_jobs = SolidQueue::Job
+            .where(finished_at: nil)
+            .where('created_at < ?', 1.hour.ago)
+            .limit(limit)
+          
+          stuck_jobs.each do |job|
+            job.update(finished_at: Time.current)
+            processed += 1
+          end
+        end
+        
+        processed
+      rescue => e
+        ::Rails.logger.error "Failed to process stuck jobs: #{e.message}"
+        0
+      end
+
+      def restart_workers!
+        # In development, we typically don't restart workers
+        # This would be implemented differently in production
+        ::Rails.logger.info "Worker restart requested (no-op in development)"
+        true
+      rescue => e
+        ::Rails.logger.error "Failed to restart workers: #{e.message}"
+        false
+      end
+
+      private
+
+      def worker_status
+        if defined?(SolidQueue::Worker)
+          {
+            count: SolidQueue::Worker.count,
+            active: SolidQueue::Worker.where('last_heartbeat_at > ?', 5.minutes.ago).count
+          }
+        else
+          { count: 0, active: 0 }
+        end
+      rescue
+        { count: 0, active: 0, error: 'Unable to check worker status' }
+      end
+
+      def queue_status
+        if defined?(SolidQueue::Job)
+          {
+            pending: SolidQueue::Job.where(finished_at: nil).count,
+            completed: SolidQueue::Job.where.not(finished_at: nil).count,
+            failed: defined?(SolidQueue::FailedExecution) ? SolidQueue::FailedExecution.count : 0
+          }
+        else
+          { pending: 0, completed: 0, failed: 0 }
+        end
+      rescue
+        { pending: 0, completed: 0, failed: 0, error: 'Unable to check queue status' }
+      end
+
+      def stuck_jobs_count
+        return 0 unless defined?(SolidQueue::Job)
+        
+        SolidQueue::Job
+          .where(finished_at: nil)
+          .where('created_at < ?', 1.hour.ago)
+          .count
+      rescue
+        0
+      end
+
+      def workers_running?
+        return false unless defined?(SolidQueue::Worker)
+        
+        SolidQueue::Worker
+          .where('last_heartbeat_at > ?', 5.minutes.ago)
+          .exists?
+      rescue
+        true # Assume workers are running if we can't check
+      end
+    end
+  end
+end