fractor 0.1.6 → 0.1.7

data/lib/fractor/continuous_server.rb

@@ -6,22 +6,23 @@ module Fractor
   # High-level wrapper for running Fractor in continuous mode.
   # Handles threading, signal handling, and results processing automatically.
   class ContinuousServer
-    attr_reader :supervisor, :work_queue
+    attr_reader :supervisor, :work_queue, :logger
 
     # Initialize a continuous server
     # @param worker_pools [Array<Hash>] Worker pool configurations
     # @param work_queue [WorkQueue, nil] Optional work queue to auto-register
     # @param log_file [String, nil] Optional log file path
-    def initialize(worker_pools:, work_queue: nil, log_file: nil)
+    # @param logger [Logger, nil] Optional logger instance for isolation (defaults to Fractor.logger)
+    def initialize(worker_pools:, work_queue: nil, log_file: nil, logger: nil)
       @worker_pools = worker_pools
       @work_queue = work_queue
       @log_file_path = log_file
       @log_file = nil
+      @logger = logger # Store instance-specific logger for isolation
       @result_callbacks = []
       @error_callbacks = []
       @supervisor = nil
       @supervisor_thread = nil
-      @results_thread = nil
       @running = false
     end
 
@@ -37,24 +38,32 @@ module Fractor
       @error_callbacks << block
     end
 
+    # Start the server (alias for run).
+    # Provides a consistent API with stop method.
+    #
+    # @see #run
+    def start
+      run
+    end
+
     # Start the server and block until shutdown
     # This method handles:
     # - Opening log file if specified
     # - Creating and starting supervisor
-    # - Starting results processing thread
-    # - Setting up signal handlers
+    # - Registering result callbacks with ResultAggregator
     # - Blocking until shutdown signal received
     def run
       setup_log_file
       setup_supervisor
+      register_result_callbacks
       start_supervisor_thread
-      start_results_thread
 
       log_message("Continuous server started")
       log_message("Press Ctrl+C to stop")
 
       begin
-        # Block until shutdown
+        # Event-driven: simply join the supervisor thread
+        # It will exit when @running = false and shutdown is complete
         @supervisor_thread&.join
       rescue Interrupt
         log_message("Interrupt received, shutting down...")
@@ -72,12 +81,11 @@ module Fractor
 
       @supervisor&.stop
 
-      # Wait for threads to finish
-      [@supervisor_thread, @results_thread].compact.each do |thread|
-        thread.join(2) if thread.alive?
-      end
-
-      log_message("Continuous server stopped")
+      # Ensure log file is closed
+      # This is important when stop() is called from outside the run() thread
+      # The run() method's ensure block will also call cleanup, but we ensure
+      # it here as well for immediate cleanup
+      cleanup
     end
 
     private
@@ -93,6 +101,7 @@ module Fractor
       @supervisor = Supervisor.new(
         worker_pools: @worker_pools,
         continuous_mode: true,
+        logger: @logger, # Pass instance-specific logger for isolation
       )
 
       # Auto-register work queue if provided
@@ -104,62 +113,43 @@ module Fractor
       end
     end
 
-    def start_supervisor_thread
-      @running = true
-      @supervisor_thread = Thread.new do
-        @supervisor.run
-      rescue StandardError => e
-        log_message("Supervisor error: #{e.message}")
-        log_message(e.backtrace.join("\n")) if ENV["FRACTOR_DEBUG"]
-      end
-
-      # Give supervisor time to start up
-      sleep(0.1)
-    end
-
-    def start_results_thread
-      @results_thread = Thread.new do
-        log_message("Results processing thread started")
-        process_results_loop
-      rescue StandardError => e
-        log_message("Results thread error: #{e.message}")
-        log_message(e.backtrace.join("\n")) if ENV["FRACTOR_DEBUG"]
-      end
-    end
-
-    def process_results_loop
-      while @running
-        sleep(0.05)
-
-        process_successful_results
-        process_error_results
+    def register_result_callbacks
+      # Register callbacks directly with ResultAggregator for event-driven processing
+      # This eliminates the need for a separate results polling thread
+      unless @result_callbacks.empty?
+        @supervisor.results.on_new_result do |result|
+          if result.success?
+            @result_callbacks.each do |callback|
+              callback.call(result)
+            rescue StandardError => e
+              log_message("Error in result callback: #{e.message}")
+            end
+          end
+        end
       end
-      log_message("Results processing thread stopped")
-    end
-
-    def process_successful_results
-      loop do
-        result = @supervisor.results.results.shift
-        break unless result
 
-        @result_callbacks.each do |callback|
-          callback.call(result)
-        rescue StandardError => e
-          log_message("Error in result callback: #{e.message}")
+      unless @error_callbacks.empty?
+        @supervisor.results.on_new_result do |result|
+          unless result.success?
+            @error_callbacks.each do |callback|
+              callback.call(result)
+            rescue StandardError => e
+              log_message("Error in error callback: #{e.message}")
+            end
+          end
         end
       end
     end
 
-    def process_error_results
-      loop do
-        error_result = @supervisor.results.errors.shift
-        break unless error_result
-
-        @error_callbacks.each do |callback|
-          callback.call(error_result)
-        rescue StandardError => e
-          log_message("Error in error callback: #{e.message}")
-        end
+    def start_supervisor_thread
+      @running = true
+      @supervisor_thread = Thread.new do
+        @supervisor.run
+      rescue StandardError => e
+        log_message("Supervisor error: #{e.message}")
+        # Use instance logger or fall back to global
+        instance_logger = @logger || Fractor.logger
+        instance_logger.debug(e.backtrace.join("\n")) if instance_logger&.debug?
       end
     end
 
@@ -178,8 +168,13 @@ module Fractor
       log_entry = "[#{timestamp}] #{message}"
 
       if @log_file && !@log_file.closed?
-        @log_file.puts(log_entry)
-        @log_file.flush
+        begin
+          @log_file.puts(log_entry)
+          @log_file.flush
+        rescue IOError
+          # File was closed in another thread, stop trying to write to it
+          @log_file = nil
+        end
       end
 
       puts log_entry

data/lib/fractor/error_formatter.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Fractor
+  # Formats error messages with rich context for debugging.
+  # Extracted from Supervisor to follow Single Responsibility Principle.
+  #
+  # @example Basic usage
+  #   formatter = ErrorFormatter.new
+  #   error_message = formatter.format(wrapped_ractor, error_result)
+  class ErrorFormatter
+    # Format error context with rich information for debugging.
+    #
+    # @param wrapped_ractor [WrappedRactor] The worker that encountered the error
+    # @param error_result [WorkResult] The error result
+    # @return [String] Formatted error message with context
+    def format(wrapped_ractor, error_result)
+      timestamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
+      worker_class = wrapped_ractor.worker_class
+      worker_name = wrapped_ractor.name
+
+      # Build contextual error message
+      lines = [
+        "=" * 80,
+        "[#{timestamp}] ERROR PROCESSING WORK",
+        "=" * 80,
+        "Worker: #{worker_name} (#{worker_class})",
+        "Work Item: #{error_result.work&.inspect || 'unknown'}",
+        "Error: #{error_result.error}",
+      ]
+
+      # Add error category and severity if available
+      if error_result.respond_to?(:error_category) && error_result.error_category
+        lines << "Category: #{error_result.error_category}"
+      end
+      if error_result.respond_to?(:error_severity) && error_result.error_severity
+        lines << "Severity: #{error_result.error_severity}"
+      end
+
+      # Add suggestions based on error type
+      suggestion = suggest_fix_for(error_result)
+      lines << "Suggestion: #{suggestion}" if suggestion
+
+      lines << "=" * 80
+      lines.join("\n")
+    end
+
+    private
+
+    # Provide contextual suggestions for common errors.
+    #
+    # @param error_result [WorkResult] The error result
+    # @return [String, nil] Suggestion string or nil
+    def suggest_fix_for(error_result)
+      error_msg = error_result.error.to_s
+
+      case error_msg
+      when /negative number/i
+        "Check if input validation is needed. Consider using AbsWorker for positive-only values."
+      when /timeout/i
+        "Consider increasing timeout duration or breaking work into smaller chunks."
+      when /memory/i
+        "Try processing smaller batches or increasing available memory."
+      when /connection/i
+        "Verify network connectivity and service availability."
+      when /undefined method/i
+        "Ensure the Worker class implements all required methods for the Work type."
+      when /nil/i
+        "Check if work items are being initialized with valid input data."
+      end
+    end
+  end
+end

data/lib/fractor/error_report_generator.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Fractor
+  # Generates error reports in multiple formats (text, JSON, Prometheus).
+  # Extracted from ErrorReporter for better separation of concerns.
+  class ErrorReportGenerator
+    # Generate a human-readable text report
+    #
+    # @param report_data [Hash] The report data from ErrorReporter
+    # @return [String] Formatted text report
+    def self.text_report(report_data)
+      lines = []
+      lines << "=" * 80
+      lines << "ERROR REPORT"
+      lines << "=" * 80
+      lines << ""
+
+      # Summary
+      lines << "SUMMARY"
+      lines << "-" * 80
+      summary = report_data[:summary]
+      lines << "Uptime:          #{summary[:uptime]}s"
+      lines << "Total Errors:    #{summary[:total_errors]}"
+      lines << "Total Successes: #{summary[:total_successes]}"
+      lines << "Error Rate:      #{summary[:error_rate]}%"
+      lines << ""
+
+      # Errors by Severity
+      lines << "Errors by Severity:"
+      summary[:errors_by_severity].each do |severity, count|
+        lines << "  #{severity.to_s.ljust(10)}: #{count}"
+      end
+      lines << ""
+
+      # Top Categories
+      lines << "TOP ERROR CATEGORIES"
+      lines << "-" * 80
+      report_data[:top_categories].each do |category, count|
+        lines << "#{category.to_s.ljust(20)}: #{count} errors"
+      end
+      lines << ""
+
+      # Top Jobs
+      unless report_data[:top_jobs].empty?
+        lines << "TOP ERROR JOBS"
+        lines << "-" * 80
+        report_data[:top_jobs].each do |job, count|
+          lines << "#{job.to_s.ljust(20)}: #{count} errors"
+        end
+        lines << ""
+      end
+
+      # Critical Errors
+      unless report_data[:critical_errors].empty?
+        lines << "CRITICAL ERRORS"
+        lines << "-" * 80
+        report_data[:critical_errors].each do |error_info|
+          lines << "Category: #{error_info[:category]}"
+          lines << "Count:    #{error_info[:count]}"
+          lines << "Recent errors:"
+          error_info[:recent].each do |err|
+            lines << "  - [#{err[:timestamp]}] #{err[:error_class]}: #{err[:error_message]}"
+          end
+          lines << ""
+        end
+      end
+
+      # Trending Errors
+      unless report_data[:trending_errors].empty?
+        lines << "TRENDING ERRORS (Increasing)"
+        lines << "-" * 80
+        report_data[:trending_errors].each do |trend|
+          stats = trend[:stats]
+          lines << "Category:    #{stats[:category]}"
+          lines << "Total Count: #{stats[:total_count]}"
+          lines << "Error Rate:  #{stats[:error_rate]}/s"
+          lines << "Trend:       #{stats[:trending]}"
+          lines << ""
+        end
+      end
+
+      lines << "=" * 80
+      lines.join("\n")
+    end
+
+    # Export errors to Prometheus format
+    #
+    # @param reporter [ErrorReporter] The error reporter instance
+    # @return [String] Prometheus metrics
+    def self.to_prometheus(reporter)
+      lines = []
+
+      # Total errors
+      lines << "# HELP fractor_errors_total Total number of errors"
+      lines << "# TYPE fractor_errors_total counter"
+      lines << "fractor_errors_total #{reporter.total_errors}"
+      lines << ""
+
+      # Total successes
+      lines << "# HELP fractor_successes_total Total number of successes"
+      lines << "# TYPE fractor_successes_total counter"
+      lines << "fractor_successes_total #{reporter.total_successes}"
+      lines << ""
+
+      # Error rate
+      lines << "# HELP fractor_error_rate Error rate percentage"
+      lines << "# TYPE fractor_error_rate gauge"
+      lines << "fractor_error_rate #{reporter.overall_error_rate}"
+      lines << ""
+
+      # Errors by severity
+      lines << "# HELP fractor_errors_by_severity Errors by severity level"
+      lines << "# TYPE fractor_errors_by_severity gauge"
+      reporter.errors_by_severity.each do |severity, count|
+        lines << "fractor_errors_by_severity{severity=\"#{severity}\"} #{count}"
+      end
+      lines << ""
+
+      # Errors by category
+      lines << "# HELP fractor_errors_by_category Errors by category"
+      lines << "# TYPE fractor_errors_by_category gauge"
+      reporter.instance_variable_get(:@by_category)&.each do |category, stats|
+        lines << "fractor_errors_by_category{category=\"#{category}\"} #{stats.total_count}"
+      end
+      lines << ""
+
+      # Errors by job
+      by_job = reporter.instance_variable_get(:@by_job)
+      unless by_job&.empty?
+        lines << "# HELP fractor_errors_by_job Errors by job name"
+        lines << "# TYPE fractor_errors_by_job gauge"
+        by_job.each do |job, stats|
+          lines << "fractor_errors_by_job{job=\"#{job}\"} #{stats.total_count}"
+        end
+        lines << ""
+      end
+
+      lines.join("\n")
+    end
+
+    # Export errors to JSON format
+    #
+    # @param report_data [Hash] The report data from ErrorReporter
+    # @param args [Array] Additional arguments for JSON generation
+    # @return [String] JSON representation
+    def self.to_json(report_data, *args)
+      report_data.to_json(*args)
+    end
+  end
+end

data/lib/fractor/error_reporter.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require_relative "error_statistics"
+require_relative "error_report_generator"
+
+module Fractor
+  # Error reporting and analytics system.
+  # Aggregates errors, tracks statistics, and provides actionable insights.
+  class ErrorReporter
+    attr_reader :start_time, :total_errors, :total_successes
+
+    def initialize
+      @start_time = Time.now
+      @total_errors = 0
+      @total_successes = 0
+      @by_category = {}
+      @by_job = {}
+      @error_handlers = []
+      @mutex = Mutex.new
+    end
+
+    # Record a work result
+    #
+    # @param work_result [WorkResult] The work result to record
+    # @param job_name [String, nil] Optional job name
+    # @return [void]
+    def record(work_result, job_name: nil)
+      @mutex.synchronize do
+        if work_result.success?
+          @total_successes += 1
+        else
+          @total_errors += 1
+          record_error(work_result, job_name)
+        end
+      end
+    end
+
+    # Register an error handler callback
+    #
+    # @yield [work_result, job_name] Block to call when error occurs
+    # @return [void]
+    def on_error(&block)
+      @error_handlers << block
+    end
+
+    # Get statistics for a category
+    #
+    # @param category [String, Symbol] The error category
+    # @return [Hash, nil] Statistics for the category
+    def category_stats(category)
+      @mutex.synchronize do
+        @by_category[category]&.to_h
+      end
+    end
+
+    # Get statistics for a job
+    #
+    # @param job_name [String] The job name
+    # @return [Hash, nil] Statistics for the job
+    def job_stats(job_name)
+      @mutex.synchronize do
+        @by_job[job_name]&.to_h
+      end
+    end
+
+    # Get overall error rate
+    #
+    # @return [Float] Error rate percentage
+    def overall_error_rate
+      total = @total_errors + @total_successes
+      return 0.0 if total.zero?
+
+      (@total_errors.to_f / total * 100).round(2)
+    end
+
+    # Get errors by severity
+    #
+    # @return [Hash] Error counts grouped by severity
+    def errors_by_severity
+      result = Hash.new(0)
+      @mutex.synchronize do
+        @by_category.each_value do |stats|
+          stats.by_severity.each do |severity, count|
+            result[severity] += count
+          end
+        end
+      end
+      result
+    end
+
+    # Get top error categories
+    #
+    # @param limit [Integer] Maximum number of categories to return
+    # @return [Hash] Top error categories with counts
+    def top_categories(limit: 5)
+      @mutex.synchronize do
+        @by_category
+          .map { |category, stats| [category, stats.total_count] }
+          .sort_by { |_category, count| -count }
+          .first(limit)
+          .to_h
+      end
+    end
+
+    # Get top error jobs
+    #
+    # @param limit [Integer] Maximum number of jobs to return
+    # @return [Hash] Top error jobs with counts
+    def top_jobs(limit: 5)
+      @mutex.synchronize do
+        @by_job
+          .map { |job, stats| [job, stats.total_count] }
+          .sort_by { |_job, count| -count }
+          .first(limit)
+          .to_h
+      end
+    end
+
+    # Get critical errors
+    #
+    # @return [Array<Hash>] Critical errors with recent occurrences
+    def critical_errors
+      errors = []
+      @mutex.synchronize do
+        @by_category.each do |category, stats|
+          critical_count = stats.by_severity[WorkResult::SEVERITY_CRITICAL] || 0
+          if critical_count.positive?
+            errors << {
+              category: category,
+              count: critical_count,
+              recent: stats.recent_errors.select do |e|
+                e[:error_severity] == WorkResult::SEVERITY_CRITICAL
+              end.last(5),
+            }
+          end
+        end
+      end
+      errors
+    end
+
+    # Get trending errors (increasing error rates)
+    #
+    # @return [Array<Hash>] Trending error categories
+    def trending_errors
+      trends = []
+      @mutex.synchronize do
+        @by_category.each do |category, stats|
+          if stats.increasing?
+            trends << { category: category,
+                        stats: stats.to_h }
+          end
+        end
+      end
+      trends
+    end
+
+    # Generate comprehensive report
+    #
+    # @return [Hash] Report data with all statistics
+    def report
+      {
+        summary: {
+          uptime: (Time.now - @start_time).round(2),
+          total_errors: @total_errors,
+          total_successes: @total_successes,
+          error_rate: overall_error_rate,
+          errors_by_severity: errors_by_severity,
+        },
+        top_categories: top_categories,
+        top_jobs: top_jobs,
+        critical_errors: critical_errors,
+        trending_errors: trending_errors,
+        category_breakdown: category_breakdown,
+      }
+    end
+
+    # Generate formatted text report
+    #
+    # @return [String] Formatted text report
+    def formatted_report
+      ErrorReportGenerator.text_report(report)
+    end
+
+    # Export to Prometheus format
+    #
+    # @return [String] Prometheus metrics
+    def to_prometheus
+      ErrorReportGenerator.to_prometheus(self)
+    end
+
+    # Export to JSON format
+    #
+    # @param args [Array] Additional arguments for JSON generation
+    # @return [String] JSON representation
+    def to_json(*args)
+      ErrorReportGenerator.to_json(report, *args)
+    end
+
+    # Reset all statistics
+    #
+    # @return [void]
+    def reset
+      @mutex.synchronize do
+        @start_time = Time.now
+        @total_errors = 0
+        @total_successes = 0
+        @by_category.clear
+        @by_job.clear
+      end
+    end
+
+    private
+
+    def record_error(work_result, job_name)
+      # Record by category
+      category = work_result.error_category
+      @by_category[category] ||= ErrorStatistics.new(category)
+      @by_category[category].record(work_result)
+
+      # Record by job if provided
+      if job_name
+        @by_job[job_name] ||= ErrorStatistics.new(job_name)
+        @by_job[job_name].record(work_result)
+      end
+
+      # Invoke error handlers
+      @error_handlers.each do |handler|
+        handler.call(work_result, job_name)
+      rescue StandardError => e
+        warn "Error in error handler: #{e.message}"
+      end
+    end
+
+    def category_breakdown
+      breakdown = {}
+      @mutex.synchronize do
+        @by_category.each do |category, stats|
+          breakdown[category] = stats.to_h
+        end
+      end
+      breakdown
+    end
+  end
+end