pigeon-rb 0.1.0

data/lib/pigeon/generators/rails/templates/initializer.rb.erb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+# Pigeon configuration
+Pigeon.configure do |config|
+  # Kafka client ID (defaults to application name)
+  config.client_id = Rails.application.class.module_parent_name.underscore
+
+  # Kafka broker configuration
+  config.kafka_brokers = ENV.fetch("KAFKA_BROKERS", "localhost:9092").split(",")
+
+  # Maximum number of retries for failed messages
+  config.max_retries = ENV.fetch("PIGEON_MAX_RETRIES", 10).to_i
+
+  # Base retry delay (will be used with exponential backoff)
+  config.retry_delay = ENV.fetch("PIGEON_RETRY_DELAY", 30).to_i # seconds
+
+  # Maximum retry delay
+  config.max_retry_delay = ENV.fetch("PIGEON_MAX_RETRY_DELAY", 86_400).to_i # 24 hours
+
+  # Whether to encrypt message payloads
+  config.encrypt_payload = ActiveModel::Type::Boolean.new.cast(ENV.fetch("PIGEON_ENCRYPT_PAYLOAD", false))
+
+  # Encryption key for payload encryption (if enabled)
+  config.encryption_key = ENV["PIGEON_ENCRYPTION_KEY"]
+
+  # Retention period for processed messages
+  config.retention_period = ENV.fetch("PIGEON_RETENTION_PERIOD", 7).to_i # days
+
+  # Use Rails logger
+  config.logger = Rails.logger
+
+  # Register sensitive fields for masking in logs
+  # config.register_sensitive_fields(%i[password email credit_card])
+
+  # Dead letter queue configuration
+  config.dead_letter_queue_enabled = ActiveModel::Type::Boolean.new.cast(ENV.fetch("PIGEON_DLQ_ENABLED", true))
+  config.dead_letter_queue_suffix = ENV.fetch("PIGEON_DLQ_SUFFIX", ".dlq")
+
+  # Schema validation configuration
+  # config.schema_validation_enabled = true
+  # config.register_schema(:user, Rails.root.join("app/schemas/user.json").read)
+end
+
+# Start background processing in production environments
+if Rails.env.production? && ActiveModel::Type::Boolean.new.cast(ENV.fetch("PIGEON_AUTO_START", false))
+  Rails.application.config.after_initialize do
+    # Don't start in Rails console or rake tasks
+    unless defined?(Rails::Console) || File.basename($PROGRAM_NAME) == "rake"
+      batch_size = ENV.fetch("PIGEON_BATCH_SIZE", 100).to_i
+      interval = ENV.fetch("PIGEON_INTERVAL", 5).to_i
+      thread_count = ENV.fetch("PIGEON_THREAD_COUNT", 2).to_i
+
+      Rails.logger.info("Starting Pigeon background processing")
+      Pigeon.start_processing(
+        batch_size: batch_size,
+        interval: interval,
+        thread_count: thread_count
+      )
+    end
+  end
+end
+
+# Schedule periodic jobs if ActiveJob is available and enabled
+if defined?(ActiveJob) && ActiveModel::Type::Boolean.new.cast(ENV.fetch("PIGEON_SCHEDULE_JOBS", false))
+  Rails.application.config.after_initialize do
+    # Don't schedule in Rails console or rake tasks
+    unless defined?(Rails::Console) || File.basename($PROGRAM_NAME) == "rake"
+      # Use a job scheduler like Sidekiq-Scheduler, Whenever, or Rails' own scheduler
+      # This is just a placeholder - implement according to your job scheduling system
+      Rails.logger.info("Scheduling Pigeon periodic jobs")
+
+      # Example for Sidekiq-Scheduler (if available)
+      if defined?(Sidekiq) && defined?(Sidekiq::Scheduler)
+        Sidekiq.set_schedule("pigeon_processor", {
+          "class" => "Pigeon::ActiveJobIntegration::ProcessorJob",
+          "every" => ENV.fetch("PIGEON_PROCESS_INTERVAL", "1m"),
+          "args" => [ENV.fetch("PIGEON_BATCH_SIZE", 100).to_i]
+        })
+
+        Sidekiq.set_schedule("pigeon_cleanup", {
+          "class" => "Pigeon::ActiveJobIntegration::CleanupJob",
+          "every" => ENV.fetch("PIGEON_CLEANUP_INTERVAL", "1h"),
+          "args" => [ENV.fetch("PIGEON_RETENTION_PERIOD", 7).to_i]
+        })
+      end
+    end
+  end
+end

data/lib/pigeon/hanami_integration.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "pigeon"
+
+module Pigeon
+  # Hanami integration for Pigeon
+  module HanamiIntegration
+    # Register the Pigeon plugin with Hanami
+    class Plugin
+      def self.install(app, **_options)
+        # Configure Pigeon with Hanami logger
+        Pigeon.configure do |config|
+          config.logger = app["logger"]
+          config.client_id = app.name.to_s.underscore
+        end
+
+        # Register repositories and relations
+        register_repositories(app)
+      end
+
+      # Register repositories with Hanami container
+      def self.register_repositories(app)
+        # Register the outbox messages repository
+        app.register_provider :repositories, namespace: true do
+          prepare do
+            require "pigeon/models/adapters/rom_adapter"
+          end
+
+          start do
+            # Define the repository class
+            repository = Class.new(ROM::Repository[:outbox_messages]) do
+              commands :create, update: :by_pk, delete: :by_pk
+
+              def find(id)
+                outbox_messages.by_pk(id).one!
+              end
+
+              def find_by_status(status, limit = 100)
+                outbox_messages.by_status(status).limit(limit).to_a
+              end
+
+              def find_ready_for_retry(limit = 100)
+                outbox_messages.ready_for_retry.limit(limit).to_a
+              end
+            end
+
+            # Register the repository
+            register "outbox_messages", repository.new(target["persistence.rom"])
+          end
+        end
+      end
+    end
+
+    # Background job for processing outbox messages
+    class ProcessorJob
+      include Dry::Monads[:result]
+
+      # Process pending outbox messages
+      # @param batch_size [Integer] Number of messages to process in one batch
+      # @return [Hash] Processing statistics
+      def self.perform(batch_size = 100)
+        Pigeon.processor.process_pending(batch_size: batch_size)
+      end
+    end
+
+    # Background job for cleaning up processed outbox messages
+    class CleanupJob
+      include Dry::Monads[:result]
+
+      # Clean up processed outbox messages
+      # @param older_than [Integer] Age threshold for cleanup in days
+      # @return [Integer] Number of records cleaned up
+      def self.perform(older_than = 7)
+        Pigeon.processor.cleanup_processed(older_than: older_than)
+      end
+    end
+  end
+end

data/lib/pigeon/health_check/kafka.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Pigeon
+  module HealthCheck
+    # Kafka health check functionality
+    module Kafka
+      # Check the health of Kafka connectivity
+      # @return [Hash] Health check result
+      def self.health
+        # Try to connect to Kafka
+        begin
+          # Use Karafka producer to check connectivity
+          Pigeon.karafka_producer.produce_sync("health_check", topic: "health_check", key: "health_check")
+
+          status = "healthy"
+          message = "Kafka connection is healthy"
+          details = { connected: true }
+        rescue StandardError => e
+          status = "critical"
+          message = "Failed to connect to Kafka: #{e.message}"
+          details = {
+            connected: false,
+            error_class: e.class.name,
+            error_message: e.message
+          }
+        end
+
+        {
+          component: "kafka",
+          status: status,
+          message: message,
+          details: details
+        }
+      end
+    end
+  end
+end

data/lib/pigeon/health_check/processor.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Pigeon
+  module HealthCheck
+    # Processor health check functionality
+    module Processor
+      # Check the health of the processor
+      # @return [Hash] Health check result
+      def self.health
+        processor_running = Pigeon.processing?
+
+        # Get processor statistics
+        stats = statistics(processor_running)
+
+        # Determine overall status
+        status, message = status(processor_running, stats)
+
+        {
+          component: "processor",
+          status: status,
+          message: message,
+          details: stats
+        }
+      end
+
+      # Get processor statistics
+      # @param processor_running [Boolean] Whether the processor is running
+      # @return [Hash] Processor statistics
+      def self.statistics(processor_running)
+        stats = {
+          running: processor_running,
+          uptime_seconds: processor_running ? (Time.now - Pigeon.processor_start_time).to_i : 0,
+          last_run_at: Pigeon.last_processing_run,
+          last_successful_run_at: Pigeon.last_successful_processing_run
+        }
+
+        # Add error rate if metrics are available
+        add_error_rate_to_stats(stats)
+
+        stats
+      end
+
+      # Add error rate to statistics if metrics are available
+      # @param stats [Hash] Statistics hash to update
+      # @return [void]
+      def self.add_error_rate_to_stats(stats)
+        return unless Pigeon.config.metrics_collector
+
+        total_processed = Pigeon.metrics_collector.get_counter(:messages_processed_total) || 0
+        total_failed = Pigeon.metrics_collector.get_counter(:messages_failed_total) || 0
+
+        stats[:error_rate] = total_processed.positive? ? (total_failed.to_f / total_processed) : 0
+      end
+
+      # Determine processor status
+      # @param processor_running [Boolean] Whether the processor is running
+      # @param stats [Hash] Processor statistics
+      # @return [Array<String, String>] Status and message
+      def self.status(processor_running, stats)
+        if !processor_running
+          ["critical", "Processor is not running"]
+        elsif stats[:last_run_at].nil? || (Time.now - stats[:last_run_at]) > 300 # 5 minutes
+          ["warning", "Processor has not run in the last 5 minutes"]
+        else
+          ["healthy", "Processor is running normally"]
+        end
+      end
+    end
+  end
+end

data/lib/pigeon/health_check/queue.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Pigeon
+  module HealthCheck
+    # Queue health check functionality
+    module Queue
+      # Check the health of the queue
+      # @return [Hash] Health check result
+      def self.health
+        # Get queue statistics
+        stats = statistics
+
+        # Determine overall status based on queue depth and age
+        status, message = status(stats)
+
+        {
+          component: "queue",
+          status: status,
+          message: message,
+          details: stats
+        }
+      end
+
+      # Get queue statistics
+      # @return [Hash] Queue statistics
+      def self.statistics
+        pending_count = Pigeon.count_outbox_messages_by_status("pending")
+        failed_count = Pigeon.count_outbox_messages_by_status("failed")
+
+        # Get the oldest pending message
+        oldest_pending = Pigeon.find_oldest_outbox_message_by_status("pending")
+        oldest_pending_age = oldest_pending ? (Time.now - oldest_pending.created_at).to_i : 0
+
+        {
+          pending_count: pending_count,
+          failed_count: failed_count,
+          oldest_pending_age_seconds: oldest_pending_age,
+          oldest_pending_id: oldest_pending&.id
+        }
+      end
+
+      # Determine queue status
+      # @param stats [Hash] Queue statistics
+      # @return [Array<String, String>] Status and message
+      def self.status(stats)
+        pending_count = stats[:pending_count]
+        age_seconds = stats[:oldest_pending_age_seconds]
+
+        message = queue_status_message(pending_count, age_seconds)
+
+        if pending_count > 1000 || age_seconds > 3600 # 1 hour
+          ["critical", message]
+        elsif pending_count > 100 || age_seconds > 300 # 5 minutes
+          ["warning", message]
+        else
+          ["healthy", "Queue is processing normally"]
+        end
+      end
+
+      # Generate queue status message
+      # @param pending_count [Integer] Number of pending messages
+      # @param age_seconds [Integer] Age of oldest message in seconds
+      # @return [String] Status message
+      def self.queue_status_message(pending_count, age_seconds)
+        "Queue has #{pending_count} pending messages, oldest is #{age_seconds} seconds old"
+      end
+    end
+  end
+end

data/lib/pigeon/health_check.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative "health_check/processor"
+require_relative "health_check/queue"
+require_relative "health_check/kafka"
+
+module Pigeon
+  # Health check module for Pigeon
+  module HealthCheck
+    # Check the health of the processor
+    # @return [Hash] Health check result
+    def self.processor_health
+      Processor.health
+    end
+
+    # Check the health of the queue
+    # @return [Hash] Health check result
+    def self.queue_health
+      Queue.health
+    end
+
+    # Check the health of Kafka connectivity
+    # @return [Hash] Health check result
+    def self.kafka_health
+      Kafka.health
+    end
+
+    # Get overall health status
+    # @return [Hash] Overall health status
+    def self.status
+      processor = processor_health
+      queue = queue_health
+      kafka = kafka_health
+
+      # Determine overall status (worst of all components)
+      components = [processor, queue, kafka]
+      status = determine_overall_status(components)
+
+      {
+        status: status,
+        components: {
+          processor: processor,
+          queue: queue,
+          kafka: kafka
+        },
+        timestamp: Time.now.iso8601
+      }
+    end
+
+    # Determine the overall status based on component statuses
+    # @param components [Array<Hash>] Component health check results
+    # @return [String] Overall status
+    def self.determine_overall_status(components)
+      if components.any? { |c| c[:status] == "critical" }
+        "critical"
+      elsif components.any? { |c| c[:status] == "warning" }
+        "warning"
+      else
+        "healthy"
+      end
+    end
+  end
+end

data/lib/pigeon/logging/structured_logger.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "logger"
+require "json"
+
+module Pigeon
+  module Logging
+    # Structured logger for Pigeon
+    class StructuredLogger
+      # Log levels
+      LEVELS = {
+        debug: Logger::DEBUG,
+        info: Logger::INFO,
+        warn: Logger::WARN,
+        error: Logger::ERROR,
+        fatal: Logger::FATAL
+      }.freeze
+
+      # Initialize a new structured logger
+      # @param logger [Logger] Base logger to use
+      # @param default_context [Hash] Default context to include in all log entries
+      def initialize(logger = nil, default_context = {})
+        @logger = logger || Logger.new($stdout)
+        @default_context = default_context || {}
+      end
+
+      # Set the log level
+      # @param level [Symbol, String, Integer] Log level
+      # @return [Integer] New log level
+      def level=(level)
+        level_int = level_to_int(level)
+        @logger.level = level_int
+      end
+
+      # Get the current log level
+      # @return [Integer] Current log level
+      def level
+        @logger.level
+      end
+
+      # Check if a log level is enabled
+      # @param level [Symbol, String, Integer] Log level
+      # @return [Boolean] Whether the log level is enabled
+      def level_enabled?(level)
+        @logger.level <= level_to_int(level)
+      end
+
+      # Log a debug message
+      # @param message [String] Log message
+      # @param context [Hash] Additional context for the log entry
+      # @return [void]
+      def debug(message, context = {})
+        log(:debug, message, context)
+      end
+
+      # Log an info message
+      # @param message [String] Log message
+      # @param context [Hash] Additional context for the log entry
+      # @return [void]
+      def info(message, context = {})
+        log(:info, message, context)
+      end
+
+      # Log a warning message
+      # @param message [String] Log message
+      # @param context [Hash] Additional context for the log entry
+      # @return [void]
+      def warn(message, context = {})
+        log(:warn, message, context)
+      end
+
+      # Log an error message
+      # @param message [String] Log message
+      # @param context [Hash] Additional context for the log entry
+      # @param error [Exception, nil] Optional error to include in the log entry
+      # @return [void]
+      def error(message, context = {}, error = nil)
+        context = context.dup
+        if error
+          context[:error] = {
+            class: error.class.name,
+            message: error.message,
+            backtrace: error.backtrace&.take(10)
+          }
+        end
+        log(:error, message, context)
+      end
+
+      # Log a fatal message
+      # @param message [String] Log message
+      # @param context [Hash] Additional context for the log entry
+      # @param error [Exception, nil] Optional error to include in the log entry
+      # @return [void]
+      def fatal(message, context = {}, error = nil)
+        context = context.dup
+        if error
+          context[:error] = {
+            class: error.class.name,
+            message: error.message,
+            backtrace: error.backtrace
+          }
+        end
+        log(:fatal, message, context)
+      end
+
+      # Check if debug level is enabled
+      # @return [Boolean] Whether debug level is enabled
+      def debug?
+        level_enabled?(:debug)
+      end
+
+      # Check if info level is enabled
+      # @return [Boolean] Whether info level is enabled
+      def info?
+        level_enabled?(:info)
+      end
+
+      # Check if warn level is enabled
+      # @return [Boolean] Whether warn level is enabled
+      def warn?
+        level_enabled?(:warn)
+      end
+
+      # Check if error level is enabled
+      # @return [Boolean] Whether error level is enabled
+      def error?
+        level_enabled?(:error)
+      end
+
+      # Check if fatal level is enabled
+      # @return [Boolean] Whether fatal level is enabled
+      def fatal?
+        level_enabled?(:fatal)
+      end
+
+      # Create a new logger with additional default context
+      # @param additional_context [Hash] Additional context to include in all log entries
+      # @return [StructuredLogger] New logger instance
+      def with_context(additional_context)
+        self.class.new(@logger, @default_context.merge(additional_context))
+      end
+
+      private
+
+      # Log a message at the specified level
+      # @param level [Symbol] Log level
+      # @param message [String] Log message
+      # @param context [Hash] Additional context for the log entry
+      # @return [void]
+      def log(level, message, context = {})
+        return unless level_enabled?(level)
+
+        # Combine default context and provided context
+        combined_context = @default_context.merge(context)
+
+        # Create the log entry
+        log_entry = {
+          timestamp: Time.now.iso8601(3),
+          level: level.to_s.upcase,
+          message: message
+        }
+
+        # Add context if present
+        log_entry[:context] = combined_context unless combined_context.empty?
+
+        # Log as JSON
+        @logger.send(level, log_entry.to_json)
+      end
+
+      # Convert a log level to its integer value
+      # @param level [Symbol, String, Integer] Log level
+      # @return [Integer] Integer log level
+      def level_to_int(level)
+        return level if level.is_a?(Integer)
+        return LEVELS[level.to_sym] if level.respond_to?(:to_sym) && LEVELS.key?(level.to_sym)
+
+        Logger::INFO
+      end
+    end
+  end
+end