pigeon-rb 0.1.0

data/lib/pigeon/configuration.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "dry-configurable"
+require "logger"
+
+module Pigeon
+  # Configuration class for Pigeon using dry-configurable
+  class Configuration
+    extend Dry::Configurable
+
+    # Karafka client ID
+    setting :client_id, default: "pigeon"
+
+    # Kafka seed brokers configuration
+    setting :kafka_brokers, default: ["localhost:9092"]
+
+    # Maximum number of retries for failed messages
+    setting :max_retries, default: 10
+
+    # Base retry delay (will be used with exponential backoff)
+    setting :retry_delay, default: 30 # seconds
+
+    # Maximum retry delay
+    setting :max_retry_delay, default: 86_400 # 24 hours
+
+    # Whether to encrypt message payloads
+    setting :encrypt_payload, default: false
+
+    # Encryption key for payload encryption
+    setting :encryption_key, default: nil
+
+    # Sensitive fields to mask in logs
+    setting :sensitive_fields, default: []
+
+    # Retention period for processed messages
+    setting :retention_period, default: 7 # days
+
+    # Logger instance
+    setting :logger, default: Logger.new($stdout).tap { |l| l.level = Logger::INFO }
+
+    # Metrics collector
+    setting :metrics_collector, default: nil
+
+    # Tracer for OpenTelemetry
+    setting :tracer, default: nil
+
+    # Karafka additional configuration
+    setting :karafka_config, default: {}
+
+    # Schema validation configuration
+    setting :schema_validation_enabled, default: false
+
+    # Registered JSON schemas for validation
+    setting :schemas, default: {}
+
+    # Schema registry URL (if using a schema registry)
+    setting :schema_registry_url, default: nil
+
+    # Dead letter queue configuration
+    setting :dead_letter_queue_enabled, default: true
+
+    # Custom dead letter queue topic suffix
+    setting :dead_letter_queue_suffix, default: ".dlq"
+
+    # Error classification for retry strategy
+    setting :error_classifications, default: {
+      # Transient errors - retry with normal backoff
+      transient: [
+        "Kafka::ConnectionError",
+        "Kafka::NetworkError",
+        "Kafka::UnknownError",
+        "Kafka::LeaderNotAvailable",
+        "Kafka::NotLeaderForPartition"
+      ],
+      # Permanent errors - mark as failed after fewer retries
+      permanent: [
+        "Kafka::InvalidTopic",
+        "Kafka::TopicAuthorizationFailed",
+        "Kafka::MessageSizeTooLarge"
+      ],
+      # Application errors - mark as failed immediately
+      application: [
+        "Pigeon::Serializer::ValidationError",
+        "JSON::ParserError"
+      ]
+    }
+
+    class << self
+      # Reset the configuration to default values
+      # @return [void]
+      def reset_config
+        # For dry-configurable 1.x
+        if respond_to?(:settings)
+          settings.each_key do |key|
+            config[key] = settings[key].default
+          end
+        # For dry-configurable 0.x
+        elsif respond_to?(:config_option_definitions)
+          config_option_definitions.each_key do |key|
+            config[key] = config_option_definitions[key].default_value
+          end
+        else
+          # Simplest approach - just create a new configuration
+          @config = Dry::Configurable::Config.new
+        end
+      end
+
+      # Register a JSON schema for validation
+      # @param name [String, Symbol] Schema name
+      # @param schema [Hash, String] JSON schema
+      # @return [void]
+      def register_schema(name, schema)
+        config.schemas[name.to_sym] = schema
+      end
+
+      # Get a registered schema
+      # @param name [String, Symbol] Schema name
+      # @return [Hash, String, nil] JSON schema or nil if not found
+      def schema(name)
+        config.schemas[name.to_sym]
+      end
+
+      # Register a sensitive field for masking
+      # @param field [String, Symbol] Field name
+      # @return [void]
+      def register_sensitive_field(field)
+        config.sensitive_fields << field.to_sym unless config.sensitive_fields.include?(field.to_sym)
+      end
+
+      # Register multiple sensitive fields for masking
+      # @param fields [Array<String, Symbol>] Field names
+      # @return [void]
+      def register_sensitive_fields(fields)
+        fields.each { |field| register_sensitive_field(field) }
+      end
+
+      # Register an error class for a specific classification
+      # @param error_class [String, Class] Error class name or class
+      # @param classification [Symbol] Error classification (:transient, :permanent, :application)
+      # @return [void]
+      def register_error_classification(error_class, classification)
+        error_name = error_class.is_a?(Class) ? error_class.name : error_class.to_s
+        config.error_classifications[classification.to_sym] ||= []
+        config.error_classifications[classification.to_sym] << error_name
+      end
+
+      # Get the classification for an error
+      # @param error [StandardError] Error instance
+      # @return [Symbol] Error classification (:transient, :permanent, :application, or :unknown)
+      def classify_error(error)
+        error_name = error.class.name
+
+        config.error_classifications.each do |classification, error_classes|
+          return classification if error_classes.include?(error_name)
+        end
+
+        :unknown
+      end
+    end
+  end
+end

data/lib/pigeon/core.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Pigeon
+  # Core functionality for Pigeon
+  module Core
+    # Configure the gem
+    # @yield [config] Configuration instance
+    # @example
+    #   Pigeon.configure do |config|
+    #     config.client_id = "my-application"
+    #     config.kafka_brokers = ["kafka1:9092", "kafka2:9092"]
+    #     config.max_retries = 5
+    #   end
+    def self.configure
+      yield(Configuration.config) if block_given?
+      initialize_karafka if @karafka_initialized.nil?
+    end
+
+    # Get the configuration
+    # @return [Dry::Configurable::Config]
+    def self.config
+      Configuration.config
+    end
+
+    # Initialize the Karafka producer
+    # @return [Karafka::Producer]
+    def self.initialize_karafka # rubocop:disable Metrics/AbcSize
+      return @karafka_producer if @karafka_initialized
+
+      # Configure Karafka
+      begin
+        Karafka::Setup::Config.setup do |karafka_config|
+          karafka_config.client_id = config.client_id
+
+          # Set required Kafka configuration if not provided
+          if !config.karafka_config[:kafka] || config.karafka_config[:kafka].empty?
+            karafka_config.kafka = {
+              "bootstrap.servers": config.kafka_brokers.join(",")
+            }
+          end
+
+          # Apply any additional Karafka configuration
+          config.karafka_config.each do |key, value|
+            karafka_config.public_send("#{key}=", value) if karafka_config.respond_to?("#{key}=")
+          end
+        end
+
+        @karafka_initialized = true
+        @karafka_producer = Karafka.producer
+      rescue StandardError => e
+        config.logger.error("Failed to initialize Karafka: #{e.message}")
+        # Return a mock producer for testing
+        @karafka_initialized = true
+        @karafka_producer = MockProducer.new
+      end
+
+      @karafka_producer
+    end
+
+    # Get the Karafka producer instance
+    # @return [Karafka::Producer]
+    def self.karafka_producer
+      initialize_karafka unless @karafka_initialized
+      @karafka_producer
+    end
+
+    # Create a new publisher instance
+    # @return [Pigeon::Publisher]
+    def self.publisher
+      Publisher.new
+    end
+
+    # Create a new processor instance
+    # @param auto_start [Boolean] Whether to automatically start processing pending messages
+    # @return [Pigeon::Processor]
+    def self.processor(auto_start: false)
+      Processor.new(auto_start: auto_start)
+    end
+
+    # Start processing pending messages
+    # @param batch_size [Integer] Number of messages to process in one batch
+    # @param interval [Integer] Interval in seconds between processing batches
+    # @param thread_count [Integer] Number of threads to use for processing
+    # @return [Boolean] Whether processing was started
+    def self.start_processing(batch_size: 100, interval: 5, thread_count: 2)
+      @processor ||= processor
+      @processor.start_processing(batch_size: batch_size, interval: interval, thread_count: thread_count)
+    end
+
+    # Stop processing pending messages
+    # @return [Boolean] Whether processing was stopped
+    def self.stop_processing
+      return false unless @processor
+
+      @processor.stop_processing
+    end
+
+    # Check if processing is running
+    # @return [Boolean] Whether processing is running
+    def self.processing?
+      @processor&.processing? || false
+    end
+  end
+end

data/lib/pigeon/encryption.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require "openssl"
+require "base64"
+
+module Pigeon
+  # Encryption module for handling payload encryption and decryption
+  module Encryption
+    DEFAULT_CIPHER = "aes-256-gcm"
+
+    # Encrypt a payload
+    # @param payload [String] Payload to encrypt
+    # @param encryption_key [String] Encryption key
+    # @param encryption_iv [String, nil] Initialization vector (optional)
+    # @return [Hash] Encrypted payload with metadata
+    def self.encrypt(payload, encryption_key = nil, encryption_iv = nil)
+      # Return the payload as is if encryption is disabled
+      return payload unless Pigeon.config.encrypt_payload
+
+      # Get the encryption key
+      key = encryption_key || self.encryption_key
+      raise EncryptionError, "Encryption key is required" unless key
+
+      # Generate a random IV if not provided (GCM uses 12-byte IV)
+      iv = encryption_iv || OpenSSL::Random.random_bytes(12)
+
+      # Encrypt the payload using AES-256-GCM
+      cipher = OpenSSL::Cipher.new(DEFAULT_CIPHER)
+      cipher.encrypt
+      cipher.key = key
+      cipher.iv = iv
+
+      # Encrypt the payload
+      encrypted_data = cipher.update(payload) + cipher.final
+
+      # Get the authentication tag
+      auth_tag = cipher.auth_tag
+
+      # Encode the data, IV, and auth tag
+      encoded_data = Base64.strict_encode64(encrypted_data)
+      encoded_iv = Base64.strict_encode64(iv)
+      encoded_auth_tag = Base64.strict_encode64(auth_tag)
+
+      # Return the encrypted payload with metadata
+      {
+        data: encoded_data,
+        iv: encoded_iv,
+        auth_tag: encoded_auth_tag,
+        cipher: DEFAULT_CIPHER
+      }
+    rescue StandardError => e
+      raise EncryptionError, "Failed to encrypt payload: #{e.message}"
+    end
+
+    # Decrypt a payload
+    # @param encrypted_payload [Hash, String] Encrypted payload with metadata
+    # @param encryption_key [String] Encryption key
+    # @return [String] Decrypted payload
+    def self.decrypt(encrypted_payload, encryption_key = nil)
+      # Return the payload as is if it's not encrypted
+      return encrypted_payload if encrypted_payload.is_a?(String)
+
+      # Get the encryption key
+      key = encryption_key || self.encryption_key
+      raise EncryptionError, "Encryption key is required" unless key
+
+      begin
+        # Extract the encrypted data, IV, and auth tag
+        encoded_data = encrypted_payload[:data]
+        encoded_iv = encrypted_payload[:iv]
+        encoded_auth_tag = encrypted_payload[:auth_tag]
+
+        # Decode the data, IV, and auth tag
+        encrypted_data = Base64.strict_decode64(encoded_data)
+        iv = Base64.strict_decode64(encoded_iv)
+        auth_tag = Base64.strict_decode64(encoded_auth_tag) if encoded_auth_tag
+
+        # Decrypt using AES-256-GCM
+        decipher = OpenSSL::Cipher.new(DEFAULT_CIPHER)
+        decipher.decrypt
+        decipher.key = key
+        decipher.iv = iv
+        decipher.auth_tag = auth_tag if auth_tag
+
+        # Decrypt the payload
+        decipher.update(encrypted_data) + decipher.final
+      rescue StandardError => e
+        raise DecryptionError, "Failed to decrypt payload: #{e.message}"
+      end
+    end
+
+    # Mask sensitive data in a payload
+    # @param payload [Hash, String] Payload to mask
+    # @param sensitive_fields [Array<String, Symbol>] Fields to mask
+    # @return [Hash, String] Masked payload
+    def self.mask_payload(payload, sensitive_fields = nil)
+      # Return the payload as is if no sensitive fields are specified
+      return payload if payload.is_a?(String) || sensitive_fields.nil? || sensitive_fields.empty?
+
+      # Convert the payload to a hash if it's a string
+      payload_hash = payload.is_a?(String) ? ::JSON.parse(payload) : payload.dup
+
+      # Mask sensitive fields
+      MaskingHelper.mask_fields(payload_hash, sensitive_fields)
+    rescue StandardError => e
+      Pigeon.config.logger.warn("Failed to mask payload: #{e.message}")
+      payload
+    end
+
+    # Get the encryption key from the configuration
+    # @return [String, nil] Encryption key
+    def self.encryption_key
+      key = key_from_config
+      key ||= key_from_environment
+      key ||= key_from_rails_credentials
+      key ||= key_from_hanami_settings
+      key
+    end
+
+    # Custom error classes
+    class EncryptionError < StandardError; end
+    class DecryptionError < StandardError; end
+
+    private_class_method def self.key_from_config
+      Pigeon.config.encryption_key
+    end
+
+    private_class_method def self.key_from_environment
+      ENV.fetch("PIGEON_ENCRYPTION_KEY", nil)
+    end
+
+    private_class_method def self.key_from_rails_credentials
+      return nil unless defined?(Rails)
+      return nil unless Rails.respond_to?(:application)
+      return nil unless Rails.application.respond_to?(:credentials)
+      return nil unless Rails.application.credentials.respond_to?(:pigeon)
+
+      Rails.application.credentials.pigeon[:encryption_key]
+    end
+
+    private_class_method def self.key_from_hanami_settings
+      return nil unless defined?(Hanami)
+      return nil unless Hanami.respond_to?(:app)
+      return nil unless Hanami.app.respond_to?(:[])
+      return nil unless Hanami.app["settings"].respond_to?(:pigeon)
+
+      Hanami.app["settings"].pigeon.encryption_key
+    end
+  end
+
+  # Helper module for masking sensitive data
+  module MaskingHelper
+    # Mask fields in a hash
+    # @param hash [Hash] Hash to mask
+    # @param sensitive_fields [Array<String, Symbol>] Fields to mask
+    # @return [Hash] Masked hash
+    def self.mask_fields(hash, sensitive_fields)
+      hash.each_with_object({}) do |(key, value), result|
+        result[key] = determine_masked_value(key, value, sensitive_fields)
+      end
+    end
+
+    # Determine the masked value for a key-value pair
+    # @param key [Object] The key
+    # @param value [Object] The value
+    # @param sensitive_fields [Array<String, Symbol>] Fields to mask
+    # @return [Object] The masked or original value
+    def self.determine_masked_value(key, value, sensitive_fields)
+      key_sym = key.to_sym
+      key_str = key.to_s
+
+      if sensitive_fields.include?(key_sym) || sensitive_fields.include?(key_str)
+        # Mask the value
+        mask_value(value)
+      elsif value.is_a?(Hash)
+        # Recursively mask nested hashes
+        mask_fields(value, sensitive_fields)
+      elsif value.is_a?(Array)
+        # Recursively mask arrays of hashes
+        value.map { |v| v.is_a?(Hash) ? mask_fields(v, sensitive_fields) : v }
+      else
+        # Keep the value as is
+        value
+      end
+    end
+
+    # Mask a value
+    # @param value [Object] Value to mask
+    # @return [String] Masked value
+    def self.mask_value(value)
+      return "[REDACTED]" if value.nil?
+
+      case value
+      when String
+        # Mask the string with asterisks, keeping the first and last characters
+        if value.length <= 4
+          "*" * value.length
+        else
+          "#{value[0]}#{value[1..-2].gsub(/[^[:space:]]/, '*')}#{value[-1]}"
+        end
+      when Numeric
+        # Mask numbers with asterisks
+        "*" * value.to_s.length
+      when TrueClass, FalseClass
+        # Don't mask booleans
+        value
+      else
+        # Mask everything else
+        "[REDACTED]"
+      end
+    end
+  end
+end

data/lib/pigeon/generators/hanami/migration_generator.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Pigeon
+  module Generators
+    module Hanami
+      # Generator for creating the outbox message table migration for Hanami applications
+      class MigrationGenerator
+        attr_reader :app_name
+
+        def initialize(app_name = nil)
+          @app_name = app_name || detect_app_name
+        end
+
+        # Generate the migration file
+        # @return [String] Path to the generated file
+        def generate
+          timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S")
+          filename = "db/migrations/#{timestamp}_create_outbox_messages.rb"
+
+          # Create the migrations directory if it doesn't exist
+          FileUtils.mkdir_p("db/migrations")
+
+          # Write the migration file
+          File.write(filename, migration_content)
+
+          filename
+        end
+
+        private
+
+        # Detect the Hanami application name
+        # @return [String] Application name
+        def detect_app_name
+          if defined?(Hanami) && Hanami.respond_to?(:app)
+            Hanami.app.name.to_s
+          else
+            "App"
+          end
+        end
+
+        # Generate the migration content
+        # @return [String] Migration content
+        def migration_content
+          <<~RUBY
+            # frozen_string_literal: true
+
+            ROM::SQL.migration do
+              change do
+                create_table :outbox_messages do
+                  primary_key :id, type: :uuid
+
+                  # Message metadata
+                  column :topic, String, null: false
+                  column :key, String
+                  column :headers, :jsonb
+                  column :partition, Integer
+
+                  # Message content
+                  column :payload, String, text: true, null: false
+
+                  # Processing metadata
+                  column :status, String, null: false, default: "pending"
+                  column :retry_count, Integer, null: false, default: 0
+                  column :max_retries, Integer
+                  column :error_message, String, text: true
+                  column :correlation_id, String
+
+                  # Timestamps
+                  column :created_at, DateTime, null: false
+                  column :updated_at, DateTime, null: false
+                  column :published_at, DateTime
+                  column :next_retry_at, DateTime
+
+                  # Indexes
+                  index :status
+                  index :next_retry_at
+                  index :correlation_id
+                  index :created_at
+                end
+              end
+            end
+          RUBY
+        end
+      end
+    end
+  end
+end

data/lib/pigeon/generators/rails/install_generator.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module Pigeon
+  module Generators
+    module Rails
+      # Generator for installing Pigeon in a Rails application
+      class InstallGenerator < ::Rails::Generators::Base
+        source_root File.expand_path("templates", __dir__)
+        desc "Installs Pigeon configuration files"
+
+        def create_initializer_file
+          template "initializer.rb.erb", "config/initializers/pigeon.rb"
+        end
+
+        def create_migration_file
+          generate "pigeon:rails:migration"
+        end
+
+        def display_post_install_message
+          say "\nPigeon has been installed! 🐦", :green
+          say "\nNext steps:"
+          say "  1. Review the configuration in config/initializers/pigeon.rb"
+          say "  2. Run migrations with: rails db:migrate"
+          say "  3. Start using Pigeon in your application"
+          say "\nFor more information, visit: https://github.com/khaile/pigeon"
+        end
+      end
+    end
+  end
+end

data/lib/pigeon/generators/rails/migration_generator.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module Pigeon
+  module Generators
+    module Rails
+      # Generator for creating the outbox message table migration for Rails applications
+      class MigrationGenerator < ::Rails::Generators::Base
+        source_root File.expand_path("templates", __dir__)
+        desc "Creates a migration for the outbox message table"
+
+        def create_migration_file
+          timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S")
+          template "create_outbox_messages.rb.erb", "db/migrate/#{timestamp}_create_outbox_messages.rb"
+        end
+      end
+    end
+  end
+end

data/lib/pigeon/generators/rails/templates/create_outbox_messages.rb.erb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class CreateOutboxMessages < ActiveRecord::Migration[<%= ::Rails::VERSION::MAJOR %>.<%= ::Rails::VERSION::MINOR %>]
+  def change
+    create_table :outbox_messages, id: :uuid do |t|
+      # Message metadata
+      t.string :topic, null: false
+      t.string :key
+      t.jsonb :headers
+      t.integer :partition
+
+      # Message content
+      t.text :payload, null: false
+
+      # Processing metadata
+      t.string :status, null: false, default: "pending"
+      t.integer :retry_count, null: false, default: 0
+      t.integer :max_retries
+      t.text :error_message
+      t.string :correlation_id
+
+      # Timestamps
+      t.timestamps
+      t.datetime :published_at
+      t.datetime :next_retry_at
+    end
+
+    # Add indexes for efficient querying
+    add_index :outbox_messages, :status
+    add_index :outbox_messages, :next_retry_at
+    add_index :outbox_messages, :correlation_id
+    add_index :outbox_messages, :created_at
+  end
+end