pigeon-rb 0.1.0

data/lib/pigeon/metrics/collector.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Pigeon
+  module Metrics
+    # Metrics collector for Pigeon
+    class Collector
+      # Initialize a new metrics collector
+      # @param adapter [Object, nil] Optional metrics adapter (e.g., Prometheus, StatsD)
+      def initialize(adapter = nil)
+        @adapter = adapter
+        @counters = {}
+        @gauges = {}
+        @histograms = {}
+      end
+
+      # Increment a counter
+      # @param name [Symbol, String] Counter name
+      # @param value [Integer] Value to increment by (default: 1)
+      # @param tags [Hash] Optional tags for the metric
+      # @return [Integer] New counter value
+      def increment(name, value = 1, tags = {})
+        counter_key = metric_key(name, tags)
+        @counters[counter_key] ||= 0
+        @counters[counter_key] += value
+
+        # If an adapter is provided, delegate to it
+        @adapter&.increment(name, value, tags) if @adapter.respond_to?(:increment)
+
+        @counters[counter_key]
+      end
+
+      # Set a gauge value
+      # @param name [Symbol, String] Gauge name
+      # @param value [Numeric] Value to set
+      # @param tags [Hash] Optional tags for the metric
+      # @return [Numeric] Set value
+      def gauge(name, value, tags = {})
+        gauge_key = metric_key(name, tags)
+        @gauges[gauge_key] = value
+
+        # If an adapter is provided, delegate to it
+        @adapter&.gauge(name, value, tags) if @adapter.respond_to?(:gauge)
+
+        value
+      end
+
+      # Record a histogram value
+      # @param name [Symbol, String] Histogram name
+      # @param value [Numeric] Value to record
+      # @param tags [Hash] Optional tags for the metric
+      # @return [Array] All recorded values for this histogram
+      def histogram(name, value, tags = {})
+        histogram_key = metric_key(name, tags)
+        @histograms[histogram_key] ||= []
+        @histograms[histogram_key] << value
+
+        # If an adapter is provided, delegate to it
+        @adapter&.histogram(name, value, tags) if @adapter.respond_to?(:histogram)
+
+        @histograms[histogram_key]
+      end
+
+      # Get the current value of a counter
+      # @param name [Symbol, String] Counter name
+      # @param tags [Hash] Optional tags for the metric
+      # @return [Integer] Current counter value
+      def get_counter(name, tags = {})
+        @counters[metric_key(name, tags)] || 0
+      end
+
+      # Get the current value of a gauge
+      # @param name [Symbol, String] Gauge name
+      # @param tags [Hash] Optional tags for the metric
+      # @return [Numeric, nil] Current gauge value
+      def get_gauge(name, tags = {})
+        @gauges[metric_key(name, tags)]
+      end
+
+      # Get all recorded values for a histogram
+      # @param name [Symbol, String] Histogram name
+      # @param tags [Hash] Optional tags for the metric
+      # @return [Array] All recorded values
+      def get_histogram(name, tags = {})
+        @histograms[metric_key(name, tags)] || []
+      end
+
+      # Get all metrics
+      # @return [Hash] All metrics
+      def all_metrics
+        {
+          counters: @counters,
+          gauges: @gauges,
+          histograms: @histograms
+        }
+      end
+
+      # Reset all metrics
+      # @return [void]
+      def reset
+        @counters = {}
+        @gauges = {}
+        @histograms = {}
+      end
+
+      # Get metrics in a format suitable for monitoring systems
+      # @return [Hash] Formatted metrics
+      def metrics_for_monitoring
+        {
+          counters: format_counters,
+          gauges: format_gauges,
+          histograms: format_histograms
+        }
+      end
+
+      # Format counters for monitoring
+      # @return [Array<Hash>] Formatted counters
+      def format_counters
+        @counters.map do |key, value|
+          name, tags = parse_metric_key(key)
+          {
+            name: name,
+            value: value,
+            tags: tags
+          }
+        end
+      end
+
+      # Format gauges for monitoring
+      # @return [Array<Hash>] Formatted gauges
+      def format_gauges
+        @gauges.map do |key, value|
+          name, tags = parse_metric_key(key)
+          {
+            name: name,
+            value: value,
+            tags: tags
+          }
+        end
+      end
+
+      # Format histograms for monitoring
+      # @return [Array<Hash>] Formatted histograms with statistics
+      def format_histograms
+        @histograms.map do |key, values|
+          next if values.empty?
+
+          name, tags = parse_metric_key(key)
+          sorted = values.sort
+
+          {
+            name: name,
+            count: values.size,
+            min: sorted.first,
+            max: sorted.last,
+            mean: values.sum / values.size.to_f,
+            median: sorted[values.size / 2],
+            p95: sorted[(values.size * 0.95).floor],
+            p99: sorted[(values.size * 0.99).floor],
+            tags: tags
+          }
+        end.compact
+      end
+
+      private
+
+      # Generate a unique key for a metric based on name and tags
+      # @param name [Symbol, String] Metric name
+      # @param tags [Hash] Tags for the metric
+      # @return [String] Unique metric key
+      def metric_key(name, tags)
+        return name.to_s if tags.nil? || tags.empty?
+
+        # Sort tags by key to ensure consistent key generation
+        sorted_tags = tags.sort_by { |k, _| k.to_s }
+        tag_str = sorted_tags.map { |k, v| "#{k}:#{v}" }.join(",")
+        "#{name}[#{tag_str}]"
+      end
+
+      # Parse a metric key into name and tags
+      # @param key [String] Metric key
+      # @return [Array<String, Hash>] Name and tags
+      def parse_metric_key(key)
+        if key.include?("[") && key.end_with?("]")
+          name, tags_str = key.split("[", 2)
+          tags_str = tags_str[0..-2] # Remove trailing "]"
+
+          tags = {}
+          tags_str.split(",").each do |tag_pair|
+            tag_key, tag_value = tag_pair.split(":", 2)
+            tags[tag_key] = tag_value if tag_key && tag_value
+          end
+
+          [name, tags]
+        else
+          [key, {}]
+        end
+      end
+    end
+  end
+end

data/lib/pigeon/mock_producer.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Pigeon
+  # Mock producer for testing
+  class MockProducer
+    def produce_sync?(_payload, **_options)
+      true
+    end
+
+    def produce_async?(_payload, **_options)
+      true
+    end
+
+    # For compatibility with the real Karafka producer
+    alias produce_sync produce_sync?
+    alias produce_async produce_async?
+  end
+end

data/lib/pigeon/models/adapters/active_record_adapter.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Pigeon
+  module Models
+    module Adapters
+      # ActiveRecord adapter for OutboxMessage
+      class ActiveRecordAdapter < Pigeon::Models::OutboxMessage
+        # Define the ActiveRecord model
+        def self.define_model
+          return @model if @model
+
+          # Define the ActiveRecord model class
+          @model = Class.new(ActiveRecord::Base) do
+            self.table_name = "outbox_messages"
+
+            # Validations
+            validates :topic, presence: true
+            validates :payload, presence: true
+            validates :status, presence: true, inclusion: { in: Pigeon::Models::OutboxMessage::STATUSES }
+            validates :retry_count, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+
+            # Serialize headers as JSON
+            serialize :headers, JSON if respond_to?(:serialize)
+          end
+
+          # Return the model class
+          @model
+        end
+
+        # Get the ActiveRecord model class
+        # @return [Class] ActiveRecord model class
+        def self.model
+          define_model
+        end
+
+        # Create a new outbox message
+        # @param attributes [Hash] Message attributes
+        # @return [OutboxMessage] New message instance
+        def self.create(attributes = {})
+          record = model.create!(prepare_attributes(attributes))
+          new_from_record(record)
+        end
+
+        # Find a message by ID
+        # @param id [String, Integer] Message ID
+        # @return [OutboxMessage, nil] Message instance or nil if not found
+        def self.find(id)
+          record = model.find_by(id: id)
+          record ? new_from_record(record) : nil
+        end
+
+        # Find messages by status
+        # @param status [String] Message status
+        # @param limit [Integer] Maximum number of messages to return
+        # @return [Array<OutboxMessage>] Array of message instances
+        def self.find_by_status(status, limit = 100)
+          records = model.where(status: status).order(created_at: :asc).limit(limit)
+          records.map { |record| new_from_record(record) }
+        end
+
+        # Find messages ready for retry
+        # @param limit [Integer] Maximum number of messages to return
+        # @return [Array<OutboxMessage>] Array of message instances
+        def self.find_ready_for_retry(limit = 100)
+          now = Time.now
+          records = model.where(status: "pending")
+                         .where("next_retry_at IS NULL OR next_retry_at <= ?", now)
+                         .order(created_at: :asc)
+                         .limit(limit)
+          records.map { |record| new_from_record(record) }
+        end
+
+        # Count messages by status
+        # @param status [String] Message status
+        # @return [Integer] Count of messages with the given status
+        def self.count_by_status(status)
+          model.where(status: status).count
+        end
+
+        # Find the oldest message by status
+        # @param status [String] Message status
+        # @return [OutboxMessage, nil] Oldest message or nil if none found
+        def self.find_oldest_by_status(status)
+          record = model.where(status: status).order(created_at: :asc).first
+          record ? new_from_record(record) : nil
+        end
+
+        # Create a new OutboxMessage instance from an ActiveRecord record
+        # @param record [ActiveRecord::Base] ActiveRecord record
+        # @return [OutboxMessage] New message instance
+        def self.new_from_record(record)
+          attributes = {}
+          ATTRIBUTES.each do |attr|
+            attributes[attr] = record.send(attr) if record.respond_to?(attr)
+          end
+          new(attributes).tap { |msg| msg.instance_variable_set(:@record, record) }
+        end
+
+        # Prepare attributes for ActiveRecord
+        # @param attributes [Hash] Raw attributes
+        # @return [Hash] Prepared attributes
+        def self.prepare_attributes(attributes)
+          attributes = attributes.dup
+
+          # Convert Time objects to the format expected by ActiveRecord
+          %i[created_at updated_at published_at next_retry_at].each do |attr|
+            attributes[attr] = attributes[attr].to_time if attributes[attr].is_a?(Time)
+          end
+
+          # Ensure headers is a hash
+          attributes[:headers] ||= {}
+
+          attributes
+        end
+
+        # Save the message
+        # @return [Boolean] Whether the save was successful
+        def save
+          if @record
+            @record.attributes = self.class.prepare_attributes(@attributes)
+            @record.save
+          else
+            @record = self.class.model.create!(self.class.prepare_attributes(@attributes))
+            true
+          end
+        rescue ActiveRecord::RecordInvalid => e
+          Pigeon.config.logger.error("Failed to save outbox message: #{e.message}")
+          false
+        end
+      end
+    end
+  end
+end

data/lib/pigeon/models/adapters/rom_adapter.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Pigeon
+  module Models
+    module Adapters
+      # ROM adapter for OutboxMessage (for Hanami applications)
+      class RomAdapter < Pigeon::Models::OutboxMessage
+        # Define the ROM relation
+        def self.define_relation
+          return @relation if @relation
+
+          # Get the ROM container
+          container = Hanami.app["persistence.rom"]
+
+          # Define the relation if it doesn't exist
+          unless container.relations.key?(:outbox_messages)
+            container.register_relation(Class.new(ROM::Relation[:sql]) do
+              schema(:outbox_messages, infer: true)
+
+              # Query methods
+              def by_status(status)
+                where(status: status).order(:created_at)
+              end
+
+              def ready_for_retry
+                now = Time.now
+                where(status: "pending")
+                  .where { next_retry_at.nil? | (next_retry_at <= now) }
+                  .order(:created_at)
+              end
+            end)
+          end
+
+          # Get the relation
+          @relation = container.relations[:outbox_messages]
+        end
+
+        # Get the ROM relation
+        # @return [ROM::Relation] ROM relation
+        def self.relation
+          define_relation
+        end
+
+        # Get the ROM repository
+        # @return [ROM::Repository] ROM repository
+        def self.repository
+          @repository ||= Hanami.app["repositories.outbox_messages"]
+        end
+
+        # Create a new outbox message
+        # @param attributes [Hash] Message attributes
+        # @return [OutboxMessage] New message instance
+        def self.create(attributes = {})
+          attributes = prepare_attributes(attributes)
+          record = repository.create(attributes)
+          new_from_record(record)
+        end
+
+        # Find a message by ID
+        # @param id [String, Integer] Message ID
+        # @return [OutboxMessage, nil] Message instance or nil if not found
+        def self.find(id)
+          record = repository.find(id)
+          record ? new_from_record(record) : nil
+        rescue ROM::TupleCountMismatchError
+          nil
+        end
+
+        # Find messages by status
+        # @param status [String] Message status
+        # @param limit [Integer] Maximum number of messages to return
+        # @return [Array<OutboxMessage>] Array of message instances
+        def self.find_by_status(status, limit = 100)
+          records = relation.by_status(status).limit(limit).to_a
+          records.map { |record| new_from_record(record) }
+        end
+
+        # Find messages ready for retry
+        # @param limit [Integer] Maximum number of messages to return
+        # @return [Array<OutboxMessage>] Array of message instances
+        def self.find_ready_for_retry(limit = 100)
+          records = relation.ready_for_retry.limit(limit).to_a
+          records.map { |record| new_from_record(record) }
+        end
+
+        # Count messages by status
+        # @param status [String] Message status
+        # @return [Integer] Count of messages with the given status
+        def self.count_by_status(status)
+          relation.by_status(status).count
+        end
+
+        # Find the oldest message by status
+        # @param status [String] Message status
+        # @return [OutboxMessage, nil] Oldest message or nil if none found
+        def self.find_oldest_by_status(status)
+          record = relation.by_status(status).order(:created_at).limit(1).one
+          record ? new_from_record(record) : nil
+        end
+
+        # Create a new OutboxMessage instance from a ROM record
+        # @param record [ROM::Struct] ROM record
+        # @return [OutboxMessage] New message instance
+        def self.new_from_record(record)
+          attributes = {}
+          ATTRIBUTES.each do |attr|
+            attributes[attr] = record.send(attr) if record.respond_to?(attr)
+          end
+          new(attributes).tap { |msg| msg.instance_variable_set(:@record, record) }
+        end
+
+        # Prepare attributes for ROM
+        # @param attributes [Hash] Raw attributes
+        # @return [Hash] Prepared attributes
+        def self.prepare_attributes(attributes)
+          attributes = attributes.dup
+
+          # Convert Time objects to the format expected by ROM
+          %i[created_at updated_at published_at next_retry_at].each do |attr|
+            attributes[attr] = attributes[attr].to_time if attributes[attr].is_a?(Time)
+          end
+
+          # Ensure headers is a hash
+          attributes[:headers] ||= {}
+
+          attributes
+        end
+
+        # Save the message
+        # @return [Boolean] Whether the save was successful
+        def save
+          if @record
+            # Update existing record
+            id = @record.id
+            attributes = self.class.prepare_attributes(@attributes)
+            self.class.repository.update(id, attributes)
+          else
+            # Create new record
+            attributes = self.class.prepare_attributes(@attributes)
+            @record = self.class.repository.create(attributes)
+          end
+          true
+        rescue StandardError => e
+          Pigeon.config.logger.error("Failed to save outbox message: #{e.message}")
+          false
+        end
+      end
+    end
+  end
+end

data/lib/pigeon/models/outbox_message.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Pigeon
+  module Models
+    # Base class for outbox message model
+    # This is a framework-agnostic representation of the outbox message
+    class OutboxMessage
+      # Attributes that should be present in all framework implementations
+      ATTRIBUTES = %i[
+        id
+        topic
+        key
+        headers
+        partition
+        payload
+        status
+        retry_count
+        max_retries
+        error_message
+        correlation_id
+        created_at
+        updated_at
+        published_at
+        next_retry_at
+      ].freeze
+
+      # Valid status values
+      STATUSES = %w[pending processing published failed].freeze
+
+      # Default values for attributes
+      DEFAULTS = {
+        status: "pending",
+        retry_count: 0,
+        headers: {},
+        created_at: -> { Time.now },
+        updated_at: -> { Time.now }
+      }.freeze
+
+      # Create a new outbox message with the given attributes
+      # @param attributes [Hash] Message attributes
+      # @return [OutboxMessage] New message instance
+      def self.create(attributes = {})
+        new(attributes)
+      end
+
+      # Find a message by ID
+      # @param id [String, Integer] Message ID
+      # @return [OutboxMessage, nil] Message instance or nil if not found
+      def self.find(id)
+        raise NotImplementedError, "#{self.class.name}#find must be implemented by a framework adapter"
+      end
+
+      # Find messages by status
+      # @param status [String] Message status
+      # @param limit [Integer] Maximum number of messages to return
+      # @return [Array<OutboxMessage>] Array of message instances
+      def self.find_by_status(status, limit = 100)
+        raise NotImplementedError, "#{self.class.name}#find_by_status must be implemented by a framework adapter"
+      end
+
+      # Find messages ready for retry
+      # @param limit [Integer] Maximum number of messages to return
+      # @return [Array<OutboxMessage>] Array of message instances
+      def self.find_ready_for_retry(limit = 100)
+        raise NotImplementedError, "#{self.class.name}#find_ready_for_retry must be implemented by a framework adapter"
+      end
+
+      # Count messages by status
+      # @param status [String] Message status
+      # @return [Integer] Count of messages with the given status
+      def self.count_by_status(status)
+        raise NotImplementedError, "#{self.class.name}#count_by_status must be implemented by a framework adapter"
+      end
+
+      # Find the oldest message by status
+      # @param status [String] Message status
+      # @return [OutboxMessage, nil] Oldest message or nil if none found
+      def self.find_oldest_by_status(status)
+        raise NotImplementedError, "#{self.class.name}#find_oldest_by_status must be implemented by a framework adapter"
+      end
+
+      # Initialize a new outbox message
+      # @param attributes [Hash] Message attributes
+      def initialize(attributes = {})
+        @attributes = DEFAULTS.dup
+        attributes.each do |key, value|
+          send("#{key}=", value) if respond_to?("#{key}=")
+        end
+      end
+
+      # Get an attribute value
+      # @param name [Symbol] Attribute name
+      # @return [Object] Attribute value
+      def [](name)
+        @attributes[name.to_sym]
+      end
+
+      # Set an attribute value
+      # @param name [Symbol] Attribute name
+      # @param value [Object] Attribute value
+      def []=(name, value)
+        @attributes[name.to_sym] = value
+      end
+
+      # Get all attributes
+      # @return [Hash] All attributes
+      def attributes
+        @attributes.dup
+      end
+
+      # Save the message
+      # @return [Boolean] Whether the save was successful
+      def save
+        raise NotImplementedError, "#{self.class.name}#save must be implemented by a framework adapter"
+      end
+
+      # Update the message attributes
+      # @param attributes [Hash] New attribute values
+      # @return [Boolean] Whether the update was successful
+      def update(attributes = {})
+        attributes.each do |key, value|
+          send("#{key}=", value) if respond_to?("#{key}=")
+        end
+        save
+      end
+
+      # Mark the message as published
+      # @return [Boolean] Whether the update was successful
+      def mark_as_published
+        self.status = "published"
+        self.published_at = Time.now
+        self.updated_at = Time.now
+        save
+      end
+
+      # Mark the message as failed
+      # @param error [Exception, String] Error that caused the failure
+      # @return [Boolean] Whether the update was successful
+      def mark_as_failed(error = nil)
+        self.status = "failed"
+        self.error_message = error.is_a?(Exception) ? "#{error.class}: #{error.message}" : error.to_s
+        self.updated_at = Time.now
+        save
+      end
+
+      # Increment the retry count and set the next retry time
+      # @return [Boolean] Whether the update was successful
+      def increment_retry_count
+        self.retry_count += 1
+        self.next_retry_at = calculate_next_retry_time
+        self.updated_at = Time.now
+        save
+      end
+
+      # Calculate the next retry time based on exponential backoff
+      # @return [Time] Next retry time
+      def calculate_next_retry_time
+        base_delay = Pigeon.config.retry_delay || 30 # 30 seconds default
+        max_delay = Pigeon.config.max_retry_delay || 86_400 # 24 hours default
+
+        # Exponential backoff: delay = base_delay * (2 ^ retry_count)
+        delay = base_delay * (2**retry_count)
+        delay = [delay, max_delay].min # Cap at max delay
+
+        Time.now + delay
+      end
+
+      # Check if the message has exceeded the maximum retry count
+      # @return [Boolean] Whether the message has exceeded the maximum retry count
+      def max_retries_exceeded?
+        max_retries = self[:max_retries] || Pigeon.config.max_retries || 10
+        retry_count >= max_retries
+      end
+
+      # Define attribute accessors for all attributes
+      ATTRIBUTES.each do |attr|
+        define_method(attr) { @attributes[attr] }
+        define_method("#{attr}=") { |value| @attributes[attr] = value }
+      end
+    end
+  end
+end