pigeon-rb 0.1.0

data/lib/pigeon/tasks/pigeon.rake

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+namespace :pigeon do # rubocop:disable Metrics/BlockLength
+  desc "Process pending outbox messages"
+  task process: :environment do
+    batch_size = ENV.fetch("BATCH_SIZE", 100).to_i
+    puts "Processing pending outbox messages (batch size: #{batch_size})..."
+    stats = Pigeon.processor.process_pending(batch_size: batch_size)
+    puts "Processed #{stats[:processed]} messages: #{stats[:succeeded]} succeeded, #{stats[:failed]} failed"
+  end
+
+  desc "Start background processing of outbox messages"
+  task start: :environment do
+    batch_size = ENV.fetch("BATCH_SIZE", 100).to_i
+    interval = ENV.fetch("INTERVAL", 5).to_i
+    thread_count = ENV.fetch("THREAD_COUNT", 2).to_i
+
+    puts "Starting background processing of outbox messages..."
+    puts "Batch size: #{batch_size}, Interval: #{interval}s, Threads: #{thread_count}"
+
+    processor = Pigeon.processor
+    if processor.start_processing(batch_size: batch_size, interval: interval, thread_count: thread_count)
+      puts "Background processing started successfully"
+
+      # Keep the process running
+      trap("INT") do
+        processor.stop_processing
+        puts "\nStopped processing"
+        exit
+      end
+      trap("TERM") do
+        processor.stop_processing
+        puts "\nStopped processing"
+        exit
+      end
+
+      # Sleep indefinitely
+      loop { sleep }
+    else
+      puts "Failed to start background processing"
+      exit 1
+    end
+  end
+
+  desc "Clean up processed outbox messages"
+  task cleanup: :environment do
+    older_than = ENV.fetch("OLDER_THAN", 7).to_i
+    puts "Cleaning up processed outbox messages older than #{older_than} days..."
+    count = Pigeon.processor.cleanup_processed(older_than: older_than)
+    puts "Cleaned up #{count} messages"
+  end
+
+  desc "Generate outbox message migration for Rails"
+  task :install, [:framework] do |_t, args|
+    if args[:framework] == "rails"
+      system "rails generate pigeon:rails:install"
+    elsif args[:framework] == "hanami"
+      system "hanami generate pigeon:rails:install"
+    else
+      puts "Invalid framework: #{args[:framework]}. Valid frameworks are: rails, hanami"
+      exit 1
+    end
+  end
+end

data/lib/pigeon/trace_api.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Pigeon
+  # Tracing API for Pigeon
+  module TraceAPI
+    # Initialize OpenTelemetry tracing
+    # @param service_name [String] Service name for traces
+    # @param exporter [OpenTelemetry::Exporter::OTLP::Exporter, nil] Optional custom exporter
+    # @return [Boolean] Whether initialization was successful
+    def self.init_tracing(service_name: "pigeon", exporter: nil)
+      Tracing.init(service_name: service_name, exporter: exporter)
+    end
+
+    # Check if OpenTelemetry tracing is available
+    # @return [Boolean] Whether OpenTelemetry is available
+    def self.tracing_available?
+      Tracing.available?
+    end
+
+    # Get the OpenTelemetry tracer
+    # @param name [String] Tracer name
+    # @return [OpenTelemetry::Tracer, nil] Tracer or nil if OpenTelemetry is not available
+    def self.tracer(name = "pigeon")
+      Tracing.tracer(name)
+    end
+
+    # Create a span for a block of code
+    # @param name [String] Span name
+    # @param attributes [Hash] Span attributes
+    # @param kind [Symbol] Span kind (:internal, :server, :client, :producer, :consumer)
+    # @yield Block to execute within the span
+    # @return [Object] Result of the block
+    def self.with_span(name, attributes: {}, kind: :internal, &)
+      Tracing.with_span(name, attributes: attributes, kind: kind, &)
+    end
+  end
+end

data/lib/pigeon/tracing/core.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Pigeon
+  module Tracing
+    # Core tracing functionality
+    module Core
+      # Check if OpenTelemetry is available
+      # @return [Boolean] Whether OpenTelemetry is available
+      def self.available?
+        defined?(OpenTelemetry)
+      end
+
+      # Initialize OpenTelemetry tracing
+      # @param service_name [String] Service name for traces
+      # @param exporter [OpenTelemetry::Exporter::OTLP::Exporter, nil] Optional custom exporter
+      # @return [Boolean] Whether initialization was successful
+      def self.init(service_name: "pigeon", exporter: nil)
+        return false unless available?
+
+        begin
+          # Configure the SDK with default exporters
+          OpenTelemetry::SDK.configure do |c|
+            c.service_name = service_name
+            c.use_all # Use all available instrumentation
+            c.add_span_processor(
+              OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+                exporter || OpenTelemetry::Exporter::OTLP::Exporter.new
+              )
+            )
+          end
+
+          true
+        rescue StandardError => e
+          Pigeon.config.logger.error("Failed to initialize OpenTelemetry: #{e.message}")
+          false
+        end
+      end
+
+      # Get the OpenTelemetry tracer
+      # @param name [String] Tracer name
+      # @return [OpenTelemetry::Tracer, nil] Tracer or nil if OpenTelemetry is not available
+      def self.tracer(name = "pigeon")
+        return nil unless available?
+
+        OpenTelemetry.tracer_provider.tracer(name)
+      end
+
+      # Create a span for a block of code
+      # @param name [String] Span name
+      # @param attributes [Hash] Span attributes
+      # @param kind [Symbol] Span kind (:internal, :server, :client, :producer, :consumer)
+      # @param parent_context [OpenTelemetry::Context, nil] Optional parent context
+      # @yield Block to execute within the span
+      # @return [Object] Result of the block
+      def self.with_span(name, attributes: {}, kind: :internal, parent_context: nil, &)
+        return yield unless available?
+
+        tracer = self.tracer
+        return yield unless tracer
+
+        parent_context ||= OpenTelemetry::Context.current
+        span_kind = span_kind_from_symbol(kind)
+
+        tracer.in_span(name, attributes: attributes, kind: span_kind, with_parent: parent_context, &)
+      end
+
+      # Extract trace context from headers
+      # @param headers [Hash] Headers containing trace context
+      # @return [OpenTelemetry::Context, nil] Extracted context or nil if not available
+      def self.extract_context(headers)
+        return nil unless available?
+        return nil unless headers
+
+        # Convert header keys to lowercase for case-insensitive matching
+        normalized_headers = {}
+        headers.each do |k, v|
+          normalized_headers[k.to_s.downcase] = v
+        end
+
+        carrier = OpenTelemetry::Context::Propagation::Rack::HeadersGetter.new(normalized_headers)
+        OpenTelemetry.propagation.extract(carrier)
+      end
+
+      # Inject trace context into headers
+      # @param headers [Hash] Headers to inject trace context into
+      # @param context [OpenTelemetry::Context, nil] Context to inject (defaults to current)
+      # @return [Hash] Headers with injected trace context
+      def self.inject_context(headers = {}, context = nil)
+        return headers unless available?
+
+        headers ||= {}
+        context ||= OpenTelemetry::Context.current
+        carrier = OpenTelemetry::Context::Propagation::Rack::HeadersSetter.new(headers)
+        OpenTelemetry.propagation.inject(carrier, context: context)
+        headers
+      end
+
+      # Convert a symbol to an OpenTelemetry span kind
+      # @param kind [Symbol] Span kind symbol
+      # @return [Integer] OpenTelemetry span kind
+      def self.span_kind_from_symbol(kind)
+        return OpenTelemetry::Trace::SpanKind::INTERNAL unless available?
+
+        case kind
+        when :server
+          OpenTelemetry::Trace::SpanKind::SERVER
+        when :client
+          OpenTelemetry::Trace::SpanKind::CLIENT
+        when :producer
+          OpenTelemetry::Trace::SpanKind::PRODUCER
+        when :consumer
+          OpenTelemetry::Trace::SpanKind::CONSUMER
+        else
+          OpenTelemetry::Trace::SpanKind::INTERNAL
+        end
+      end
+    end
+  end
+end

data/lib/pigeon/tracing/messaging.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module Pigeon
+  module Tracing
+    # Messaging-specific tracing functionality
+    module Messaging
+      # Create a span for publishing a message
+      # @param topic [String] Kafka topic
+      # @param payload [Hash, String] Message payload
+      # @param key [String, nil] Optional message key
+      # @param headers [Hash, nil] Optional message headers
+      # @param correlation_id [String, nil] Optional correlation ID
+      # @yield [span, context, headers] Block to execute within the span
+      # @yieldparam span [OpenTelemetry::Span] Active span
+      # @yieldparam context [OpenTelemetry::Context] Span context
+      # @yieldparam headers [Hash] Headers with injected trace context
+      # @return [Object] Result of the block
+      def self.trace_publish(topic:, payload:, key: nil, headers: nil, correlation_id: nil)
+        return yield(nil, nil, headers || {}) unless Core.available?
+
+        # Create span attributes
+        attributes = publish_span_attributes(topic, payload, key, correlation_id)
+
+        # Create the span
+        Core.with_span("publish_message", attributes: attributes, kind: :producer) do |span|
+          # Get the current context with the span
+          context = OpenTelemetry::Context.current
+
+          # Inject trace context into headers
+          trace_headers = headers ? headers.dup : {}
+          Core.inject_context(trace_headers, context)
+
+          # Add correlation ID to span
+          span.add_attributes("messaging.correlation_id" => correlation_id) if correlation_id
+
+          # Yield to the block with the span, context, and headers
+          yield span, context, trace_headers
+        end
+      end
+
+      # Create span attributes for publishing
+      # @param topic [String] Kafka topic
+      # @param payload [Hash, String] Message payload
+      # @param key [String, nil] Optional message key
+      # @param correlation_id [String, nil] Optional correlation ID
+      # @return [Hash] Span attributes
+      def self.publish_span_attributes(topic, payload, key, correlation_id)
+        attributes = {
+          "messaging.system" => "kafka",
+          "messaging.destination" => topic,
+          "messaging.destination_kind" => "topic",
+          "messaging.operation" => "publish"
+        }
+
+        # Add optional attributes
+        attributes["messaging.message_id"] = correlation_id if correlation_id
+        attributes["messaging.kafka.key"] = key if key
+        attributes["messaging.message_payload_size_bytes"] = payload.bytesize if payload.respond_to?(:bytesize)
+
+        attributes
+      end
+
+      # Create a span for processing a message
+      # @param message [Pigeon::Models::OutboxMessage] Message to process
+      # @yield [span] Block to execute within the span
+      # @yieldparam span [OpenTelemetry::Span] Active span
+      # @return [Object] Result of the block
+      def self.trace_process(message)
+        return yield(nil) unless Core.available?
+
+        # Extract trace context from message headers if available
+        parent_context = message.headers ? Core.extract_context(message.headers) : nil
+
+        # Create span attributes
+        attributes = process_span_attributes(message)
+
+        # Create the span
+        Core.with_span(
+          "process_message",
+          attributes: attributes,
+          kind: :consumer,
+          parent_context: parent_context
+        ) do |span|
+          # Add additional attributes
+          add_process_span_attributes(span, message)
+
+          # Yield to the block with the span
+          yield span
+        end
+      end
+
+      # Create span attributes for processing
+      # @param message [Pigeon::Models::OutboxMessage] Message to process
+      # @return [Hash] Span attributes
+      def self.process_span_attributes(message)
+        attributes = {
+          "messaging.system" => "kafka",
+          "messaging.destination" => message.topic,
+          "messaging.destination_kind" => "topic",
+          "messaging.operation" => "process",
+          "messaging.kafka.key" => message.key,
+          "messaging.message_id" => message.id.to_s
+        }
+
+        # Add correlation ID if available
+        attributes["messaging.correlation_id"] = message.correlation_id if message.correlation_id
+
+        attributes
+      end
+
+      # Add additional attributes to a process span
+      # @param span [OpenTelemetry::Span] Span to add attributes to
+      # @param message [Pigeon::Models::OutboxMessage] Message being processed
+      # @return [void]
+      def self.add_process_span_attributes(span, message)
+        return unless span && message.partition
+
+        span.add_attributes({
+                              "messaging.kafka.partition" => message.partition,
+                              "messaging.kafka.message.retry_count" => message.retry_count
+                            })
+      end
+
+      # Create a span for batch processing
+      # @param batch_size [Integer] Number of messages to process
+      # @yield [span] Block to execute within the span
+      # @yieldparam span [OpenTelemetry::Span] Active span
+      # @return [Object] Result of the block
+      def self.trace_batch_process(batch_size, &)
+        return yield(nil) unless Core.available?
+
+        # Create span attributes
+        attributes = {
+          "messaging.system" => "kafka",
+          "messaging.operation" => "batch_process",
+          "messaging.batch.size" => batch_size
+        }
+
+        # Create the span
+        Core.with_span("process_batch", attributes: attributes, &)
+      end
+    end
+  end
+end

data/lib/pigeon/tracing.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+begin
+  require "opentelemetry"
+  require "opentelemetry/sdk"
+  require "opentelemetry/exporter/otlp"
+  require "opentelemetry/instrumentation/all"
+rescue LoadError
+  # OpenTelemetry is optional, so we can continue without it
+end
+
+require_relative "tracing/core"
+require_relative "tracing/messaging"
+
+module Pigeon
+  # Tracing module for Pigeon using OpenTelemetry
+  module Tracing
+    class << self
+      # Check if OpenTelemetry is available
+      # @return [Boolean] Whether OpenTelemetry is available
+      def available?
+        Core.available?
+      end
+
+      # Initialize OpenTelemetry tracing
+      # @param service_name [String] Service name for traces
+      # @param exporter [OpenTelemetry::Exporter::OTLP::Exporter, nil] Optional custom exporter
+      # @return [Boolean] Whether initialization was successful
+      def init(service_name: "pigeon", exporter: nil)
+        Core.init(service_name: service_name, exporter: exporter)
+      end
+
+      # Get the OpenTelemetry tracer
+      # @param name [String] Tracer name
+      # @return [OpenTelemetry::Tracer, nil] Tracer or nil if OpenTelemetry is not available
+      def tracer(name = "pigeon")
+        Core.tracer(name)
+      end
+
+      # Create a span for a block of code
+      # @param name [String] Span name
+      # @param attributes [Hash] Span attributes
+      # @param kind [Symbol] Span kind (:internal, :server, :client, :producer, :consumer)
+      # @param parent_context [OpenTelemetry::Context, nil] Optional parent context
+      # @yield Block to execute within the span
+      # @return [Object] Result of the block
+      def with_span(name, attributes: {}, kind: :internal, parent_context: nil, &)
+        Core.with_span(name, attributes: attributes, kind: kind, parent_context: parent_context, &)
+      end
+
+      # Extract trace context from headers
+      # @param headers [Hash] Headers containing trace context
+      # @return [OpenTelemetry::Context, nil] Extracted context or nil if not available
+      def extract_context(headers)
+        Core.extract_context(headers)
+      end
+
+      # Inject trace context into headers
+      # @param headers [Hash] Headers to inject trace context into
+      # @param context [OpenTelemetry::Context, nil] Context to inject (defaults to current)
+      # @return [Hash] Headers with injected trace context
+      def inject_context(headers = {}, context = nil)
+        Core.inject_context(headers, context)
+      end
+
+      # Create a span for publishing a message
+      # @param topic [String] Kafka topic
+      # @param payload [Hash, String] Message payload
+      # @param key [String, nil] Optional message key
+      # @param headers [Hash, nil] Optional message headers
+      # @param correlation_id [String, nil] Optional correlation ID
+      # @yield [span, context, headers] Block to execute within the span
+      # @yieldparam span [OpenTelemetry::Span] Active span
+      # @yieldparam context [OpenTelemetry::Context] Span context
+      # @yieldparam headers [Hash] Headers with injected trace context
+      # @return [Object] Result of the block
+      def trace_publish(topic:, payload:, key: nil, headers: nil, correlation_id: nil, &)
+        Messaging.trace_publish(
+          topic: topic,
+          payload: payload,
+          key: key,
+          headers: headers,
+          correlation_id: correlation_id,
+          &
+        )
+      end
+
+      # Create a span for processing a message
+      # @param message [Pigeon::Models::OutboxMessage] Message to process
+      # @yield [span] Block to execute within the span
+      # @yieldparam span [OpenTelemetry::Span] Active span
+      # @return [Object] Result of the block
+      def trace_process(message, &)
+        Messaging.trace_process(message, &)
+      end
+
+      # Create a span for batch processing
+      # @param batch_size [Integer] Number of messages to process
+      # @yield [span] Block to execute within the span
+      # @yieldparam span [OpenTelemetry::Span] Active span
+      # @return [Object] Result of the block
+      def trace_batch_process(batch_size, &)
+        Messaging.trace_batch_process(batch_size, &)
+      end
+    end
+  end
+end

data/lib/pigeon/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Pigeon
+  VERSION = "0.1.0"
+end

data/lib/pigeon.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "pigeon/version"
+require "pigeon/logging/structured_logger"
+require "pigeon/configuration"
+require "pigeon/models/outbox_message"
+require "pigeon/models/adapters/active_record_adapter"
+require "pigeon/models/adapters/rom_adapter"
+require "pigeon/serializer"
+require "pigeon/encryption"
+require "pigeon/tracing"
+require "pigeon/publisher"
+require "pigeon/processor"
+require "pigeon/health_check"
+require "karafka"
+
+# Load Rails integration if Rails is defined
+require "pigeon/railtie" if defined?(Rails)
+
+# Load Hanami integration if Hanami is defined
+require "pigeon/hanami_integration" if defined?(Hanami)
+
+# Load generators if Rails is defined
+if defined?(Rails)
+  require "pigeon/generators/rails/migration_generator"
+  require "pigeon/generators/rails/install_generator"
+end
+
+# Load internal modules
+require_relative "pigeon/mock_producer"
+require_relative "pigeon/core"
+require_relative "pigeon/schema"
+require_relative "pigeon/security"
+require_relative "pigeon/outbox"
+require_relative "pigeon/monitoring"
+require_relative "pigeon/trace_api"
+require_relative "pigeon/api"
+
+# Main module for the Pigeon gem
+module Pigeon
+  class Error < StandardError; end
+
+  # Include API modules
+  class << self
+    include API::CoreAPI
+    include API::SchemaAPI
+    include API::SecurityAPI
+    include API::OutboxAPI
+    include API::MonitoringAPI
+    include API::TracingAPI
+  end
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: pigeon-rb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Khai Le
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-07-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: dry-configurable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: karafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+description: A Ruby gem that implements the outbox pattern for Kafka message publishing
+  to ensure message durability and delivery reliability
+email:
+- khaile.to@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/pigeon.rb
+- lib/pigeon/active_job_integration.rb
+- lib/pigeon/api.rb
+- lib/pigeon/configuration.rb
+- lib/pigeon/core.rb
+- lib/pigeon/encryption.rb
+- lib/pigeon/generators/hanami/migration_generator.rb
+- lib/pigeon/generators/rails/install_generator.rb
+- lib/pigeon/generators/rails/migration_generator.rb
+- lib/pigeon/generators/rails/templates/create_outbox_messages.rb.erb
+- lib/pigeon/generators/rails/templates/initializer.rb.erb
+- lib/pigeon/hanami_integration.rb
+- lib/pigeon/health_check.rb
+- lib/pigeon/health_check/kafka.rb
+- lib/pigeon/health_check/processor.rb
+- lib/pigeon/health_check/queue.rb
+- lib/pigeon/logging/structured_logger.rb
+- lib/pigeon/metrics/collector.rb
+- lib/pigeon/mock_producer.rb
+- lib/pigeon/models/adapters/active_record_adapter.rb
+- lib/pigeon/models/adapters/rom_adapter.rb
+- lib/pigeon/models/outbox_message.rb
+- lib/pigeon/monitoring.rb
+- lib/pigeon/outbox.rb
+- lib/pigeon/processor.rb
+- lib/pigeon/processor/background_processor.rb
+- lib/pigeon/publisher.rb
+- lib/pigeon/railtie.rb
+- lib/pigeon/schema.rb
+- lib/pigeon/security.rb
+- lib/pigeon/serializer.rb
+- lib/pigeon/tasks/pigeon.rake
+- lib/pigeon/trace_api.rb
+- lib/pigeon/tracing.rb
+- lib/pigeon/tracing/core.rb
+- lib/pigeon/tracing/messaging.rb
+- lib/pigeon/version.rb
+homepage: https://github.com/khaile/pigeon
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/khaile/pigeon
+  changelog_uri: https://github.com/khaile/pigeon/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'false'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.8
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Kafka outbox pattern implementation for Ruby applications
+test_files: []