pigeon-rb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1232e3eec460a373578ba9c8add7e6ae10d6c9db29e6b6cf017cbe019933b06c
+  data.tar.gz: 5427465799048a3ef0e9b4e1f72afdbf1001960f217fbd919cb6111a524bcbb6
+SHA512:
+  metadata.gz: 4c8ae8ca31274ac1327c940d2c6b9d17254572e6a2b4ff8ccdce9135663e2ec809b690d2190b6dd742ca2f488bd079d3b793365d0491af67354b4e96a4244ed1
+  data.tar.gz: 0e0c5c09c14610f688b62612d9dd6a1e920aed952594003e8cb998c8e416fefb57002e54d93dc08e2530eb26ffbb2ae2a0207e44270091d7bc6726240ac2e58d

data/README.md

@@ -0,0 +1,343 @@
+# Pigeon
+
+A robust Ruby gem that implements the outbox pattern for Kafka message publishing, ensuring reliable message delivery with enterprise features like encryption, tracing, monitoring, and framework integrations.
+
+## Features
+
+- **Outbox Pattern**: Reliable message delivery with database-backed persistence
+- **Encryption**: AES-256-GCM encryption for sensitive payloads
+- **Tracing**: OpenTelemetry integration for distributed tracing
+- **Monitoring**: Comprehensive metrics and health checks
+- **Framework Integration**: Rails and Hanami support with generators
+- **ActiveJob Integration**: Background job processing for message handling
+- **Security**: Payload masking and sensitive data protection
+- **Health Checks**: Processor, queue, and Kafka connectivity monitoring
+- **Structured Logging**: Context-aware logging with correlation IDs
+- **Schema Validation**: Message schema registration and validation
+- **Retry Logic**: Configurable retry policies with exponential backoff
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pigeon'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install pigeon
+```
+
+## Quick Start
+
+### Rails Integration
+
+Install Pigeon in your Rails application:
+
+```bash
+$ rake pigeon:install[rails]
+```
+
+This will create:
+- Configuration file at `config/initializers/pigeon.rb`
+- Database migration for the outbox messages table
+
+Run the migration:
+
+```bash
+$ rails db:migrate
+```
+
+### Hanami Integration
+
+For Hanami applications, generate the migration:
+
+```bash
+$ rake pigeon:install[hanami]
+```
+
+## Configuration
+
+Configure the gem with your Kafka and application details:
+
+```ruby
+Pigeon.configure do |config|
+  # Basic configuration
+  config.client_id = "my-application"
+  config.kafka_brokers = ["kafka1:9092", "kafka2:9092"]
+
+  # Retry configuration
+  config.max_retries = 5
+  config.retry_delay = 60 # seconds
+  config.max_retry_delay = 3600 # 1 hour
+
+  # Encryption (optional)
+  config.encrypt_payload = false
+  config.encryption_key = ENV["PIGEON_ENCRYPTION_KEY"]
+
+  # Retention and cleanup
+  config.retention_period = 7 # days
+
+  # Karafka-specific configuration
+  config.karafka_config = {
+    delivery: :async,
+    kafka: {
+      'bootstrap.servers': 'kafka1:9092,kafka2:9092',
+      'request.required.acks': 1
+    }
+  }
+end
+```
+
+### Environment Variables
+
+Pigeon supports configuration via environment variables:
+
+```bash
+PIGEON_CLIENT_ID=my-app
+PIGEON_KAFKA_BROKERS=kafka1:9092,kafka2:9092
+PIGEON_ENCRYPTION_KEY=your-32-byte-encryption-key
+PIGEON_MAX_RETRIES=5
+PIGEON_RETRY_DELAY=60
+```
+
+## Usage
+
+### Publishing Messages
+
+```ruby
+# Simple publishing
+Pigeon.publisher.publish(
+  topic: "user-events",
+  payload: { user_id: 123, action: "signup" }
+)
+
+# With encryption and sensitive field masking
+Pigeon.publisher.publish(
+  topic: "user-events",
+  payload: { user_id: 123, email: "user@example.com", password: "secret" },
+  key: "user-123",
+  headers: { "source" => "web-app" },
+  encrypt: true,
+  sensitive_fields: [:password, :credit_card]
+)
+
+# Synchronous publishing (immediate attempt)
+Pigeon.publisher.publish(
+  topic: "critical-events",
+  payload: { event: "payment_processed" },
+  sync: true
+)
+```
+
+### Processing Messages
+
+```ruby
+# Process pending messages
+stats = Pigeon.processor.process_pending(batch_size: 100)
+puts "Processed: #{stats[:processed]}, Succeeded: #{stats[:succeeded]}, Failed: #{stats[:failed]}"
+
+# Process a specific message
+success = Pigeon.processor.process_message("message-id")
+
+# Clean up old processed messages
+cleaned = Pigeon.processor.cleanup_processed(older_than: 14) # days
+puts "Cleaned up #{cleaned} messages"
+
+# Start background processing
+Pigeon.start_processing(batch_size: 100, interval: 5, thread_count: 2)
+
+# Stop background processing
+Pigeon.stop_processing
+```
+
+### Encryption
+
+```ruby
+# Enable encryption in configuration
+Pigeon.configure do |config|
+  config.encrypt_payload = true
+  config.encryption_key = "your-32-byte-encryption-key"
+end
+
+# Encrypt a payload
+encrypted = Pigeon.encrypt(payload.to_json)
+decrypted = Pigeon.decrypt(encrypted)
+
+# Mask sensitive fields for logging
+masked = Pigeon.mask_payload(payload, [:password, :credit_card])
+```
+
+### Tracing
+
+```ruby
+# Initialize OpenTelemetry tracing
+Pigeon.init_tracing(service_name: "my-app")
+
+# Trace message publishing
+Pigeon.with_span("publish_message", attributes: { topic: "user-events" }) do
+  Pigeon.publisher.publish(topic: "user-events", payload: data)
+end
+
+# Trace message processing
+Pigeon.trace_process(message) do |span|
+  # Process the message
+  span.set_attribute("message.id", message.id)
+end
+```
+
+### Health Checks
+
+```ruby
+# Check overall health
+health = Pigeon.health_status
+puts "Status: #{health[:status]}"
+
+# Check individual components
+processor_health = Pigeon.processor_health
+queue_health = Pigeon.queue_health
+kafka_health = Pigeon.kafka_health
+```
+
+### Monitoring
+
+```ruby
+# Get metrics
+metrics = Pigeon.metrics
+puts "Messages published: #{metrics[:messages_published]}"
+puts "Messages processed: #{metrics[:messages_processed]}"
+
+# Get monitoring-friendly metrics
+monitoring_metrics = Pigeon.metrics_for_monitoring
+
+# Reset metrics
+Pigeon.reset_metrics
+```
+
+### Schema Validation
+
+```ruby
+# Register a schema
+Pigeon.register_schema("user_event", {
+  type: "object",
+  properties: {
+    user_id: { type: "integer" },
+    action: { type: "string" }
+  },
+  required: ["user_id", "action"]
+})
+
+# Register sensitive fields
+Pigeon.register_sensitive_fields([:password, :credit_card, :ssn])
+```
+
+### ActiveJob Integration
+
+```ruby
+# Process messages in background
+Pigeon::ActiveJobIntegration::ProcessorJob.perform_later(100)
+
+# Clean up old messages in background
+Pigeon::ActiveJobIntegration::CleanupJob.perform_later(7)
+```
+
+## API Reference
+
+### Core API
+
+```ruby
+# Configuration
+Pigeon.configure(&block)
+Pigeon.config
+
+# Publishing
+Pigeon.publisher.publish(topic:, payload:, **options)
+
+# Processing
+Pigeon.processor.process_pending(batch_size: 100)
+Pigeon.processor.process_message(id)
+Pigeon.processor.cleanup_processed(older_than: 7)
+
+# Background processing
+Pigeon.start_processing(batch_size: 100, interval: 5, thread_count: 2)
+Pigeon.stop_processing
+Pigeon.processing?
+```
+
+### Security API
+
+```ruby
+# Encryption
+Pigeon.encrypt(payload, encryption_key = nil)
+Pigeon.decrypt(encrypted_payload, encryption_key = nil)
+Pigeon.mask_payload(payload, sensitive_fields = nil)
+```
+
+### Monitoring API
+
+```ruby
+# Metrics
+Pigeon.metrics
+Pigeon.metrics_for_monitoring
+Pigeon.reset_metrics
+
+# Health checks
+Pigeon.health_status
+Pigeon.processor_health
+Pigeon.queue_health
+Pigeon.kafka_health
+
+# Logging
+Pigeon.logger_with_context(context = {})
+Pigeon.log_level = :info
+```
+
+### Tracing API
+
+```ruby
+# Tracing setup
+Pigeon.init_tracing(service_name: "my-app", exporter: nil)
+Pigeon.tracing_available?
+
+# Spans
+Pigeon.with_span(name, attributes: {}, kind: :internal, &block)
+Pigeon.tracer(name = "pigeon")
+```
+
+### Outbox API
+
+```ruby
+# Message management
+Pigeon.create_outbox_message(attributes = {})
+Pigeon.find_outbox_message(id)
+Pigeon.find_outbox_messages_by_status(status, limit = 100)
+Pigeon.find_outbox_messages_ready_for_retry(limit = 100)
+Pigeon.count_outbox_messages_by_status(status)
+Pigeon.find_oldest_outbox_message_by_status(status)
+```
+
+## Examples
+
+See the `examples/` directory for complete working examples:
+
+- `encryption_example.rb` - Demonstrates AES-256-GCM encryption and payload masking
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/pigeon/active_job_integration.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Pigeon
+  # ActiveJob integration for Pigeon
+  module ActiveJobIntegration
+    # Job for processing pending outbox messages
+    class ProcessorJob < ActiveJob::Base
+      queue_as :default
+
+      # Process pending outbox messages
+      # @param batch_size [Integer] Number of messages to process in one batch
+      # @return [Hash] Processing statistics
+      def perform(batch_size = 100)
+        Pigeon.processor.process_pending(batch_size: batch_size)
+      end
+    end
+
+    # Job for cleaning up processed outbox messages
+    class CleanupJob < ActiveJob::Base
+      queue_as :pigeon_cleanup
+
+      # Clean up processed outbox messages
+      # @param older_than [Integer] Age threshold for cleanup in days
+      # @return [Integer] Number of records cleaned up
+      def perform(older_than = 7)
+        Pigeon.processor.cleanup_processed(older_than: older_than)
+      end
+    end
+  end
+end

data/lib/pigeon/api.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Pigeon
+  # API methods for the Pigeon module
+  module API
+    # Delegate methods to Core module
+    module CoreAPI
+      def configure(&)
+        Core.configure(&)
+      end
+
+      def config
+        Core.config
+      end
+
+      def initialize_karafka
+        Core.initialize_karafka
+      end
+
+      def karafka_producer
+        Core.karafka_producer
+      end
+
+      def publisher
+        Core.publisher
+      end
+
+      def processor(auto_start: false)
+        Core.processor(auto_start: auto_start)
+      end
+
+      def start_processing(batch_size: 100, interval: 5, thread_count: 2)
+        Core.start_processing(batch_size: batch_size, interval: interval, thread_count: thread_count)
+      end
+
+      def stop_processing
+        Core.stop_processing
+      end
+
+      def processing?
+        Core.processing?
+      end
+    end
+
+    # Delegate methods to Schema module
+    module SchemaAPI
+      def register_schema(name, schema)
+        Schema.register_schema(name, schema)
+      end
+
+      def schema(name)
+        Schema.schema(name)
+      end
+
+      def register_sensitive_field(field)
+        Schema.register_sensitive_field(field)
+      end
+
+      def register_sensitive_fields(fields)
+        Schema.register_sensitive_fields(fields)
+      end
+    end
+
+    # Delegate methods to Security module
+    module SecurityAPI
+      def encrypt(payload, encryption_key = nil)
+        Security.encrypt(payload, encryption_key)
+      end
+
+      def decrypt(encrypted_payload, encryption_key = nil)
+        Security.decrypt(encrypted_payload, encryption_key)
+      end
+
+      def mask_payload(payload, sensitive_fields = nil)
+        Security.mask_payload(payload, sensitive_fields)
+      end
+    end
+
+    # Delegate methods to Outbox module
+    module OutboxAPI
+      def outbox_message_adapter
+        Outbox.outbox_message_adapter
+      end
+
+      def create_outbox_message(attributes = {})
+        Outbox.create_outbox_message(attributes)
+      end
+
+      def find_outbox_message(id)
+        Outbox.find_outbox_message(id)
+      end
+
+      def find_outbox_messages_by_status(status, limit = 100)
+        Outbox.find_outbox_messages_by_status(status, limit)
+      end
+
+      def find_outbox_messages_ready_for_retry(limit = 100)
+        Outbox.find_outbox_messages_ready_for_retry(limit)
+      end
+
+      def count_outbox_messages_by_status(status)
+        Outbox.count_outbox_messages_by_status(status)
+      end
+
+      def find_oldest_outbox_message_by_status(status)
+        Outbox.find_oldest_outbox_message_by_status(status)
+      end
+    end
+
+    # Delegate methods to Monitoring module
+    module MonitoringAPI
+      def metrics_collector
+        Monitoring.metrics_collector
+      end
+
+      def metrics
+        Monitoring.metrics
+      end
+
+      def metrics_for_monitoring
+        Monitoring.metrics_for_monitoring
+      end
+
+      def reset_metrics
+        Monitoring.reset_metrics
+      end
+
+      def logger_with_context(context = {})
+        Monitoring.logger_with_context(context)
+      end
+
+      def log_level=(level)
+        Monitoring.log_level = level
+      end
+
+      def log_level
+        Monitoring.log_level
+      end
+
+      def processor_start_time
+        Monitoring.processor_start_time
+      end
+
+      def last_processing_run
+        Monitoring.last_processing_run
+      end
+
+      def last_successful_processing_run
+        Monitoring.last_successful_processing_run
+      end
+
+      def processor_start_time=(time)
+        Monitoring.processor_start_time = time
+      end
+
+      def last_processing_run=(time)
+        Monitoring.last_processing_run = time
+      end
+
+      def last_successful_processing_run=(time)
+        Monitoring.last_successful_processing_run = time
+      end
+
+      def processor_health
+        Monitoring.processor_health
+      end
+
+      def queue_health
+        Monitoring.queue_health
+      end
+
+      def kafka_health
+        Monitoring.kafka_health
+      end
+
+      def health_status
+        Monitoring.health_status
+      end
+    end
+
+    # Delegate methods to TraceAPI module
+    module TracingAPI
+      def init_tracing(service_name: "pigeon", exporter: nil)
+        TraceAPI.init_tracing(service_name: service_name, exporter: exporter)
+      end
+
+      def tracing_available?
+        TraceAPI.tracing_available?
+      end
+
+      def tracer(name = "pigeon")
+        TraceAPI.tracer(name)
+      end
+
+      def with_span(name, attributes: {}, kind: :internal, &)
+        TraceAPI.with_span(name, attributes: attributes, kind: kind, &)
+      end
+    end
+  end
+end