sidekiq_sqs_processor 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 840d04b0ed53863a1d63d982a96adb4b44534151e2052dd508b6639e7a875f1d
+  data.tar.gz: 7392d9426507e8d20f9c932bedd1291ce86299a4606442602fa18b17c4ece62d
+SHA512:
+  metadata.gz: 811fce502fecb1e1fed02d8e26f9836e3059091c583370f3864739df3509308df347a07b64c2be1d66ef63923f155531e7ab0e8cbdf9a68ab679171e09eb8816
+  data.tar.gz: 59cd2987302ed96f7004586b9a2cdc82c2669b7fce0b2fcefd5d47f2edcc4a83bb0007a5a177988370312515d5b198617601cda50d026f3f0f414f0b2aeaef52

data/README.md

@@ -0,0 +1,315 @@
+# Sidekiq SQS Processor
+
+[![Gem Version](https://badge.fury.io/rb/sidekiq_sqs_processor.svg)](https://badge.fury.io/rb/sidekiq_sqs_processor)
+[![Build Status](https://github.com/yourusername/sidekiq_sqs_processor/workflows/CI/badge.svg)](https://github.com/yourusername/sidekiq_sqs_processor/actions)
+
+A Ruby gem that seamlessly integrates Amazon SQS with Sidekiq for efficient and scalable message processing. Process SQS messages asynchronously with the power and monitoring capabilities of Sidekiq.
+
+## Features
+
+- **Automatic SQS polling** - Continuously polls SQS queues for new messages
+- **Parallel processing** - Process messages across multiple Sidekiq workers
+- **SNS integration** - Auto-unwraps SNS notifications
+- **Configurable polling strategies** - Choose between continuous or scheduled polling
+- **Automatic retries** - Leverage Sidekiq's retry mechanism with SQS visibility timeout
+- **Easy message routing** - Route messages to different workers based on content
+- **Rails integration** - Simple setup with generators for Rails applications
+- **Comprehensive monitoring** - Track processing stats and worker health
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq_sqs_processor'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself:
+
+```bash
+$ gem install sidekiq_sqs_processor
+```
+
+## Dependencies
+
+This gem requires the following dependencies in your application:
+
+- Sidekiq (~> 7.0)
+- aws-sdk-sqs (~> 1.0)
+- Rails (>= 6.0)
+
+### Rails Setup
+
+If you're using Rails, run the built-in generator to create the initializer and example worker:
+
+```bash
+$ rails generate sidekiq_sqs_processor:install
+```
+
+## Basic Configuration
+
+### Rails Applications
+
+After running the installer, edit the generated initializer at `config/initializers/sidekiq_sqs_processor.rb`:
+
+```ruby
+SidekiqSqsProcessor.configure do |config|
+  # AWS region for SQS queues
+  config.aws_region = 'us-east-1'
+
+  # SQS queue URLs to poll
+  config.queue_urls = [
+    'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue'
+  ]
+  
+  # Polling configuration
+  config.polling_type = :continuous
+end
+```
+
+### Non-Rails Applications
+
+For non-Rails applications, add this to your initialization code:
+
+```ruby
+require 'sidekiq_sqs_processor'
+
+SidekiqSqsProcessor.configure do |config|
+  # AWS region for SQS queues
+  config.aws_region = 'us-east-1'
+
+  # SQS queue URLs to poll
+  config.queue_urls = [
+    'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue'
+  ]
+  
+  # Start polling when Sidekiq starts
+  Sidekiq.configure_server do |sidekiq_config|
+    sidekiq_config.on(:startup) do
+      SidekiqSqsProcessor.start_continuous_poller
+    end
+    
+    sidekiq_config.on(:shutdown) do
+      SidekiqSqsProcessor.stop_continuous_poller
+    end
+  end
+end
+```
+
+## Creating Workers
+
+### Using the Generator (Rails)
+
+```bash
+$ rails generate sidekiq_sqs_processor:worker UserNotification
+```
+
+This will create:
+- `app/workers/user_notification_worker.rb`
+- `spec/workers/user_notification_worker_spec.rb` (if using RSpec)
+- or `test/workers/user_notification_worker_test.rb` (if using Test::Unit)
+
+### Manual Worker Creation
+
+Create a class that inherits from `SidekiqSqsProcessor::BaseWorker`:
+
+```ruby
+class MyWorker < SidekiqSqsProcessor::BaseWorker
+  # Override the process_message method to handle your SQS messages
+  def process_message(message_body)
+    # message_body is already parsed from JSON if it was a JSON string
+    
+    # Your processing logic here
+    User.find_by(id: message_body['user_id'])&.notify(message_body['message'])
+    
+    # Return a result (optional)
+    { status: 'success' }
+  end
+end
+```
+
+## Usage Examples
+
+### Basic Message Processing
+
+```ruby
+class OrderProcessor < SidekiqSqsProcessor::BaseWorker
+  def process_message(message_body)
+    if message_body.is_a?(Hash) && message_body['order_id']
+      # Process an order
+      process_order(message_body['order_id'], message_body['items'])
+    else
+      logger.error("Invalid order message format: #{message_body.inspect}")
+    end
+  end
+  
+  private
+  
+  def process_order(order_id, items)
+    # Your order processing logic
+    Order.process(order_id, items)
+  end
+end
+```
+
+### Handling Different Message Types
+
+```ruby
+class EventProcessor < SidekiqSqsProcessor::BaseWorker
+  def process_message(message_body)
+    case message_body['event_type']
+    when 'user_created'
+      create_user(message_body['data'])
+    when 'order_placed'
+      process_order(message_body['data'])
+    when 'payment_received'
+      process_payment(message_body['data'])
+    else
+      logger.warn("Unknown event type: #{message_body['event_type']}")
+    end
+  end
+  
+  # ... processing methods ...
+end
+```
+
+### Handling SNS Messages
+
+SNS messages are automatically detected and unwrapped, so the `message_body` will contain the inner message content:
+
+```ruby
+class NotificationProcessor < SidekiqSqsProcessor::BaseWorker
+  def process_message(message_body)
+    # For an SNS message, the SNS envelope has been removed
+    # and message_body is the parsed content of the SNS Message field
+    
+    logger.info("Processing notification: #{message_body.inspect}")
+    # ... your processing logic ...
+  end
+end
+```
+
+### Direct Message Enqueueing
+
+You can also enqueue messages directly to a worker without going through SQS:
+
+```ruby
+# Enqueue a message to a specific worker
+SidekiqSqsProcessor.enqueue_message(
+  MyWorker, 
+  { order_id: 123, items: ['item1', 'item2'] }
+)
+```
+
+## Configuration Options
+
+### AWS Configuration
+
+```ruby
+config.aws_region = 'us-east-1'              # AWS region
+config.aws_access_key_id = 'YOUR_KEY'        # Optional - uses env vars by default
+config.aws_secret_access_key = 'YOUR_SECRET' # Optional - uses env vars by default
+```
+
+### SQS Configuration
+
+```ruby
+config.queue_urls = ['https://sqs.region.amazonaws.com/account/queue']
+config.visibility_timeout = 300    # 5 minutes
+config.wait_time_seconds = 20      # 20 seconds (long polling)
+config.max_number_of_messages = 10 # Max messages per receive call
+```
+
+### Polling Configuration
+
+```ruby
+config.polling_type = :continuous  # :continuous or :scheduled
+config.polling_frequency = 60      # Only used for scheduled (seconds)
+config.polling_enabled = true      # Enable/disable polling
+config.poll_on_startup = true      # Start polling when app starts
+```
+
+### Sidekiq Configuration
+
+```ruby
+config.worker_queue_name = 'sqs_default' # Default Sidekiq queue
+config.worker_retry_count = 5            # Default retry count
+```
+
+### Error Handling
+
+```ruby
+# Custom error handler
+config.error_handler = ->(error, context) do
+  # Send to your error tracking service
+  Sentry.capture_exception(error, extra: context)
+end
+```
+
+## Best Practices
+
+### Message Structure
+
+For best routing and processing, use structured JSON messages:
+
+```json
+{
+  "type": "order_placed",
+  "data": {
+    "order_id": "12345",
+    "customer_id": "67890",
+    "items": [
+      {"id": "item1", "quantity": 2},
+      {"id": "item2", "quantity": 1}
+    ]
+  },
+  "metadata": {
+    "source": "web",
+    "timestamp": "2025-04-21T10:15:30Z"
+  }
+}
+```
+
+### Worker Organization
+
+- Create separate workers for different message types or domains
+- Use class inheritance for common processing logic
+- Keep workers small and focused
+
+### Error Handling
+
+- Use the `error_handler` config option for global error reporting
+- Implement custom error handling in your workers for specific cases
+- Let Sidekiq handle retries for transient failures
+
+### Visibility Timeout
+
+- Set your SQS visibility timeout longer than your Sidekiq job timeout
+- A good rule of thumb: visibility_timeout = (average_processing_time * 5) + max_retry_delay
+
+### Message Size Limits
+
+Remember that SQS has a 256KB message size limit. For larger data:
+- Store the data externally (e.g., S3) and include a reference in the message
+- Split large datasets across multiple messages
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/yourusername/sidekiq_sqs_processor. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/yourusername/sidekiq_sqs_processor/blob/master/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+

data/lib/generators/sidekiq_sqs_processor/install_generator.rb

@@ -0,0 +1,71 @@
+require 'rails/generators/base'
+
+module SidekiqSqsProcessor
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path('templates', __dir__)
+      
+      desc "Creates a SidekiqSqsProcessor initializer and configuration for your Rails application."
+      
+      class_option :queue_urls, type: :array, default: [],
+                   desc: "List of SQS queue URLs to process"
+      
+      class_option :continuous, type: :boolean, default: true,
+                   desc: "Use continuous polling (true) or scheduled polling (false)"
+                   
+      class_option :queue_name, type: :string, default: "default",
+                   desc: "Default Sidekiq queue name for the worker"
+                   
+      class_option :aws_region, type: :string, default: 'us-east-1',
+                   desc: "AWS region for SQS queues"
+      
+      def create_initializer_file
+        template "initializer.rb.tt", "config/initializers/sidekiq_sqs_processor.rb"
+      end
+      
+      def create_example_worker
+        template "worker.rb.tt", "app/workers/example_sqs_worker.rb"
+      end
+      
+      def show_readme
+        readme "README.txt" if behavior == :invoke
+      end
+      
+      private
+      
+      def continuous_polling?
+        options[:continuous]
+      end
+      
+      def polling_type
+        continuous_polling? ? ':continuous' : ':scheduled'
+      end
+      
+      def queue_urls
+        urls = options[:queue_urls]
+        if urls.empty?
+          "[] # Add your queue URLs here, e.g. ['https://sqs.us-east-1.amazonaws.com/123456789012/my-queue']"
+        else
+          urls.inspect
+        end
+      end
+      
+      def aws_region
+        options[:aws_region]
+      end
+      
+      def worker_class_name
+        "ExampleSqsWorker"
+      end
+      
+      def queue_name
+        options[:queue_name]
+      end
+      
+      def retry_option
+        ", retry: 2" # Default Sidekiq retry behavior
+      end
+    end
+  end
+end
+

data/lib/generators/sidekiq_sqs_processor/templates/README.txt

@@ -0,0 +1,41 @@
+
+=== SidekiqSqsProcessor Installation ===
+
+SidekiqSqsProcessor has been installed in your Rails application.
+
+== Next Steps ==
+
+1. Review and update the configuration in:
+   config/initializers/sidekiq_sqs_processor.rb
+   
+   Make sure to set your SQS queue URLs!
+
+2. Check the example worker:
+   app/workers/example_sqs_worker.rb
+   
+   Update it to handle your specific message types.
+
+3. Run Sidekiq to start processing messages:
+   bundle exec sidekiq
+
+== Worker Generation ==
+
+To create additional SQS message workers:
+
+$ rails generate sidekiq_sqs_processor:worker UserNotification
+
+This will create:
+  app/workers/user_notification_worker.rb
+  spec/workers/user_notification_worker_spec.rb (if using RSpec)
+  
+== Advanced Configuration ==
+
+* Set AWS credentials in your environment variables:
+  export AWS_ACCESS_KEY_ID=your_key
+  export AWS_SECRET_ACCESS_KEY=your_secret
+  
+* Configure Sidekiq as usual in config/sidekiq.yml
+
+* Enable polling in development mode:
+  export ENABLE_SQS_POLLING_IN_
+

data/lib/generators/sidekiq_sqs_processor/templates/initializer.rb.tt

@@ -0,0 +1,66 @@
+
+# SidekiqSqsProcessor configuration
+SidekiqSqsProcessor.configure do |config|
+  # AWS credentials and region
+  # By default, SidekiqSqsProcessor will use the following credential sources (in order):
+  # 1. AWS SDK's default credential provider chain:
+  #    - Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+  #    - Shared credentials file (~/.aws/credentials)
+  #    - EC2 instance profile or ECS task role
+  #
+  # Uncomment the lines below to explicitly set credentials
+  # config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID']
+  # config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+  
+  # AWS region for SQS queues
+  config.aws_region = '<%= aws_region %>'
+
+  # SQS queue URLs (required)
+  # A list of SQS queue URLs to poll for messages
+  config.queue_urls = <%= queue_urls %>
+  
+  # SQS message options
+  config.visibility_timeout = 300    # 5 minutes
+  config.wait_time_seconds = 20      # 20 seconds (long polling)
+  config.max_number_of_messages = 10 # Max messages per receive call
+  
+  # Polling configuration
+  config.polling_type = <%= polling_type %> # :continuous or :scheduled
+  config.polling_frequency = 60      # Only used for scheduled polling (in seconds)
+  
+  <% if Rails.env.development? %>
+  # Disable polling in development by default
+  # Set ENABLE_SQS_POLLING_IN_DEV=true to enable
+  config.polling_enabled = ENV['ENABLE_SQS_POLLING_IN_DEV'] == 'true'
+  <% else %>
+  config.polling_enabled = true     # Enable polling in production
+  <% end %>
+  
+  # Start polling when the application starts
+  config.poll_on_startup = true
+  
+  # Sidekiq options
+  config.worker_queue_name = 'sqs_default' # Default queue name for SQS worker jobs
+  config.worker_retry_count = 5            # Number of retries for failed jobs
+  
+  # Default message routing
+  # If set, all messages will be routed to this worker class
+  # If nil, will attempt to determine worker from message content
+  # config.default_worker_class = 'ExampleSqsWorker'
+  
+  # Custom error handling (optional)
+  # Uncomment to add custom error reporting (e.g., to a monitoring service)
+  # config.error_handler = ->(error, context) do
+  #   # Report to your error monitoring service
+  #   Raven.capture_exception(error, extra: context) if defined?(Raven)
+  #   Rails.logger.error("SQS Error: #{error.message}\nContext: #{context.inspect}")
+  # end
+  
+  # Set logger - by default uses Rails.logger
+  config.logger = Rails.logger
+  config.log_level = Rails.env.production? ? Logger::INFO : Logger::DEBUG
+end
+
+# If using sidekiq-cron for scheduled polling, the Railtie will
+# automatically set up the cron job based on your configuration.
+

data/lib/generators/sidekiq_sqs_processor/templates/worker.rb.tt

@@ -0,0 +1,59 @@
+
+class <%= worker_class_name %> < SidekiqSqsProcessor::BaseWorker
+  # Configure Sidekiq options if needed
+  sidekiq_options queue: '<%= queue_name %>'<%= retry_option %>
+  
+  # Process an SQS message
+  # @param message_body [Hash, String] The parsed message body
+  # @return [Any] Result of processing (will be logged)
+  def process_message(message_body)
+    # Log the message content
+    logger.info("Processing message: #{message_body.inspect}")
+    
+    # Your message processing logic goes here
+    # 
+    # Example: Handle different message types based on content
+    if message_body.is_a?(Hash)
+      case message_body['type']
+      when 'user_created'
+        process_user_created(message_body['data'])
+      when 'order_placed'
+        process_order_placed(message_body['data'])
+      else
+        process_generic_hash(message_body)
+      end
+    else
+      process_generic_message(message_body)
+    end
+    
+    # Return success result (optional)
+    { status: 'success' }
+  end
+  
+  private
+  
+  def process_user_created(user_data)
+    logger.info("Processing user created event")
+    # Your user creation logic here
+    # Example:
+    # User.create!(user_data)
+  end
+  
+  def process_order_placed(order_data)
+    logger.info("Processing order placed event")
+    # Your order processing logic here
+    # Example:
+    # OrderProcessor.new(order_data).process
+  end
+  
+  def process_generic_hash(data)
+    logger.info("Processing generic hash data with keys: #{data.keys.join(', ')}")
+    # Generic hash handling
+  end
+  
+  def process_generic_message(message)
+    logger.info("Processing generic message: #{message}")
+    # Generic message handling
+  end
+end
+

data/lib/generators/sidekiq_sqs_processor/templates/worker_spec.rb.tt

@@ -0,0 +1,54 @@
+
+require 'rails_helper'
+
+RSpec.describe <%= worker_class_name %> do
+  let(:worker) { described_class.new }
+  
+  describe '#process_message' do
+    context 'with a hash message' do
+      let(:message) { { 'type' => 'user_created', 'data' => { 'name' => 'Test User' } } }
+      
+      it 'processes the message correctly' do
+        expect(worker).to receive(:process_user_created).with(message['data'])
+        result = worker.process_message(message)
+        expect(result).to eq(status: 'success')
+      end
+    end
+    
+    context 'with a string message' do
+      let(:message) { 'test message' }
+      
+      it 'processes the message correctly' do
+        expect(worker).to receive(:process_generic_message).with(message)
+        result = worker.process_message(message)
+        expect(result).to eq(status: 'success')
+      end
+    end
+  end
+  
+  describe 'SQS integration' do
+    it 'correctly deletes the message from SQS when processing succeeds' do
+      # Create a test message that simulates SQS format
+      message_data = {
+        'receipt_handle' => 'test-receipt-handle',
+        'queue_url' => 'https://sqs.us-east-1.amazonaws.com/123456789012/test-queue',
+        'message_id' => 'test-message-id',
+        'body' => '{"type":"test","data":{"id":123}}'
+      }
+      
+      # Mock the SQS client
+      sqs_client = instance_double(Aws::SQS::Client)
+      allow(SidekiqSqsProcessor).to receive(:sqs_client).and_return(sqs_client)
+      
+      # Expect delete_message to be called
+      expect(sqs_client).to receive(:delete_message).with(
+        queue_url: message_data['queue_url'],
+        receipt_handle: message_data['receipt_handle']
+      )
+      
+      # Perform the job
+      worker.perform(message_data)
+    end
+  end
+end
+

data/lib/generators/sidekiq_sqs_processor/templates/worker_test.rb.tt

@@ -0,0 +1,50 @@
+
+require 'test_helper'
+
+class <%= worker_class_name %>Test < ActiveSupport::TestCase
+  setup do
+    @worker = <%= worker_class_name %>.new
+  end
+  
+  test "processes a hash message correctly" do
+    message = { 'type' => 'user_created', 'data' => { 'name' => 'Test User' } }
+    
+    @worker.expects(:process_user_created).with(message['data'])
+    result = @worker.process_message(message)
+    
+    assert_equal({ status: 'success' }, result)
+  end
+  
+  test "processes a string message correctly" do
+    message = 'test message'
+    
+    @worker.expects(:process_generic_message).with(message)
+    result = @worker.process_message(message)
+    
+    assert_equal({ status: 'success' }, result)
+  end
+  
+  test "deletes the message from SQS when processing succeeds" do
+    # Create a test message that simulates SQS format
+    message_data = {
+      'receipt_handle' => 'test-receipt-handle',
+      'queue_url' => 'https://sqs.us-east-1.amazonaws.com/123456789012/test-queue',
+      'message_id' => 'test-message-id',
+      'body' => '{"type":"test","data":{"id":123}}'
+    }
+    
+    # Mock the SQS client
+    sqs_client = mock()
+    SidekiqSqsProcessor.stubs(:sqs_client).returns(sqs_client)
+    
+    # Expect delete_message to be called
+    sqs_client.expects(:delete_message).with(
+      queue_url: message_data['queue_url'],
+      receipt_handle: message_data['receipt_handle']
+    )
+    
+    # Perform the job
+    @worker.perform(message_data)
+  end
+end
+