sqs_processor 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6a7f784c9d006a263a073a856b1eccf16608848e03edf06ebc769c3f68d8748c
+  data.tar.gz: 6c42692cd419b8137f0b97109c1f215092952458ef7c1d17e283bb57f8aad6e8
+SHA512:
+  metadata.gz: 281cf0dfbf315f4c9c5163e6017c5c0a0a549d777780a4f02b2f2d4fd49454d4b726fdfcfacc8d37d5e58e9cfe20a6583128284a8a5bf6d3cd206489e996f9ff
+  data.tar.gz: 3f46cf8f400ccd3619b5586a6ecfb54c5d7cde6d0b84f40abfdc664ba30e5fc95180a5090a277172b33333f39e49f7ee06ea916ac8aa70389a615c2d7cce150b

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 SQS Processor
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE. 

data/README.md

@@ -0,0 +1,293 @@
+# SQS Processor Gem
+
+A Ruby gem for processing messages from Amazon SQS queues with configurable message handling and error recovery.
+
+## Features
+
+- **Long Polling**: Efficiently polls SQS with configurable wait times
+- **Batch Processing**: Processes multiple messages per batch
+- **Error Handling**: Robust error handling with message retention on failure
+- **Customizable**: Extensible message processing logic
+- **Logging**: Comprehensive logging with configurable levels
+- **Command Line Options**: Flexible configuration via command line arguments
+- **Environment Variables**: Support for AWS credentials and configuration via environment variables
+
+## Prerequisites
+
+- Ruby 2.6 or higher
+- AWS credentials configured (via environment variables, IAM roles, or AWS CLI)
+- SQS queue URL
+
+## Installation
+
+### As a Gem
+
+```bash
+gem install sqs_processor
+```
+
+### From Source
+
+1. Clone the repository
+2. Install dependencies:
+
+```bash
+bundle install
+```
+
+3. Build and install the gem:
+
+```bash
+bundle exec rake install
+```
+
+4. Copy the environment example and configure your settings:
+
+```bash
+cp env.example .env
+```
+
+5. Edit `.env` with your AWS credentials and SQS queue URL:
+
+```bash
+DATA_SYNC_AWS_ACCESS_KEY_ID=your_access_key_here
+DATA_SYNC_AWS_SECRET_ACCESS_KEY=your_secret_key_here
+DATA_SYNC_AWS_REGION=us-east-1
+DATA_SYNC_SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/123456789012/your-queue-name
+```
+
+## Usage
+
+### Command Line Usage
+
+```bash
+sqs_processor [options]
+```
+
+### Command Line Options
+
+```bash
+sqs_processor [options]
+
+Options:
+  -q, --queue-url URL              SQS Queue URL
+  -r, --region REGION              AWS Region
+  -m, --max-messages NUMBER        Max messages per batch (default: 10)
+  -v, --visibility-timeout SECONDS Visibility timeout in seconds (default: 30)
+  -h, --help                       Show this help message
+```
+
+### Examples
+
+```bash
+# Basic usage with environment variables
+sqs_processor
+
+# Specify queue URL and region
+sqs_processor -q https://sqs.us-west-2.amazonaws.com/123456789012/my-queue -r us-west-2
+
+# Custom batch size and visibility timeout
+sqs_processor -m 5 -v 60
+```
+
+### Programmatic Usage
+
+#### Basic Usage
+
+```ruby
+require 'sqs_processor'
+
+# Create a processor instance
+processor = SQSProcessor::Processor.new(
+  queue_url: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue',
+  region: 'us-east-1',
+  max_messages: 10,
+  visibility_timeout: 30,
+  aws_access_key_id: 'your-access-key',
+  aws_secret_access_key: 'your-secret-key'
+)
+
+# Start processing messages
+processor.process_messages
+```
+
+## Message Processing
+
+The gem uses a hook-based approach for message processing. You must implement the `handle_message` method in your subclass to define how messages should be processed.
+
+### Hook Method
+
+The `handle_message(message, body)` method receives:
+- `message`: The SQS message object (contains message_id, receipt_handle, etc.)
+- `body`: The parsed JSON body of the message
+
+Return `true` if processing was successful (message will be deleted from queue), or `false` if processing failed (message will remain in queue for retry).
+
+### Example Message Format
+
+```json
+{
+  "event_type": "data_sync",
+  "dataset_id": "12345",
+  "timestamp": "2024-01-01T00:00:00Z"
+}
+```
+
+### Custom Processing
+
+To implement custom message processing logic, create a subclass of `SQSProcessor::Processor` and override the `handle_message` method:
+
+```ruby
+require 'sqs_processor'
+
+class MyCustomProcessor < SQSProcessor::Processor
+  def handle_message(message, body)
+    # This method receives the SQS message object and parsed body
+    # Return true if processing was successful, false otherwise
+    
+    case body['event_type']
+    when 'data_sync'
+      process_data_sync(body)
+    when 'report_generation'
+      process_report_generation(body)
+    else
+      logger.warn "Unknown event type: #{body['event_type']}"
+      false
+    end
+  end
+
+  private
+
+  def process_data_sync(body)
+    logger.info "Processing data sync for dataset: #{body['dataset_id']}"
+    # Your custom logic here
+    true
+  end
+
+  def process_report_generation(body)
+    logger.info "Processing report generation for report: #{body['report_id']}"
+    # Your custom logic here
+    true
+  end
+end
+
+# Usage
+processor = MyCustomProcessor.new(
+  queue_url: 'your-queue-url',
+  region: 'us-east-1',
+  aws_access_key_id: 'your-access-key',
+  aws_secret_access_key: 'your-secret-key'
+)
+processor.process_messages
+```
+
+## Configuration
+
+### Constructor Parameters
+
+The `SQSProcessor::Processor.new` method accepts the following parameters:
+
+- `queue_url:` (required) - The SQS queue URL
+- `region:` (optional, default: 'us-east-1') - AWS region
+- `max_messages:` (optional, default: 10) - Maximum messages per batch
+- `visibility_timeout:` (optional, default: 30) - Message visibility timeout in seconds
+- `aws_access_key_id:` (optional) - AWS access key ID
+- `aws_secret_access_key:` (optional) - AWS secret access key
+- `aws_session_token:` (optional) - AWS session token for temporary credentials
+- `aws_credentials:` (optional) - Pre-configured AWS credentials object
+- `logger:` (optional) - Custom logger instance
+
+### Environment Variables
+
+- `DATA_SYNC_AWS_ACCESS_KEY_ID`: Your AWS access key
+- `DATA_SYNC_AWS_SECRET_ACCESS_KEY`: Your AWS secret key
+- `DATA_SYNC_AWS_SESSION_TOKEN`: Your AWS session token (optional, for temporary credentials)
+- `DATA_SYNC_AWS_REGION`: AWS region (default: us-east-1)
+- `DATA_SYNC_SQS_QUEUE_URL`: Your SQS queue URL
+
+### AWS Credentials
+
+The gem supports multiple ways to provide AWS credentials:
+
+1. **Initializer Configuration**: Set credentials in the initializer block
+2. **Environment Variables**: Use the `DATA_SYNC_` prefixed environment variables
+3. **AWS SDK Default Chain**: If no credentials are provided, the AWS SDK will use its default credential provider chain (IAM roles, AWS CLI, etc.)
+4. **Direct Parameter**: Pass credentials directly to the processor constructor
+
+The script supports multiple ways to provide AWS credentials:
+
+1. **Environment Variables**: Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+2. **IAM Roles**: If running on EC2 with IAM roles
+3. **AWS CLI**: If you have AWS CLI configured
+4. **AWS SDK Default Credential Provider Chain**: Automatic credential resolution
+
+## Error Handling
+
+- **JSON Parse Errors**: Messages with invalid JSON are logged but kept in queue
+- **Processing Errors**: Failed messages remain in queue for retry
+- **Network Errors**: Automatic retry with exponential backoff
+- **Queue Errors**: Comprehensive error logging with stack traces
+
+## Monitoring
+
+The script provides detailed logging including:
+
+- Queue attributes (message counts)
+- Message processing status
+- Error details with stack traces
+- Processing performance metrics
+
+## Best Practices
+
+1. **Set appropriate visibility timeout**: Should be longer than your processing time
+2. **Use long polling**: Reduces API calls and costs
+3. **Handle errors gracefully**: Return `false` from processing methods to keep messages in queue
+4. **Monitor queue depth**: Use the built-in queue attribute reporting
+5. **Use appropriate batch sizes**: Balance between throughput and memory usage
+
+## Troubleshooting
+
+### Common Issues
+
+1. **"Queue URL is required"**: Set `DATA_SYNC_SQS_QUEUE_URL` environment variable or use `-q` option
+2. **"Access Denied"**: Check AWS credentials and SQS permissions
+3. **"Queue does not exist"**: Verify queue URL and region
+4. **Messages not being processed**: Check visibility timeout and processing logic
+
+### Debug Mode
+
+To enable debug logging, modify the logger level in the script:
+
+```ruby
+@logger.level = Logger::DEBUG
+```
+
+## Development
+
+### Running Tests
+
+```bash
+bundle exec rspec
+```
+
+### Code Style
+
+```bash
+bundle exec rubocop
+```
+
+### Building the Gem
+
+```bash
+bundle exec rake build
+```
+
+### Publishing the Gem
+
+```bash
+bundle exec rake release
+```
+
+## License
+
+This gem is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. 

data/bin/sqs_processor

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+require 'sqs_processor'
+require 'optparse'
+
+def parse_options
+  options = {
+    queue_url: nil,
+    region: nil,
+    max_messages: 10,
+    visibility_timeout: 30
+  }
+
+  OptionParser.new do |opts|
+    opts.banner = 'Usage: sqs_processor [options]'
+
+    opts.on('-q', '--queue-url URL', 'SQS Queue URL') do |url|
+      options[:queue_url] = url
+    end
+
+    opts.on('-r', '--region REGION', 'AWS Region') do |region|
+      options[:region] = region
+    end
+
+    opts.on('-m', '--max-messages NUMBER', Integer, 'Max messages per batch (default: 10)') do |num|
+      options[:max_messages] = num
+    end
+
+    opts.on('-v', '--visibility-timeout SECONDS', Integer, 'Visibility timeout in seconds (default: 30)') do |timeout|
+      options[:visibility_timeout] = timeout
+    end
+
+    opts.on('-h', '--help', 'Show this help message') do
+      puts opts
+      exit
+    end
+  end.parse!
+
+  options
+end
+
+if __FILE__ == $0
+  options = parse_options
+
+  begin
+    # Load environment variables
+    Dotenv.load
+
+    # Get queue URL from options or environment
+    queue_url = options[:queue_url] || ENV.fetch('DATA_SYNC_SQS_QUEUE_URL', nil)
+
+    unless queue_url
+      puts 'Error: Queue URL is required. Set DATA_SYNC_SQS_QUEUE_URL environment variable or use -q option.'
+      exit 1
+    end
+
+    processor = SQSProcessor::Processor.new(
+      queue_url: queue_url,
+      region: options[:region] || ENV['DATA_SYNC_AWS_REGION'] || 'us-east-1',
+      max_messages: options[:max_messages],
+      visibility_timeout: options[:visibility_timeout],
+      aws_access_key_id: ENV.fetch('DATA_SYNC_AWS_ACCESS_KEY_ID', nil),
+      aws_secret_access_key: ENV.fetch('DATA_SYNC_AWS_SECRET_ACCESS_KEY', nil),
+      aws_session_token: ENV.fetch('DATA_SYNC_AWS_SESSION_TOKEN', nil)
+    )
+
+    # Show queue attributes before starting
+    processor.get_queue_attributes
+
+    # Start processing messages
+    processor.process_messages
+  rescue StandardError => e
+    puts "Error: #{e.message}"
+    puts e.backtrace.join("\n")
+    exit 1
+  end
+end

data/lib/sqs_processor/processor.rb

@@ -0,0 +1,153 @@
+require 'aws-sdk-sqs'
+require 'json'
+require 'logger'
+require 'dotenv'
+require 'optparse'
+
+# Load environment variables
+Dotenv.load
+
+module SQSProcessor
+  class Processor
+    attr_reader :sqs_client, :queue_url, :logger
+
+    def initialize(queue_url:, region: 'us-east-1', max_messages: 10, visibility_timeout: 30,
+                   aws_access_key_id: nil, aws_secret_access_key: nil, aws_session_token: nil,
+                   aws_credentials: nil, logger: nil)
+      @queue_url = queue_url
+      @region = region
+      @max_messages = max_messages
+      @visibility_timeout = visibility_timeout
+      @aws_credentials = aws_credentials
+      @aws_access_key_id = aws_access_key_id
+      @aws_secret_access_key = aws_secret_access_key
+      @aws_session_token = aws_session_token
+
+      setup_logger(logger)
+      setup_sqs_client
+
+      raise 'Queue URL is required.' unless @queue_url
+    end
+
+    def setup_logger(custom_logger = nil)
+      @logger = custom_logger || Logger.new(STDOUT)
+      @logger.level = Logger::INFO
+      @logger.formatter = proc do |severity, datetime, progname, msg|
+        "#{datetime.strftime('%Y-%m-%d %H:%M:%S')} [#{severity}] #{msg}\n"
+      end
+    end
+
+    def setup_sqs_client
+      credentials = if @aws_credentials
+                      @aws_credentials
+                    elsif @aws_access_key_id && @aws_secret_access_key
+                      Aws::Credentials.new(@aws_access_key_id, @aws_secret_access_key, @aws_session_token)
+                    elsif ENV['DATA_SYNC_AWS_ACCESS_KEY_ID'] && ENV['DATA_SYNC_AWS_SECRET_ACCESS_KEY']
+                      Aws::Credentials.new(ENV['DATA_SYNC_AWS_ACCESS_KEY_ID'], ENV['DATA_SYNC_AWS_SECRET_ACCESS_KEY'])
+                    else
+                      Aws::Credentials.new
+                    end
+
+      @sqs_client = Aws::SQS::Client.new(
+        region: @region,
+        credentials: credentials
+      )
+    end
+
+    def process_messages
+      logger.info 'Starting SQS message processing...'
+      logger.info "Queue URL: #{@queue_url}"
+      logger.info "Max messages per batch: #{@max_messages}"
+      logger.info "Visibility timeout: #{@visibility_timeout} seconds"
+
+      loop do
+        receive_messages
+        sleep 1 # Small delay between polling cycles
+      rescue StandardError => e
+        logger.error "Error in message processing loop: #{e.message}"
+        logger.error e.backtrace.join("\n")
+        sleep 5 # Longer delay on error
+      end
+    end
+
+    def receive_messages
+      response = @sqs_client.receive_message(
+        queue_url: @queue_url,
+        max_number_of_messages: @max_messages,
+        visibility_timeout: @visibility_timeout,
+        wait_time_seconds: 20 # Long polling
+      )
+
+      if response.messages.empty?
+        logger.debug 'No messages received'
+        return
+      end
+
+      logger.info "Received #{response.messages.length} message(s)"
+
+      response.messages.each do |message|
+        process_single_message(message)
+      end
+    end
+
+    def process_single_message(message)
+      logger.info "Processing message: #{message.message_id}"
+
+      begin
+        # Parse message body
+        body = JSON.parse(message.body)
+        logger.info "Message body: #{body}"
+
+        # Call the hook method that should be implemented by the host application
+        result = handle_message(body['Message'])
+
+        if result
+          # Delete the message from queue after successful processing
+          delete_message(message)
+          logger.info "Successfully processed and deleted message: #{message.message_id}"
+        else
+          logger.warn "Message processing returned false, keeping message in queue: #{message.message_id}"
+        end
+      rescue JSON::ParserError => e
+        logger.error "Failed to parse message body as JSON: #{e.message}"
+        logger.error "Raw message body: #{message.body}"
+        # Keep message in queue for manual inspection
+      rescue StandardError => e
+        logger.error "Error processing message #{message.message_id}: #{e.message}"
+        logger.error e.backtrace.join("\n")
+        # Keep message in queue for retry
+      end
+    end
+
+    # Hook method that should be implemented by the host application
+    # Override this method in your subclass to implement custom message processing
+    def handle_message(message, body)
+      logger.warn 'handle_message method not implemented. Override this method in your subclass.'
+      false
+    end
+
+    def delete_message(message)
+      @sqs_client.delete_message(
+        queue_url: @queue_url,
+        receipt_handle: message.receipt_handle
+      )
+    rescue StandardError => e
+      logger.error "Failed to delete message #{message.message_id}: #{e.message}"
+    end
+
+    def get_queue_attributes
+      response = @sqs_client.get_queue_attributes(
+        queue_url: @queue_url,
+        attribute_names: ['All']
+      )
+
+      attributes = response.attributes
+      logger.info 'Queue attributes:'
+      logger.info "  Approximate number of messages: #{attributes['ApproximateNumberOfMessages']}"
+      logger.info "  Approximate number of messages not visible: #{attributes['ApproximateNumberOfMessagesNotVisible']}"
+      logger.info "  Approximate number of messages delayed: #{attributes['ApproximateNumberOfMessagesDelayed']}"
+
+      attributes
+    end
+  end
+end

data/lib/sqs_processor/version.rb

@@ -0,0 +1,3 @@
+module SQSProcessor
+  VERSION = "0.1.0"
+end 

data/lib/sqs_processor.rb

@@ -0,0 +1,7 @@
+require_relative 'sqs_processor/version'
+require_relative 'sqs_processor/processor'
+
+module SQSProcessor
+  class Error < StandardError; end
+  # Your code goes here...
+end

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: sqs_processor
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Your Name
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: A comprehensive Ruby gem for processing messages from Amazon SQS queues
+  with configurable message handling, error recovery, and extensible processing logic.
+email:
+- your.email@example.com
+executables:
+- sqs_processor
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/sqs_processor
+- lib/sqs_processor.rb
+- lib/sqs_processor/processor.rb
+- lib/sqs_processor/version.rb
+homepage: https://github.com/yourusername/sqs_processor
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: A Ruby gem for processing messages from Amazon SQS queues
+test_files: []