sidekiq_sqs_processor 0.1.0

data/lib/generators/sidekiq_sqs_processor/worker_generator.rb

@@ -0,0 +1,54 @@
+
+require 'rails/generators/named_base'
+
+module SidekiqSqsProcessor
+  module Generators
+    class WorkerGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path('templates', __dir__)
+      
+      desc "Creates a SidekiqSqsProcessor worker for processing SQS messages."
+      
+      class_option :queue, type: :string, default: nil,
+                   desc: "The Sidekiq queue for this worker (defaults to sqs_[worker_name])"
+      
+      class_option :retry, type: :numeric, default: nil,
+                   desc: "Number of retries for this worker (defaults to configuration value)"
+      
+      class_option :test, type: :boolean, default: true,
+                   desc: "Generate test file for this worker"
+                   
+      def create_worker_file
+        template "worker.rb.tt", "app/workers/#{file_name}_worker.rb"
+      end
+      
+      def create_test_file
+        return unless options[:test]
+        
+        if defined?(RSpec) || File.exist?(File.join(destination_root, 'spec'))
+          template "worker_spec.rb.tt", "spec/workers/#{file_name}_worker_spec.rb"
+        elsif File.exist?(File.join(destination_root, 'test'))
+          template "worker_test.rb.tt", "test/workers/#{file_name}_worker_test.rb"
+        end
+      end
+      
+      private
+      
+      def queue_name
+        options[:queue] || "sqs_#{file_name.underscore}"
+      end
+      
+      def retry_option
+        if options[:retry]
+          ", retry: #{options[:retry]}"
+        else
+          ""
+        end
+      end
+      
+      def worker_class_name
+        "#{class_name}Worker"
+      end
+    end
+  end
+end
+

data/lib/sidekiq_sqs_processor/base_worker.rb

@@ -0,0 +1,82 @@
+require "sidekiq"
+
+module SidekiqSqsProcessor
+  class BaseWorker
+    include Sidekiq::Worker
+
+    class << self
+      def process_message_automatically
+        return @process_message_automatically if defined?(@process_message_automatically)
+        @process_message_automatically = true
+      end
+
+      def process_message_automatically=(value)
+        @process_message_automatically = value
+      end
+
+      def inherited(subclass)
+        super
+        # Copy the process_message_automatically value to subclasses
+        subclass.process_message_automatically = process_message_automatically
+      end
+    end
+
+    # Configure Sidekiq options with defaults
+    sidekiq_options(
+      retry: 2,     # Default retry count
+      queue: 'default' # Default queue name
+    )
+
+    def perform(message_data)
+      process_message_data(message_data) if self.class.process_message_automatically
+    end
+
+    # Override this method in your worker
+    def process_message(body)
+      raise NotImplementedError, "You must implement process_message in your worker"
+    end
+
+    private
+
+    def process_message_data(message_data)
+      begin
+        # Parse and process the message
+        body = JSON.parse(message_data["body"])
+        process_message(body)
+
+        # Delete the message from SQS after successful processing
+        delete_sqs_message(message_data)
+      rescue JSON::ParserError => e
+        handle_error(e, message_data, "Failed to parse message body")
+      rescue Aws::SQS::Errors::ServiceError => e
+        handle_error(e, message_data, "SQS service error")
+      rescue StandardError => e
+        handle_error(e, message_data, "Error processing message")
+      end
+    end
+
+    def delete_sqs_message(message_data)
+      return unless message_data["queue_url"] && message_data["receipt_handle"]
+      
+      SidekiqSqsProcessor.sqs_client.delete_message(
+        queue_url: message_data["queue_url"],
+        receipt_handle: message_data["receipt_handle"]
+      )
+    rescue Aws::SQS::Errors::ServiceError => e
+      handle_error(e, message_data, "Failed to delete message from SQS")
+    end
+
+    def handle_error(error, message_data, description)
+      SidekiqSqsProcessor.handle_error(error, {
+        worker: self.class.name,
+        message: message_data,
+        description: description
+      })
+      raise # Re-raise to trigger Sidekiq retry
+    end
+
+    def logger
+      SidekiqSqsProcessor.configuration.logger || Sidekiq.logger
+    end
+  end
+end

data/lib/sidekiq_sqs_processor/configuration.rb

@@ -0,0 +1,182 @@
+module SidekiqSqsProcessor
+  class Configuration
+    # AWS credentials
+    attr_accessor :aws_access_key_id, :aws_secret_access_key, :aws_credentials
+    attr_reader :aws_region, :queue_urls, :max_number_of_messages,
+                :visibility_timeout, :wait_time_seconds, :poller_thread_count,
+                :error_handler, :polling_enabled, :polling_type, :poll_on_startup,
+                :polling_frequency, :log_level
+
+    # Worker configuration
+    attr_accessor :worker_retry_count, :worker_queue_name, :logger
+
+    def initialize
+      @aws_region = "us-east-1"
+      @queue_urls = []
+      @max_number_of_messages = 10
+      @visibility_timeout = 30
+      @wait_time_seconds = 20
+      @poller_thread_count = 1
+      @worker_retry_count = 25
+      @worker_queue_name = "default"
+      @error_handler = default_error_handler
+      @polling_enabled = true
+      @polling_type = :continuous
+      @poll_on_startup = true
+      @polling_frequency = 60 # seconds
+      @log_level = Logger::INFO
+    end
+
+    def aws_region=(region)
+      @aws_region = region&.strip
+    end
+
+    def queue_urls=(urls)
+      @queue_urls = Array(urls).map(&:strip)
+    end
+
+    def add_queue_url(url)
+      @queue_urls << url.strip
+      @queue_urls.uniq!
+    end
+
+    def max_number_of_messages=(value)
+      @max_number_of_messages = value.to_i
+    end
+
+    def visibility_timeout=(value)
+      @visibility_timeout = value.to_i
+    end
+
+    def wait_time_seconds=(value)
+      @wait_time_seconds = value.to_i
+    end
+
+    def poller_thread_count=(value)
+      @poller_thread_count = value.to_i
+    end
+
+    def error_handler=(handler)
+      if !handler.respond_to?(:call) && !handler.nil?
+        raise ArgumentError, "Error handler must be callable (respond to #call)"
+      end
+      @error_handler = handler
+    end
+    
+    def polling_enabled=(value)
+      @polling_enabled = !!value
+    end
+    
+    def polling_type=(value)
+      value = value.to_sym if value.respond_to?(:to_sym)
+      unless [:continuous, :scheduled].include?(value)
+        raise ArgumentError, "polling_type must be :continuous or :scheduled"
+      end
+      @polling_type = value
+    end
+    
+    def poll_on_startup=(value)
+      @poll_on_startup = !!value
+    end
+    
+    def polling_frequency=(value)
+      @polling_frequency = [value.to_i, 1].max
+    end
+    
+    def log_level=(level)
+      if level.is_a?(Integer) && (0..5).include?(level)
+        @log_level = level
+      elsif level.is_a?(Symbol) || level.is_a?(String)
+        level_map = {
+          debug: Logger::DEBUG,
+          info: Logger::INFO,
+          warn: Logger::WARN,
+          error: Logger::ERROR,
+          fatal: Logger::FATAL,
+          unknown: Logger::UNKNOWN
+        }
+        sym_level = level.to_sym.downcase
+        if level_map.key?(sym_level)
+          @log_level = level_map[sym_level]
+        else
+          raise ArgumentError, "Invalid log level: #{level}. Valid levels are: #{level_map.keys.join(', ')}"
+        end
+      else
+        raise ArgumentError, "Log level must be an Integer (0-5) or one of: debug, info, warn, error, fatal, unknown"
+      end
+    end
+    def handle_error(error, context = {})
+      if @error_handler
+        @error_handler.call(error, context)
+      else
+        default_error_handler.call(error, context)
+      end
+    end
+
+    def validate!
+      errors = []
+
+      # AWS Config
+      if aws_region.nil? || aws_region.empty?
+        errors << "aws_region is not configured"
+      end
+
+      # Validate thread and message settings before queue URLs
+      if !poller_thread_count.positive?
+        errors << "poller_thread_count must be positive"
+      end
+
+      if !max_number_of_messages.between?(1, 10)
+        errors << "max_number_of_messages must be between 1 and 10"
+      end
+
+      if !wait_time_seconds.between?(0, 20)
+        errors << "wait_time_seconds must be between 0 and 20"
+      end
+
+      if !visibility_timeout.positive?
+        errors << "visibility_timeout must be positive"
+      end
+      
+      if ![:continuous, :scheduled].include?(polling_type)
+        errors << "polling_type must be :continuous or :scheduled"
+      end
+      
+      if !polling_frequency.positive?
+        errors << "polling_frequency must be positive"
+      end
+      
+      # Queue URLs should be checked last
+      if queue_urls.empty?
+        errors << "queue_urls is empty"
+      end
+
+      # Raise first specific error for single-error tests
+      if errors.length == 1
+        raise ArgumentError, errors.first
+      end
+
+      # Raise all errors together
+      if errors.any?
+        raise ArgumentError, "Invalid configuration: #{errors.join(', ')}"
+      end
+
+      true
+    end
+
+    private
+
+    def default_error_handler
+      ->(error, context = {}) do
+        logger = self.logger || Sidekiq.logger
+        if error.is_a?(Exception)
+          logger.error(error.message)
+          logger.error(error.backtrace.join("\n")) if error.backtrace
+        else
+          logger.error(error.to_s)
+        end
+        logger.error("Context: #{context.inspect}") unless context.empty?
+      end
+    end
+  end
+end

data/lib/sidekiq_sqs_processor/continuous_poller.rb

@@ -0,0 +1,128 @@
+require 'singleton'
+
+module SidekiqSqsProcessor
+  class ContinuousPoller
+    include Singleton
+
+    def initialize
+      @running = false
+      @threads = []
+      @mutex = Mutex.new
+    end
+
+    def start
+      return false if running?
+
+      @mutex.synchronize do
+        @running = true
+        start_polling_threads
+      end
+
+      true
+    end
+
+    def stop
+      return false unless running?
+
+      @mutex.synchronize do
+        @running = false
+        stop_polling_threads
+      end
+
+      true
+    end
+
+    def running?
+      @running
+    end
+
+    def stats
+      {
+        running: running?,
+        threads: @threads.count,
+        queue_urls: SidekiqSqsProcessor.configuration.queue_urls
+      }
+    end
+
+    private
+
+    def start_polling_threads
+      SidekiqSqsProcessor.configuration.queue_urls.each do |queue_url|
+        SidekiqSqsProcessor.configuration.poller_thread_count.times do
+          thread = Thread.new do
+            poll_queue(queue_url) while running?
+          end
+          @threads << thread
+        end
+      end
+    end
+
+    def stop_polling_threads
+      @threads.each(&:exit)
+      @threads.each(&:join)
+      @threads.clear
+    end
+
+    def poll_queue(queue_url)
+      response = receive_messages(queue_url)
+      process_messages(response.messages, queue_url)
+    rescue StandardError => e
+      SidekiqSqsProcessor.handle_error(e, { queue_url: queue_url })
+      sleep(1) # Brief pause before retrying
+    end
+
+    def receive_messages(queue_url)
+      SidekiqSqsProcessor.sqs_client.receive_message(
+        queue_url: queue_url,
+        max_number_of_messages: SidekiqSqsProcessor.configuration.max_number_of_messages,
+        visibility_timeout: SidekiqSqsProcessor.configuration.visibility_timeout,
+        wait_time_seconds: SidekiqSqsProcessor.configuration.wait_time_seconds,
+        attribute_names: ["All"],
+        message_attribute_names: ["All"]
+      )
+    end
+    def process_messages(messages, queue_url)
+      messages.each do |message|
+        message_data = nil
+        worker_class = nil
+        
+        begin
+          # Convert to a hash for passing to the worker
+          message_data = {
+            "message_id" => message.message_id,
+            "receipt_handle" => message.receipt_handle,
+            "body" => message.body,
+            "attributes" => message.attributes,
+            "message_attributes" => message.message_attributes,
+            "md5_of_body" => message.md5_of_body,
+            "queue_url" => queue_url
+          }
+          
+          worker_class = find_worker_for_message(message)
+          if worker_class
+            # Simply call perform_async, which will be handled appropriately in test vs prod
+            worker_class.perform_async(message_data)
+          end
+        rescue StandardError => e
+          # Handle worker error without re-raising
+          data_to_pass = message_data || message
+          handle_worker_error(e, worker_class&.name, data_to_pass, queue_url)
+        end
+      end
+    end
+    
+    def handle_worker_error(error, worker_name, message, queue_url)
+      context = {
+        queue_url: queue_url,
+        worker: worker_name || "Unknown",
+        message: message
+      }
+      SidekiqSqsProcessor.handle_error(error, context)
+    end
+
+    def find_worker_for_message(message)
+      worker_name = "SqsProcessorWorker"
+      SidekiqSqsProcessor.find_worker_class(worker_name)
+    end
+  end
+end

data/lib/sidekiq_sqs_processor/railtie.rb

@@ -0,0 +1,102 @@
+require 'rails'
+require 'sidekiq'
+
+module SidekiqSqsProcessor
+  # Rails integration for SidekiqSqsProcessor
+  # Handles initialization, configuration, and lifecycle management
+  class Railtie < Rails::Railtie
+    initializer "sidekiq_sqs_processor.configure_rails_initialization" do |app|
+      # Set default logger to Rails logger if not specified
+      SidekiqSqsProcessor.configuration.logger ||= Rails.logger
+      
+      # Disable polling in test environment by default
+      if Rails.env.test? && !ENV['ENABLE_SQS_POLLING_IN_TEST']
+        SidekiqSqsProcessor.configuration.polling_enabled = false
+      end
+      
+      # Disable polling in development by default unless explicitly enabled
+      if Rails.env.development? && !ENV['ENABLE_SQS_POLLING_IN_DEV']
+        SidekiqSqsProcessor.configuration.polling_enabled = false
+      end
+      
+      # Configure Sidekiq server middleware and lifecycle hooks
+      if defined?(Sidekiq)
+        Sidekiq.configure_server do |config|
+          # Start continuous poller when Sidekiq server starts
+          # Only if polling is enabled and type is continuous
+          config.on(:startup) do
+            if SidekiqSqsProcessor.configuration.polling_type == :continuous && 
+               SidekiqSqsProcessor.configuration.polling_enabled &&
+               SidekiqSqsProcessor.configuration.poll_on_startup
+              
+              # Only run the poller in the scheduler process if using Sidekiq Enterprise
+              # For regular Sidekiq, this will run in every process
+              if !defined?(Sidekiq::Enterprise) || Sidekiq.schedule?
+                Rails.logger.info("Starting SidekiqSqsProcessor continuous poller")
+                SidekiqSqsProcessor.start_continuous_poller
+              end
+            end
+          end
+          
+          # Stop continuous poller when Sidekiq server shuts down
+          config.on(:shutdown) do
+            if SidekiqSqsProcessor.continuous_poller_running?
+              Rails.logger.info("Stopping SidekiqSqsProcessor continuous poller")
+              SidekiqSqsProcessor.stop_continuous_poller
+            end
+          end
+        end
+      end
+      
+      # Set up scheduled polling if enabled
+      if defined?(Sidekiq::Cron) && 
+         SidekiqSqsProcessor.configuration.polling_type == :scheduled &&
+         SidekiqSqsProcessor.configuration.polling_enabled
+        
+        # Only set up in server mode
+        if Sidekiq.server?
+          frequency = SidekiqSqsProcessor.configuration.polling_frequency
+          
+          # Convert seconds to cron expression
+          # Minimum 1 minute for cron
+          minutes = [frequency / 60, 1].max
+          cron_expression = minutes == 1 ? "* * * * *" : "*/#{minutes} * * * *"
+          
+          # Create the cron job
+          Sidekiq::Cron::Job.create(
+            name: 'SQS Polling Job',
+            cron: cron_expression,
+            class: 'SidekiqSqsProcessor::ScheduledPoller',
+            queue: 'critical'
+          )
+          
+          Rails.logger.info("Registered SidekiqSqsProcessor scheduled poller with cron: #{cron_expression}")
+        end
+      end
+    end
+    
+    # Expose rake tasks if available
+    rake_tasks do
+      load "tasks/sidekiq_sqs_processor_tasks.rake" if File.exist?(File.join(File.dirname(__FILE__), "../tasks/sidekiq_sqs_processor_tasks.rake"))
+    end
+    
+    # Register Rails generators
+    generators do
+      require_relative "../generators/sidekiq_sqs_processor/install_generator"
+      require_relative "../generators/sidekiq_sqs_processor/worker_generator"
+    end
+    
+    # Add local configuration options to Rails application configuration
+    config.after_initialize do |app|
+      # Validate configuration if in production
+      if Rails.env.production? && SidekiqSqsProcessor.configuration.polling_enabled
+        # Ensure configuration is valid
+        unless SidekiqSqsProcessor.configuration.valid?
+          Rails.logger.error("Invalid SidekiqSqsProcessor configuration")
+          SidekiqSqsProcessor.configuration.validate!  # This will raise an error with details
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq_sqs_processor/version.rb

@@ -0,0 +1,5 @@
+
+module SidekiqSqsProcessor
+  VERSION = "0.1.0"
+end
+