pika_que 0.1.6 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a9831c848341b3207490dfa3050b377cea179be7
-  data.tar.gz: 7ac6527b9df72fb0e2ea7741fb2e3cff8c36a684
+SHA256:
+  metadata.gz: 9b776bf89fdc3f017c2d6f7b02a0c7cbf78e260b6d30917db25764b152f485c0
+  data.tar.gz: f7319738ad4c7ea1f0d205ebda1a7bf3b639f6ca1d85e77a5e73afa0d4fd909d
 SHA512:
-  metadata.gz: 82309e8a1fa9d93b76f5881f5693059720cc0b496b8a0c7ceacdd20594b6ff70b0575ea8bbf5a0117d0ebffe5243d5da734ca13ac5770ddbcd0b3b58792f6394
-  data.tar.gz: e7ba16c2307b6c53a89db6d51d1320b6d40c26066871aa1a4c54c63aa4f59ca585c121b2ca69aa1b7a13e2de3b62e0ee98fd743c7e1a7c3a42e7de4c8d5a9236
+  metadata.gz: c37d9efa4413e0785483e05f082c193caa40cb1d43459772621e1ac8ccebd1572853f97ff259e8aca8133c45dbad438c9a184a5bd441f56f0711a011f39d897f
+  data.tar.gz: 53a818627f9cbfd6bbb5eec5b148d1a2adb82b7ac48a7487e49f03ecfccb9cc73c9debc4ea0f4edc631621c61358a58f66c8e93a2fb16eadb0f054445c30b576

data/.github/workflows/ruby.yml

@@ -0,0 +1,38 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7', '3.0']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@55283cc23133118229fd3f97f9336ee23a179fcf # v1.146.0
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.ruby-version

@@ -0,0 +1 @@
+2.7.5

data/.travis.yml

@@ -1,4 +1,4 @@
 language: ruby
 rvm:
-  - 2.2.6
-before_install: gem install bundler -v 1.11.2
+  - 2.7.5
+before_install: gem install bundler -v 1.17.3

data/README.md

@@ -1,5 +1,7 @@
 # PikaQue
 
+![example workflow](https://github.com/dwkoogt/pika_que/actions/workflows/ruby.yml/badge.svg)
+
 A RabbitMQ background processing framework for Ruby with built-in support for Rails integration.
 
 PikaQue is inspired by Sneakers, Hutch, and Sidekiq. It is intended to implement more support for Rails a la Sidekiq.

data/examples/demo.rb

@@ -6,12 +6,12 @@ require 'pika_que/publisher'
 
 require 'dev_worker'
 require 'demo_worker'
-require 'demo_reporter'
+require 'demo_middleware'
 
 PikaQue.logger.level = ::Logger::DEBUG
 
 PikaQue.middleware do |chain|
-  chain.add DemoReporter
+  chain.add DemoMiddleware
 end
 
 workers = [DemoWorker,DevWorker]

data/examples/demo_conpriority.rb

@@ -33,10 +33,11 @@ class LowPriorityWorker
 
 end
 
-PikaQue.config.add_processor(workers: [LowPriorityWorker], concurrency: 10)
-PikaQue.config.add_processor(workers: [HighPriorityWorker], concurrency: 10)
+PikaQue.config[:processors] << { workers: [LowPriorityWorker], concurrency: 10 }
+PikaQue.config[:processors] << { workers: [HighPriorityWorker], concurrency: 10 }
 
 runner = PikaQue::Runner.new
+runner.setup_processors
 
 begin
   runner.run

data/examples/demo_delay.rb

@@ -16,10 +16,11 @@ end
 
 PikaQue.logger.level = ::Logger::DEBUG
 
-PikaQue.config.add_processor(PikaQue.config.delete(:delay_options))
-PikaQue.config.add_processor(workers: [DemoWorker])
+PikaQue.config[:delay] = true
+PikaQue.config[:processors] << { workers: [DemoWorker] }
 
 runner = PikaQue::Runner.new
+runner.setup_processors
 
 begin
   runner.run

data/examples/demo_dlx_retry.rb

@@ -0,0 +1,43 @@
+# > bundle exec ruby examples/demo_dlx_retry.rb
+# Retry using x-dead-letter-exchange
+# Constant backoff only
+require 'pika_que'
+require 'pika_que/worker'
+require 'pika_que/handlers/dlx_retry_handler'
+
+class DlxWorker
+  include PikaQue::Worker
+  from_queue "pika-que-dlx", :arguments => { :'x-dead-letter-exchange' => 'pika-que-retry-60' }
+  handle_with PikaQue::Handlers::DLXRetryHandler, retry_max_times: 3, retry_dlx: 'pika-que-retry-60'
+
+  def perform(msg)
+    logger.info msg["msg"]
+    raise "BOOM!"
+    ack!
+  end
+
+end
+
+PikaQue.logger.level = ::Logger::DEBUG
+
+workers = [DlxWorker]
+
+begin
+  pro = PikaQue::Processor.new(workers: workers)
+  pro.start
+rescue => e
+  puts e
+  puts e.backtrace.join("\n")
+end
+
+sleep 3
+
+DlxWorker.enqueue({ msg: "retry message" })
+
+sleep 200
+
+pro.stop
+
+puts "bye"
+
+exit 1

data/examples/demo_middleware.rb

@@ -0,0 +1,19 @@
+class DemoMiddleware
+
+  def initialize(opts = {})
+    STDOUT.sync = true
+  end
+  
+  def call(worker, delivery_info, metadata, msg)
+    puts "entering middleware DemoMiddleware for msg: #{msg}"
+    begin
+      yield
+    rescue => e
+      puts "error caught in middleware DemoMiddleware for msg: #{msg}, error: #{e.message}"
+      raise e
+    ensure
+      puts "leaving middleware DemoMiddleware for msg: #{msg}"
+    end
+  end
+
+end

data/examples/demo_retry.rb

@@ -1,4 +1,6 @@
 # > bundle exec ruby examples/demo_retry.rb
+# Retry with routing to a headers exchange
+# Select between constant backoff and exponential backoff modes
 require 'pika_que'
 require 'pika_que/worker'
 require 'pika_que/handlers/retry_handler'

data/lib/active_job/queue_adapters/pika_que_adapter.rb

@@ -1,5 +1,4 @@
-require 'pika_que'
-require 'pika_que/codecs/rails'
+require 'pika_que/rails_worker'
 require 'thread'
 
 module ActiveJob
@@ -20,21 +19,16 @@ module ActiveJob
       class << self
         def enqueue(job) #:nodoc:
           @monitor.synchronize do
-            JobWrapper.enqueue job.serialize, to_queue: job.queue_name
+            PikaQue::RailsWorker.enqueue job.serialize, to_queue: job.queue_name
           end
         end
 
         def enqueue_at(job, timestamp) #:nodoc:
           @monitor.synchronize do
-            JobWrapper.enqueue_at job.serialize, timestamp, routing_key: job.queue_name
+            PikaQue::RailsWorker.enqueue_at job.serialize, timestamp, routing_key: job.queue_name
           end
         end
       end
-
-      class JobWrapper #:nodoc:
-        extend PikaQue::Worker::ClassMethods
-        config codec: PikaQue::Codecs::RAILS
-      end
     end
 
     class PikaQueRails5
@@ -44,20 +38,15 @@ module ActiveJob
 
       def enqueue(job) #:nodoc:
         @monitor.synchronize do
-          JobWrapper.enqueue job.serialize, to_queue: job.queue_name
+          PikaQue::RailsWorker.enqueue job.serialize, to_queue: job.queue_name, priority: job.priority
         end
       end
 
       def enqueue_at(job, timestamp) #:nodoc:
         @monitor.synchronize do
-          JobWrapper.enqueue_at job.serialize, timestamp, routing_key: job.queue_name
+          PikaQue::RailsWorker.enqueue_at job.serialize, timestamp, routing_key: job.queue_name
         end
       end
-
-      class JobWrapper #:nodoc:
-        extend PikaQue::Worker::ClassMethods
-        config codec: PikaQue::Codecs::RAILS
-      end
     end
 
     PikaQueAdapter = (::Rails::VERSION::MAJOR < 5) ? PikaQueRails4 : PikaQueRails5

data/lib/pika_que/cli.rb

@@ -11,7 +11,7 @@ module PikaQue
 
     def parse(args = ARGV)
       opts = parse_options(args)
-      config.merge!(opts)
+      init_config(opts)
       init_logger
       daemonize
       write_pid
@@ -20,10 +20,9 @@ module PikaQue
     def run
 
       load_app
+      prepare_server
 
-      PikaQue.middleware
-
-      runner = Runner.new
+      runner = Runner.new.tap{ |r| r.setup_processors }
 
       begin
 
@@ -43,6 +42,13 @@ module PikaQue
       PikaQue.config
     end
 
+    def init_config(opts)
+      if opts[:config]
+        config.load(File.expand_path(opts[:config]))
+      end
+      config.merge!(opts)
+    end
+
     def init_logger
       PikaQue::Logging.init_logger(config[:logfile]) if config[:logfile]
       PikaQue.logger.level = ::Logger::WARN if config[:quiet]
@@ -101,17 +107,16 @@ module PikaQue
       else
         require(File.expand_path(config[:require])) || raise(ArgumentError, 'require returned false')
       end
+    end
 
-      if config[:delay]
-        config.add_processor(config.delete(:delay_options))
-      else
-        config.delete(:delay_options)
+    def prepare_server
+      PikaQue.middleware do |chain|
+        config[:middlewares].each{ |m| chain.add PikaQue::Util.constantize(m) } if config[:middlewares]
       end
 
-      if config[:workers]
-        config.add_processor({ workers: config.delete(:workers) })
+      PikaQue.reporters do |rptrs|
+        config[:reporters].each{ |r| rptrs << PikaQue::Util.constantize(r).new }
       end
-
     end
 
     def parse_options(args)
@@ -148,6 +153,14 @@ module PikaQue
           opts[:workers] = arg.split(",")
         end
 
+        o.on '--no-delay', "turn off delay processor" do |arg|
+          opts[:delay] = arg
+        end
+
+        o.on '-C', '--config PATH', "path to config yml file" do |arg|
+          opts[:config] = arg
+        end
+
         o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
           opts[:logfile] = arg
         end

data/lib/pika_que/configuration.rb

@@ -1,4 +1,5 @@
 require 'forwardable'
+require 'yaml'
 
 require 'pika_que/codecs/json'
 require 'pika_que/codecs/noop'
@@ -81,6 +82,12 @@ module PikaQue
       @config[:vhost] = AMQ::Settings.parse_amqp_url(@config[:amqp]).fetch(:vhost, '/')
     end
 
+    def load(filename)
+      loaded = YAML.load_file(filename)
+      converted = JSON.parse(JSON.dump(loaded), symbolize_names: true)
+      merge! converted
+    end
+
     def merge!(other = {})
       @config = deep_merge(@config, other)
     end
@@ -97,16 +104,5 @@ module PikaQue
       first.merge(second, &merger)
     end
 
-    def processor(opts = {})
-      {
-        :processor        => PikaQue::Processor,
-        :workers          => []
-      }.merge(opts)
-    end
-
-    def add_processor(opts = {})
-      @config[:processors] << processor(opts)
-    end
-
   end
 end

data/lib/pika_que/handlers/dlx_retry_handler.rb

@@ -0,0 +1,150 @@
+module PikaQue
+  module Handlers
+    class DLXRetryHandler
+
+      # Create following exchanges with retry_prefix = pika-que and default backoff
+      # pika-que-retry-60
+      # pika-que-retry-requeue
+      # pika-que-error
+      # and following queue
+      # pika-que-retry-60 (with default backoff)
+      #
+
+      DEFAULT_RETRY_OPTS = {
+        :retry_prefix             => 'pika-que',
+        :retry_max_times          => 5,
+        :retry_backoff            => 60,
+        :retry_backoff_multiplier => 1000,
+      }.freeze
+
+      def initialize(opts = {})
+        @opts = PikaQue.config.merge(DEFAULT_RETRY_OPTS).merge(opts)
+        @connection = opts[:connection] || PikaQue.connection
+        @channel = @connection.create_channel
+        @error_monitor = Monitor.new
+
+        @max_retries = @opts[:retry_max_times]
+        @backoff_multiplier = @opts[:retry_backoff_multiplier] # This is for example/dev/test
+
+        @retry_ex_name = @opts[:retry_dlx] || "#{@opts[:retry_prefix]}-retry-#{@opts[:retry_backoff]}"
+        @retry_name = "#{@opts[:retry_prefix]}-retry"
+        @requeue_name = "#{@opts[:retry_prefix]}-retry-requeue"
+        @error_name = "#{@opts[:retry_prefix]}-error"
+
+        @queue_name_lookup = {}
+
+        setup_exchanges
+        setup_queues
+      end
+
+      def bind_queue(queue, routing_key)
+        # bind the worker queue to requeue exchange
+        @queue_name_lookup[routing_key] = queue.name
+        queue.bind(@requeue_exchange, :routing_key => routing_key)
+      end
+
+      def handle(response_code, channel, delivery_info, metadata, msg, error = nil)
+        case response_code
+        when :ack
+          PikaQue.logger.debug "DLXRetryHandler acknowledge <#{msg}>"
+          channel.acknowledge(delivery_info.delivery_tag, false)
+        when :reject
+          PikaQue.logger.debug "DLXRetryHandler reject retry <#{msg}>"
+          handle_retry(channel, delivery_info, metadata, msg, :reject)
+        when :requeue
+          PikaQue.logger.debug "DLXRetryHandler requeue <#{msg}>"
+          channel.reject(delivery_info.delivery_tag, true)
+        else
+          PikaQue.logger.debug "DLXRetryHandler error retry <#{msg}>"
+          handle_retry(channel, delivery_info, metadata, msg, error)
+        end
+      end
+
+      def close
+        @channel.close unless @channel.closed?
+      end
+
+      private
+
+      def setup_exchanges
+        @retry_exchange, @error_exchange, @requeue_exchange = [@retry_ex_name, @error_name, @requeue_name].map do |name|
+          PikaQue.logger.debug "DLXRetryHandler creating exchange=#{name}"
+          @channel.exchange(name, :type => 'topic', :durable => exchange_durable?)
+        end
+      end
+
+      def setup_queues
+        bo = @opts[:retry_backoff]
+
+        PikaQue.logger.debug "DLXRetryHandler creating queue=#{@retry_name}-#{bo} x-dead-letter-exchange=#{@requeue_name}"
+        backoff_queue = @channel.queue("#{@retry_name}-#{bo}",
+                                      :durable => queue_durable?,
+                                      :arguments => {
+                                        :'x-dead-letter-exchange' => @requeue_name,
+                                        :'x-message-ttl' => bo * @backoff_multiplier
+                                      })
+        backoff_queue.bind(@retry_exchange, :routing_key => '#')
+
+        PikaQue.logger.debug "DLXRetryHandler creating queue=#{@error_name}"
+        @error_queue = @channel.queue(@error_name, :durable => queue_durable?)
+        @error_queue.bind(@error_exchange, :routing_key => '#')
+      end
+
+      def queue_durable?
+        @opts.fetch(:queue_options, {}).fetch(:durable, false)
+      end
+
+      def exchange_durable?
+        @opts.fetch(:exchange_options, {}).fetch(:durable, false)
+      end
+
+      def handle_retry(channel, delivery_info, metadata, msg, reason)
+        # +1 for the current attempt
+        num_attempts = failure_count(metadata[:headers], delivery_info) + 1
+        if num_attempts <= @max_retries
+          # Publish message to the x-dead-letter-exchange (ie. retry exchange)
+          PikaQue.logger.info "DLXRetryHandler msg=retrying, count=#{num_attempts}, headers=#{metadata[:headers] || {}}"
+          
+          channel.reject(delivery_info.delivery_tag, false)
+        else
+          PikaQue.logger.info "DLXRetryHandler msg=failing, retried_count=#{num_attempts - 1}, headers=#{metadata[:headers]}, reason=#{reason}"
+
+          publish_error(delivery_info, msg)
+          channel.acknowledge(delivery_info.delivery_tag, false)
+        end
+      end
+
+      # Uses the x-death header to determine the number of failures this job has
+      # seen in the past. This does not count the current failure. So for
+      # instance, the first time the job fails, this will return 0, the second
+      # time, 1, etc.
+      # @param headers [Hash] Hash of headers that Rabbit delivers as part of
+      #   the message
+      # @return [Integer] Count of number of failures.
+      def failure_count(headers, delivery_info)
+        if headers.nil? || headers['x-death'].nil?
+          0
+        else
+          queue_name = headers['x-first-death-queue'] || @queue_name_lookup[delivery_info.routing_key]
+          x_death_array = headers['x-death'].select do |x_death|
+            x_death['queue'] == queue_name
+          end
+          if x_death_array.count > 0 && x_death_array.first['count']
+            # Newer versions of RabbitMQ return headers with a count key
+            x_death_array.inject(0) {|sum, x_death| sum + x_death['count']}
+          else
+            # Older versions return a separate x-death header for each failure
+            x_death_array.count
+          end
+        end
+      end
+
+      def publish_error(delivery_info, msg)
+        @error_monitor.synchronize do
+          @error_exchange.publish(msg, routing_key: delivery_info.routing_key)
+        end
+      end
+
+    end
+  end
+end

data/lib/pika_que/handlers/error_handler.rb

@@ -3,19 +3,22 @@ module PikaQue
     class ErrorHandler
 
       DEFAULT_ERROR_OPTS = {
-        :exchange     => 'pika-que-error',
-        :exchange_options => { :type => :topic },
-        :queue        => 'pika-que-error',
-        :routing_key  => '#'
+        :error_prefix => 'pika-que'
       }.freeze
      
       def initialize(opts = {})
         @opts = PikaQue.config.merge(DEFAULT_ERROR_OPTS).merge(opts)
         @connection = @opts[:connection] || PikaQue.connection
         @channel = @connection.create_channel
-        @exchange = @channel.exchange(@opts[:exchange], type: exchange_type, durable: exchange_durable?)
-        @queue = @channel.queue(@opts[:queue], durable: queue_durable?)
-        @queue.bind(@exchange, routing_key: @opts[:routing_key])
+        error_ex_name = error_q_name = "#{@opts[:error_prefix]}-error"
+        if @opts[:queue]
+          # handle deprecated options
+          error_ex_name = @opts[:exchange]
+          error_q_name = @opts[:queue]
+        end
+        @exchange = @channel.exchange(error_ex_name, type: :topic, durable: exchange_durable?)
+        @queue = @channel.queue(error_q_name, durable: queue_durable?)
+        @queue.bind(@exchange, routing_key: '#')
         @monitor = Monitor.new
       end
 
@@ -36,7 +39,7 @@ module PikaQue
         else
           PikaQue.logger.debug "ErrorHandler publishing <#{msg}> to [#{@queue.name}]"
           publish(delivery_info, msg)
-          channel.reject(delivery_info.delivery_tag, false)
+          channel.acknowledge(delivery_info.delivery_tag, false)
         end
       end
 
@@ -54,10 +57,6 @@ module PikaQue
         @opts.fetch(:exchange_options, {}).fetch(:durable, false)
       end
 
-      def exchange_type
-        @opts.fetch(:exchange_options, {}).fetch(:type, :topic)
-      end
-
       def publish(delivery_info, msg)
         @monitor.synchronize do
           @exchange.publish(msg, routing_key: delivery_info.routing_key)

data/lib/pika_que/handlers/retry_handler.rb

@@ -138,12 +138,12 @@ module PikaQue
           end
 
           publish_retry(delivery_info, msg, { backoff: backoff_ttl, count: num_attempts })
-          channel.reject(delivery_info.delivery_tag, false)
+          channel.acknowledge(delivery_info.delivery_tag, false)
         else
           PikaQue.logger.info "RetryHandler msg=failing, retried_count=#{num_attempts - 1}, headers=#{metadata[:headers]}, reason=#{reason}"
 
           publish_error(delivery_info, msg)
-          channel.reject(delivery_info.delivery_tag, false)
+          channel.acknowledge(delivery_info.delivery_tag, false)
         end
       end
 

data/lib/pika_que/handlers.rb

@@ -4,6 +4,7 @@ module PikaQue
     autoload :DefaultHandler, 'pika_que/handlers/default_handler'
     autoload :ErrorHandler, 'pika_que/handlers/error_handler'
     autoload :RetryHandler, 'pika_que/handlers/retry_handler'
+    autoload :DLXRetryHandler, 'pika_que/handlers/dlx_retry_handler'
 
   end
 end

data/lib/pika_que/middleware/request_store.rb

@@ -0,0 +1,14 @@
+# Add manually if you need it
+module PikaQue
+  module Middleware
+    class RequestStore
+
+      def call(*args)
+        yield
+      ensure
+        ::RequestStore.clear!
+      end
+
+    end
+  end
+end

data/lib/pika_que/rails.rb

@@ -1,5 +1,5 @@
-require 'pika_que/worker'
-require 'pika_que/codecs/rails'
+require 'pika_que/rails_worker'
+require 'pika_que/util'
 
 module PikaQue
   class Rails < ::Rails::Engine
@@ -20,10 +20,10 @@ module PikaQue
     config.after_initialize do
       config_file = ::Rails.root.join('config').join('pika_que.yml')
       if File.exist? config_file
-        PIKA_QUE_CONFIG = YAML.load_file(config_file)
+        PIKA_QUE_CONFIG = YAML.load_file(config_file).deep_symbolize_keys
       else
         mailer_queue = (::Rails::VERSION::MAJOR < 5) ? ActionMailer::DeliveryJob.queue_name : ActionMailer::Base.deliver_later_queue_name
-        PIKA_QUE_CONFIG = { "processors" => [{ "workers" => [{ "queue" => ActiveJob::Base.queue_name }, { "queue" => mailer_queue.to_s }] }] }
+        PIKA_QUE_CONFIG = { processors: [{ workers: [{ queue: ActiveJob::Base.queue_name }, { queue: mailer_queue.to_s }] }] }
       end
 
       workers_dir = ::Rails.root.join('app').join('workers')
@@ -33,28 +33,30 @@ module PikaQue
         worker_files = []
       end
 
-      PIKA_QUE_CONFIG['processors'].each do |processor|
+      PIKA_QUE_CONFIG[:processors].each do |processor|
         workers = []
-        processor['workers'].each do |worker|
-          queue = worker['queue']
-          worker_name = worker['worker'] || "#{queue.underscore.classify}Worker"
-          Object.const_set(worker_name, Class.new do
-              include PikaQue::Worker
-              from_queue queue
-              config codec: PikaQue::Codecs::RAILS
-
-              def perform(msg)
-                ActiveJob::Base.execute msg
-                ack!
+        processor[:workers].each do |worker|
+          if worker.is_a? Hash
+            if worker[:worker]
+              worker_name = worker[:worker]
+            else
+              queue_name = worker[:queue_name] || worker[:queue]
+              queue_opts = worker[:queue_opts] || {}
+              worker_name = "#{queue_name.underscore.classify}Worker"
+              unless worker_files.detect{ |w| w =~ /#{worker_name.underscore}/ }
+                PikaQue::Util.register_worker_class(worker_name, PikaQue::RailsWorker, queue_name, queue_opts)
               end
             end
-          ) unless worker_files.detect{ |w| w =~ /#{worker_name.underscore}/ }
+          else
+            worker_name = worker
+          end
           workers << worker_name
         end
-        proc_args = processor.symbolize_keys
-        proc_args[:workers] = workers
-        PikaQue.logger.info "Adding rails processor: #{proc_args}"
-        PikaQue.config.add_processor(proc_args)
+        processor[:workers] = workers
+        unless PikaQue.config[:workers] || PikaQue.config[:config]
+          PikaQue.logger.info "Adding rails processor: #{processor}"
+          PikaQue.config[:processors] << processor
+        end
       end
     end
 

data/lib/pika_que/rails_worker.rb

@@ -0,0 +1,15 @@
+require 'pika_que/worker'
+require 'pika_que/codecs/rails'
+
+module PikaQue
+  class RailsWorker
+    include PikaQue::Worker
+    config codec: PikaQue::Codecs::RAILS
+
+    def perform(msg)
+      ActiveJob::Base.execute msg
+      ack!
+    end
+
+  end
+end

data/lib/pika_que/runner.rb

@@ -2,23 +2,51 @@ module PikaQue
   class Runner
 
     def run
-      run_config = {}
-
       # TODO anything to add to run_config?
+      run_config = {}
 
-      @processors = []
-      PikaQue.config[:processors].each do |processor_hash|
+      @processes = []
+      processors.each do |processor_hash|
         _processor = PikaQue::Util.constantize(processor_hash[:processor]).new(processor_hash.merge(run_config))
         _processor.start
-        @processors << _processor
+        @processes << _processor
       end
     end
 
     # halt? pause?
     def stop
-      @processors.each(&:stop)
+      @processes.each(&:stop)
       PikaQue.connection.disconnect!
     end
 
+    def setup_processors
+      add_processor(config[:delay_options]) if config[:delay]
+      if config[:workers]
+        add_processor({ workers: config[:workers] })
+      else
+        config[:processors].each{ |p| add_processor(p) }
+      end
+    end
+
+    def processor(opts = {})
+      {
+        :processor        => PikaQue::Processor,
+        :workers          => []
+      }.merge(opts)
+    end
+
+    def add_processor(opts = {})
+      classified_workers = { :workers => PikaQue::Util.worker_classes(opts[:workers]) }
+      processors << processor(opts.merge(classified_workers))
+    end
+
+    def processors
+      @processors ||= []
+    end
+
+    def config
+      PikaQue.config
+    end
+
   end
 end

data/lib/pika_que/util.rb

@@ -1,3 +1,5 @@
+require "dry/inflector"
+
 module PikaQue
   module Util
     extend self
@@ -13,5 +15,35 @@ module PikaQue
       end
     end
 
+    def register_worker_class(worker_name, base_class, queue_name, queue_opts = {}, handler_class = nil, handler_opts = {}, local_config = {})
+      Object.const_set(worker_name, Class.new(base_class) do
+          from_queue queue_name, queue_opts         
+          handle_with handler_class, handler_opts if handler_class
+          config local_config if local_config.any?
+        end
+      )
+    end
+
+    def worker_classes(workers = [])
+      return [] if workers.nil?
+      
+      workers.map do |worker|
+        if worker.is_a? Hash
+          if worker[:worker]
+            worker[:worker]
+          else
+            queue_name = worker[:queue_name] || worker[:queue]
+            "#{inflector.classify(inflector.underscore(queue_name))}Worker"
+          end
+        else
+          worker
+        end
+      end
+    end
+
+    def inflector
+      @inflector ||= Dry::Inflector.new
+    end
+
   end
 end

data/lib/pika_que/version.rb

@@ -1,3 +1,3 @@
 module PikaQue
-  VERSION = "0.1.6"
+  VERSION = "0.3.0"
 end

data/lib/pika_que.rb

@@ -42,14 +42,16 @@ module PikaQue
   end
 
   def self.reporters
-    config[:reporters] << PikaQue::Reporters::LogReporter.new if config[:reporters].empty?
-    config[:reporters]
+    @reporters ||= [PikaQue::Reporters::LogReporter.new]
+    yield @reporters if block_given?
+    @reporters
   end
 
   def self.reset!
     @config = nil
     @connection = nil
     @chain = nil
+    @reporters = nil
   end
 
 end

data/pika_que.gemspec

@@ -27,12 +27,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'bunny', '~> 2.6'
+  spec.add_dependency 'bunny', '~> 2.19.0'
   spec.add_dependency 'concurrent-ruby', '~> 1.0'
   spec.add_dependency 'json', '~> 1.8'
+  spec.add_dependency 'dry-inflector'
 
-  spec.add_development_dependency "bundler", "~> 1.11"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "simplecov", "~> 0.15"
+  spec.add_development_dependency "bundler", "~> 2.1.4"
+  spec.add_development_dependency "rake", "~> 12.3.0"
+  spec.add_development_dependency "rspec", "~> 3.7.0"
+  spec.add_development_dependency "simplecov", "~> 0.16.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pika_que
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.3.0
 platform: ruby
 authors:
 - Dong Wook Koo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-26 00:00:00.000000000 Z
+date: 2024-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: 2.19.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: 2.19.0
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -52,62 +52,76 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: dry-inflector
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: 2.1.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: 2.1.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 3.7.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 3.7.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: 0.16.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: 0.16.1
 description: Ruby background processor for RabbitMQ.
 email:
 - dwkoogt@gmail.com
@@ -116,8 +130,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -129,9 +145,10 @@ files:
 - examples/demo.rb
 - examples/demo_conpriority.rb
 - examples/demo_delay.rb
+- examples/demo_dlx_retry.rb
+- examples/demo_middleware.rb
 - examples/demo_oneoff.rb
 - examples/demo_priority.rb
-- examples/demo_reporter.rb
 - examples/demo_retry.rb
 - examples/demo_worker.rb
 - examples/dev_worker.rb
@@ -152,6 +169,7 @@ files:
 - lib/pika_que/handlers.rb
 - lib/pika_que/handlers/default_handler.rb
 - lib/pika_que/handlers/delay_handler.rb
+- lib/pika_que/handlers/dlx_retry_handler.rb
 - lib/pika_que/handlers/error_handler.rb
 - lib/pika_que/handlers/retry_handler.rb
 - lib/pika_que/launcher.rb
@@ -161,9 +179,11 @@ files:
 - lib/pika_que/metrics/null_metric.rb
 - lib/pika_que/middleware/active_record.rb
 - lib/pika_que/middleware/chain.rb
+- lib/pika_que/middleware/request_store.rb
 - lib/pika_que/processor.rb
 - lib/pika_que/publisher.rb
 - lib/pika_que/rails.rb
+- lib/pika_que/rails_worker.rb
 - lib/pika_que/reporters.rb
 - lib/pika_que/reporters/log_reporter.rb
 - lib/pika_que/runner.rb
@@ -191,8 +211,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Ruby background processor for RabbitMQ.

data/examples/demo_reporter.rb

@@ -1,19 +0,0 @@
-class DemoReporter
-
-  def initialize(opts = {})
-    STDOUT.sync = true
-  end
-  
-  def call(worker, delivery_info, metadata, msg)
-    puts "entering middleware DemoReporter for msg: #{msg}"
-    begin
-      yield
-    rescue => e
-      puts "error caught in middleware DemoReporter for msg: #{msg}, error: #{e.message}"
-      raise e
-    ensure
-      puts "leaving middleware DemoReporter for msg: #{msg}"
-    end
-  end
-
-end