pika_que 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: faba17eaee828074e195258b47a178d6e90e8674
-  data.tar.gz: '04959436f6a2e742b7cea6ba805d4b10f0b187ca'
+  metadata.gz: a9831c848341b3207490dfa3050b377cea179be7
+  data.tar.gz: 7ac6527b9df72fb0e2ea7741fb2e3cff8c36a684
 SHA512:
-  metadata.gz: e5dc91623fa960d2eb130798ee6a1f89ae7fe2a5fe2199201b1c92bea7e55d6b53512a3a14b373d2bb8b7361a14777f57073aa73cfd33dea678fe0d8d2de6690
-  data.tar.gz: 9c70e4d7229bcb37409144d4905e9a94e9d6690e9a9babb17a2f397d3e384f4d5d48557702edc885c11c5ff09ea8636bf180a0e9f1bc9df73caa57fbff161975
+  metadata.gz: 82309e8a1fa9d93b76f5881f5693059720cc0b496b8a0c7ceacdd20594b6ff70b0575ea8bbf5a0117d0ebffe5243d5da734ca13ac5770ddbcd0b3b58792f6394
+  data.tar.gz: e7ba16c2307b6c53a89db6d51d1320b6d40c26066871aa1a4c54c63aa4f59ca585c121b2ca69aa1b7a13e2de3b62e0ee98fd743c7e1a7c3a42e7de4c8d5a9236

data/.rspec

@@ -1,2 +1,2 @@
---format documentation
+--format progress
 --color

data/README.md

@@ -24,12 +24,10 @@ To create a worker:
 ```ruby
 class PokeWorker
   include PikaQue::Worker
-  from_queue "rocket"
+  from_queue "poke"
 
   def perform(msg)
     # do something with msg["greeting"]
-    # or below for active job
-    ActiveJob::Base.execute msg
     ack!
   end
 end
@@ -45,21 +43,27 @@ To run server:
 
     $ bundle exec pika_que
     
-### Rails and ActiveJob
 
-Create workers in:
+### Rails and ActiveJob Quickstart
+
+Create workers(not required for ActiveJob) in:
 
     app/workers
     
-Create a initializer file `pika_que.rb` in:
+Create a config file `pika_que.yml` in config:
 
-    config/initializers
+    config/pika_que.yml
     
-```ruby
-# pika_que.rb
+```yml
+# pika_que.yml
+processors:
+  - workers:
+    - queue: default
+    - queue: mailers
+    - queue: your-active-job-queue-name
+  - workers:
+    - worker: PokeWorker
 
-# setup workers here. see source for available options
-PikaQue.config.add_processor(workers: [PokeWorker])
 ```
 
 Set the backend for active job in `config/application.rb`:
@@ -68,9 +72,11 @@ Set the backend for active job in `config/application.rb`:
 
 Then run the server.
 
-## Specs
+For more details, see [wiki](https://github.com/dwkoogt/pika_que/wiki/Rails-Setup).
+
+### Examples
 
-Coming soon. See examples for reference.
+See examples for more usage reference.
 
 ## Development
 

data/examples/demo_conpriority.rb

@@ -0,0 +1,62 @@
+# > bundle exec ruby examples/demo_conpriority.rb
+# https://www.rabbitmq.com/consumer-priority.html
+# https://www.rabbitmq.com/blog/2013/12/16/using-consumer-priorities-with-rabbitmq/
+#
+# Consumer Priority
+# HighPriorityWorker will process more messages than LowPriorityWorker
+#
+require 'pika_que'
+require 'pika_que/worker'
+require 'pika_que/runner'
+
+PikaQue.logger.level = ::Logger::DEBUG
+
+class HighPriorityWorker
+  include PikaQue::Worker
+  from_queue "pika-que-demo", :priority => 10
+
+  def perform(msg)
+    logger.info "HighPriorityWorker #{msg['msg']}"
+    ack!
+  end
+
+end
+
+class LowPriorityWorker
+  include PikaQue::Worker
+  from_queue "pika-que-demo", :priority => 1
+
+  def perform(msg)
+    logger.info "LowPriorityWorker #{msg['msg']}"
+    ack!
+  end
+
+end
+
+PikaQue.config.add_processor(workers: [LowPriorityWorker], concurrency: 10)
+PikaQue.config.add_processor(workers: [HighPriorityWorker], concurrency: 10)
+
+runner = PikaQue::Runner.new
+
+begin
+  runner.run
+rescue => e
+  puts e
+  puts e.backtrace.join("\n")
+end
+
+sleep 3
+
+pub = PikaQue::Publisher.new()
+
+600.times do |i|
+  pub.publish({ msg: "hello world #{i}" }, routing_key: 'pika-que-demo')
+end
+
+sleep 3
+
+runner.stop
+
+puts "bye"
+
+exit 1

data/examples/demo_priority.rb

@@ -1,35 +1,25 @@
 # > bundle exec ruby examples/demo_priority.rb
+#
+# Priority Queue with message priorities
+#
 require 'pika_que'
 require 'pika_que/worker'
 
 PikaQue.logger.level = ::Logger::DEBUG
 
-class HighPriorityWorker
+class PriorityWorker
   include PikaQue::Worker
-  from_queue "pika-que-priority", :arguments => { :'x-max-priority' => 10 }, :priority => 10
+  from_queue "pika-que-priority", :arguments => { :'x-max-priority' => 10 }
 
   def perform(msg)
-    logger.info msg["msg"]
+    logger.info msg['msg']
     ack!
   end
 
 end
 
-class LowPriorityWorker
-  include PikaQue::Worker
-  from_queue "pika-que-priority", :arguments => { :'x-max-priority' => 10 }, :priority => 1
-
-  def perform(msg)
-    logger.info msg["msg"]
-    ack!
-  end
-
-end
-
-workers = [HighPriorityWorker,LowPriorityWorker]
-
 begin
-  pro = PikaQue::Processor.new(workers: workers, concurrency: 10)
+  pro = PikaQue::Processor.new(workers: [PriorityWorker], concurrency: 2)
   pro.start
 rescue => e
   puts e
@@ -38,9 +28,10 @@ end
 
 sleep 3
 
+pub = PikaQue::Publisher.new()
 300.times do |i|
-  LowPriorityWorker.enqueue({ msg: "low priority #{i}" })
-  HighPriorityWorker.enqueue({ msg: "high priority #{i}" })
+  prty = (i % 2) == 0 ? 1 : 10
+  pub.publish({ msg: "hello world #{i} priority #{prty}" }, routing_key: 'pika-que-priority', priority: prty)
 end
 
 sleep 3

data/lib/pika_que.rb

@@ -46,6 +46,12 @@ module PikaQue
     config[:reporters]
   end
 
+  def self.reset!
+    @config = nil
+    @connection = nil
+    @chain = nil
+  end
+
 end
 
 require 'pika_que/rails' if defined?(::Rails::Engine)

data/lib/pika_que/cli.rb

@@ -45,7 +45,7 @@ module PikaQue
 
     def init_logger
       PikaQue::Logging.init_logger(config[:logfile]) if config[:logfile]
-      PikaQue.logger.level = ::Logger::WARN if config[:quite]
+      PikaQue.logger.level = ::Logger::WARN if config[:quiet]
       PikaQue.logger.level = ::Logger::DEBUG if config[:verbose]
     end
 
@@ -132,8 +132,8 @@ module PikaQue
           opts[:environment] = arg
         end
 
-        o.on '-q', '--quite', "Print quite output" do |arg|
-          opts[:quite] = arg
+        o.on '-q', '--quiet', "Print quiet output" do |arg|
+          opts[:quiet] = arg
         end
 
         o.on '-v', '--verbose', "Print verbose output" do |arg|

data/lib/pika_que/configuration.rb

@@ -76,7 +76,7 @@ module PikaQue
     # }
 
     def initialize
-      @config = DEFAULT_CONFIG.dup
+      @config = Marshal.load(Marshal.dump(DEFAULT_CONFIG))
       @config[:amqp]  = ENV.fetch('RABBITMQ_URL', 'amqp://guest:guest@localhost:5672')
       @config[:vhost] = AMQ::Settings.parse_amqp_url(@config[:amqp]).fetch(:vhost, '/')
     end

data/lib/pika_que/handlers/error_handler.rb

@@ -36,7 +36,7 @@ module PikaQue
         else
           PikaQue.logger.debug "ErrorHandler publishing <#{msg}> to [#{@queue.name}]"
           publish(delivery_info, msg)
-          channel.acknowledge(delivery_info.delivery_tag, false)
+          channel.reject(delivery_info.delivery_tag, false)
         end
       end
 

data/lib/pika_que/handlers/retry_handler.rb

@@ -72,7 +72,7 @@ module PikaQue
 
       #####################################################
       # formula
-      # base X = 0, 30, 60, 120, 180, etc defaults to 0
+      # base X = 0, 15(2x), 30(3x), 45(4x), 60(5x), 120, 180, etc defaults to 0
       # (X + 15) * 2 ** (count + 1)
       def self.backoff_periods(max_retries, backoff_base)
         (1..max_retries).map{ |c| next_ttl(c, backoff_base) }
@@ -95,7 +95,12 @@ module PikaQue
 
       def setup_queues
         if @opts[:retry_mode] == :const
-          bo = @opts[:retry_const_backoff]
+          backoffs = [@opts[:retry_const_backoff]]
+        else
+          backoffs = RetryHandler.backoff_periods(@max_retries, @backoff_base)
+        end
+
+        backoffs.each do |bo|
           PikaQue.logger.debug "RetryHandler creating queue=#{@retry_name}-#{bo} x-dead-letter-exchange=#{@requeue_name}"
           backoff_queue = @channel.queue("#{@retry_name}-#{bo}",
                                         :durable => queue_durable?,
@@ -104,18 +109,6 @@ module PikaQue
                                           :'x-message-ttl' => bo * @backoff_multiplier
                                         })
           backoff_queue.bind(@retry_exchange, :arguments => { :backoff => bo })
-        else
-          backoffs = RetryHandler.backoff_periods(@max_retries, @backoff_base)
-          backoffs.each do |bo|
-            PikaQue.logger.debug "RetryHandler creating queue=#{@retry_name}-#{bo} x-dead-letter-exchange=#{@requeue_name}"
-            backoff_queue = @channel.queue("#{@retry_name}-#{bo}",
-                                          :durable => queue_durable?,
-                                          :arguments => {
-                                            :'x-dead-letter-exchange' => @requeue_name,
-                                            :'x-message-ttl' => bo * @backoff_multiplier
-                                          })
-            backoff_queue.bind(@retry_exchange, :arguments => { :backoff => bo })
-          end
         end
 
         PikaQue.logger.debug "RetryHandler creating queue=#{@error_name}"
@@ -147,7 +140,7 @@ module PikaQue
           publish_retry(delivery_info, msg, { backoff: backoff_ttl, count: num_attempts })
           channel.reject(delivery_info.delivery_tag, false)
         else
-          PikaQue.logger.info "RetryHandler msg=failing, retry_count=#{num_attempts}, headers=#{metadata[:headers]}, reason=#{reason}"
+          PikaQue.logger.info "RetryHandler msg=failing, retried_count=#{num_attempts - 1}, headers=#{metadata[:headers]}, reason=#{reason}"
 
           publish_error(delivery_info, msg)
           channel.reject(delivery_info.delivery_tag, false)

data/lib/pika_que/launcher.rb

@@ -80,10 +80,10 @@ module PikaQue
       Thread.list.each do |thread|
         logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
         if thread.backtrace
-            logger.warn thread.backtrace.join("\n")
-          else
-            logger.warn "<no backtrace available>"
-          end
+          logger.warn thread.backtrace.join("\n")
+        else
+          logger.warn "<no backtrace available>"
+        end
       end
     end
 

data/lib/pika_que/metrics.rb

@@ -11,7 +11,7 @@ module PikaQue
     def self.init_metrics
       if PikaQue.config[:metrics]
         @metrics = PikaQue.config[:metrics].new
-      elsif PikaQue.config[:quite]
+      elsif PikaQue.config[:quiet]
         @metrics = PikaQue::Metrics::NullMetric.new
       else
         @metrics = PikaQue::Metrics::LogMetric.new

data/lib/pika_que/processor.rb

@@ -29,7 +29,7 @@ module PikaQue
         Thread.current['label'] = 'processor'
         setup
         process
-      end.abort_on_exception = true
+      end.tap{ |t| t.abort_on_exception = true }
     end
 
     def stop

data/lib/pika_que/publisher.rb

@@ -21,5 +21,9 @@ module PikaQue
       @exchange.publish(msg, options)
     end
 
+    def exchange_name
+      @opts[:exchange]
+    end
+
   end
 end

data/lib/pika_que/subscriber.rb

@@ -32,36 +32,40 @@ module PikaQue
       @consumer = queue.subscribe(:block => false, :manual_ack => @opts[:ack], :arguments => worker.consumer_arguments) do | delivery_info, metadata, msg |
         # TODO make idletime configurable on thread pool? default is 60.
         pool.post do
-          res = nil
-          error = nil
-          begin
-            decoded_msg = @codec.decode(msg)
-            metrics.measure("work.#{worker.class.name}.time") do
-              PikaQue.middleware.invoke(worker, delivery_info, metadata, decoded_msg) do
-                res = worker.work(delivery_info, metadata, decoded_msg)
-              end
-            end
-            logger.debug "done processing #{res} <#{msg}>"
-          rescue => worker_err
-            res = :error
-            error = worker_err
-            notify_reporters(worker_err, worker.class, msg)
-          end
+          handle_message(worker, delivery_info, metadata, msg)
+        end
+      end
+    end
 
-          if @opts[:ack]
-            begin
-              handler.handle(res, broker.channel, delivery_info, metadata, msg, error)
-              metrics.increment("work.#{worker.class.name}.handled.#{res}") 
-            rescue => handler_err
-              notify_reporters(handler_err, handler.class, msg)
-              metrics.increment("work.#{worker.class.name}.handler.error") 
-            end
-          else
-            metrics.increment("work.#{worker.class.name}.handled.noop") 
+    def handle_message(worker, delivery_info, metadata, msg)
+      res = nil
+      error = nil
+      begin
+        decoded_msg = @codec.decode(msg)
+        metrics.measure("work.#{worker.class.name}.time") do
+          PikaQue.middleware.invoke(worker, delivery_info, metadata, decoded_msg) do
+            res = worker.work(delivery_info, metadata, decoded_msg)
           end
-          metrics.increment("work.#{worker.class.name}.processed")
         end
+        logger.debug "done processing #{res} <#{msg}>"
+      rescue => worker_err
+        res = :error
+        error = worker_err
+        notify_reporters(worker_err, worker.class, msg)
+      end
+
+      if @opts[:ack]
+        begin
+          handler.handle(res, broker.channel, delivery_info, metadata, msg, error)
+          metrics.increment("work.#{worker.class.name}.handled.#{res}") 
+        rescue => handler_err
+          notify_reporters(handler_err, handler.class, msg)
+          metrics.increment("work.#{worker.class.name}.handler.error") 
+        end
+      else
+        metrics.increment("work.#{worker.class.name}.handled.noop") 
       end
+      metrics.increment("work.#{worker.class.name}.processed")
     end
 
     def unsubscribe

data/lib/pika_que/version.rb

@@ -1,3 +1,3 @@
 module PikaQue
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

data/lib/pika_que/worker.rb

@@ -75,7 +75,7 @@ module PikaQue
       alias_method :perform_async, :enqueue
 
       def enqueue_at(msg, timestamp, opts={})
-        opts[:to_queue] ||= "#{PikaQue.config[:exchange]}-delay"
+        opts[:to_queue] ||= "#{publisher.exchange_name}-delay"
         work_queue = opts.delete(:routing_key) || (queue_opts[:routing_key] if queue_opts) || queue_name
         opts[:headers] = { work_at: timestamp, work_queue: work_queue }
 

data/pika_que.gemspec

@@ -34,4 +34,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "simplecov", "~> 0.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pika_que
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Dong Wook Koo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-10 00:00:00.000000000 Z
+date: 2018-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
 description: Ruby background processor for RabbitMQ.
 email:
 - dwkoogt@gmail.com
@@ -113,6 +127,7 @@ files:
 - bin/console
 - bin/setup
 - examples/demo.rb
+- examples/demo_conpriority.rb
 - examples/demo_delay.rb
 - examples/demo_oneoff.rb
 - examples/demo_priority.rb
@@ -177,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2.2
+rubygems_version: 2.4.5.2
 signing_key: 
 specification_version: 4
 summary: Ruby background processor for RabbitMQ.