disquo 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f9ffa4b76dc43b762c79ff560a981b71d8e6bfb
-  data.tar.gz: 4c320fe08f60dd1d9fc18a190e65ef73d06cff64
+  metadata.gz: e4a5a02702349eb8b6c55ccda7678d0ec5374f9d
+  data.tar.gz: 2793c75864c561a6d96833c7a171c687b0bda854
 SHA512:
-  metadata.gz: 143053e6aa72ef607a876baa1059457e1db32a75e300448529feb3363ec991919b5c70bae3c147b6ac0b9a3095af862fb4c9dd8962bca0b2702943d650c48984
-  data.tar.gz: ef07e3fe2b6c36d4a759d9564475881b43a9bb9319016440662e49b3e3de5aed64d912c5995a56b08abe4ddeb04d62f2cca09bf3ec3a3e9874e528e36297d7c1
+  metadata.gz: bd458037bec4658dd490234ca46ab75f82f3f74e40deb1e6a65d95eea6e3ddf49d3a7f0770885389830be92fed26124e3a66422fad584382fc21ec33ee9f0618
+  data.tar.gz: e0575ada72338c810ee9186104f75b1478d8518f7a91bd5e5ed2218282c1d5c52ea99988b69c8c0550197cbf04d5e2d89919d1a6a7175d21e261f5733572b707

data/.gitignore

@@ -0,0 +1 @@
+pkg/

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    disquo (0.3.0)
+    disquo (0.3.1)
       concurrent-ruby
       connection_pool
       disque

data/disquo.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = "disquo"
-  s.version     = "0.3.0"
+  s.version     = "0.3.1"
   s.platform    = Gem::Platform::RUBY
 
   s.licenses    = ["Apache-2.0"]

data/lib/disquo/worker.rb

@@ -1,6 +1,6 @@
 require 'disquo'
-require 'concurrent'
 require 'concurrent/executor/fixed_thread_pool'
+require 'concurrent/atomic/atomic_fixnum'
 
 class Disquo::Worker
   attr_reader :disque, :queues, :wait_time, :wait_count
@@ -10,19 +10,18 @@ class Disquo::Worker
   # @param [Hash] options
   # @option [Array<String>] :queues queues to watch. Default: ["default"]
   # @option [Integer] :concurrency the number of concurrent threads. Default: 10
-  # @option [Numeric] :wait_time maximum time (in seconds) to block for when retrieving next batch. Default: 1s
-  # @option [Integer] :wait_count the minimum number of jobs to wait for when retrieving next batch. Default: 100
-  def initialize(disque, queues: [Disquo::DEFAULT_QUEUE], concurrency: 10, wait_time: 1, wait_count: 100)
+  # @option [Numeric] :wait_time maximum time (in seconds) to wait for jobs when retrieving next batch. Default: 1s
+  def initialize(disque, queues: [Disquo::DEFAULT_QUEUE], concurrency: 10, wait_time: 1)
     @disque     = disque
     @queues     = Array(queues)
     @threads    = Concurrent::FixedThreadPool.new(concurrency)
+    @busy       = Concurrent::AtomicFixnum.new
     @wait_time  = wait_time
-    @wait_count = wait_count
   end
 
   # Run starts the worker
   def run
-    Disquo.logger.info "Starting worker - queues: #{queues.inspect}, concurrency: #{@threads.max_length}"
+    Disquo.logger.info "Worker starting - queues: #{queues.inspect}, concurrency: #{@threads.max_length}"
 
     begin
       run_cycle
@@ -30,14 +29,14 @@ class Disquo::Worker
       handle_exception(e)
     end until @stopped
 
+    Disquo.logger.info "Worker shutting down"
     @threads.shutdown
   end
 
   # Blocks until worker is stopped
   def wait(timeout = nil)
-    Disquo.logger.info "Waiting for worker shutdown"
     @threads.wait_for_termination(timeout)
-    Disquo.logger.info "Shutdown complete"
+    Disquo.logger.info "Worker shutdown complete"
   end
 
   # Stops the worker
@@ -50,57 +49,69 @@ class Disquo::Worker
   private
 
     def run_cycle
-      jobs = next_batch
-
+      jobs = Array(next_batch)
       until @stopped || jobs.empty?
         job = jobs.shift
-        perform(*job)
+        schedule(*job)
       end
     ensure
       requeue(jobs) unless jobs.nil? || jobs.empty?
     end
 
     def next_batch
+      count = @threads.max_length - @busy.value
+      if count < 1
+        sleep(wait_time.fdiv(2))
+        return
+      end
+
       jobs = disque.with do |cn|
-        cn.fetch from: queues, timeout: (wait_time*1000).to_i, count: wait_count
+        cn.fetch from: queues, timeout: (wait_time*1000).to_i, count: count
       end
+
       @is_down = nil
-      Array(jobs)
-    rescue => e
-      if !@is_down
-        @is_down = true
-        handle_exception(e, message: 'Error retrieving jobs:')
-      end
-      sleep(1)
-      []
+      jobs
+    rescue Errno::ECONNREFUSED => e
+      handle_disque_exception e, message: "Failed to retrieve jobs:", notrace: true
+      nil
     end
 
-    def perform(queue, job_id, payload)
+    def schedule(queue, job_id, payload)
+      @busy.increment
       @threads.post do
-        thread_id = Thread.current.object_id.to_s(36)
-        Disquo.logger.info { "Process #{payload} - thread: #{thread_id}, job: #{job_id}" }
-
         begin
-          class_name, args = Disquo.load_job(payload)
-
-          job = Object.const_get(class_name).new
-          job.disque = disque
-          job.queue  = queue
-          job.job_id = job_id
-          job.perform(*args)
-        rescue => e
-          handle_exception e, message: "Error processing #{payload} - thread: #{thread_id}, job: #{job_id}:"
-
-          disque.with {|cn| cn.call :nack, job_id }
-          return
+          perform(queue, job_id, payload)
+        ensure
+          @busy.decrement
         end
+      end
+    end
 
-        begin
-          disque.with {|cn| cn.call :ackjob, job_id }
-        rescue => e
-          handle_exception e, message: "Error ACKing #{payload} - thread: #{thread_id}, job: #{job_id}:"
-          return
-        end
+    def perform(queue, job_id, payload)
+      thread_id = Thread.current.object_id.to_s(36)
+      Disquo.logger.info { "Process #{payload} - thread: #{thread_id}, job: #{job_id}" }
+
+      begin
+        class_name, args = Disquo.load_job(payload)
+
+        job = Object.const_get(class_name).new
+        job.disque = disque
+        job.queue  = queue
+        job.job_id = job_id
+        job.perform(*args)
+      rescue => e
+        handle_exception e, message: "Error processing #{payload} - thread: #{thread_id}, job: #{job_id}:"
+
+        disque.with {|cn| cn.call :nack, job_id }
+        return
+      end
+
+      begin
+        disque.with {|cn| cn.call :ackjob, job_id }
+        @is_down = nil
+      rescue Errno::ECONNREFUSED => e
+        handle_disque_exception e, message: "Failed to ACK job #{job_id}:", notrace: true
+        retry unless @stopped
       end
     end
 
@@ -109,11 +120,19 @@ class Disquo::Worker
       disque.with {|cn| cn.call :enqueue, *ids }
     end
 
+    def handle_disque_exception(e, opts = {})
+      if !@is_down
+        @is_down = true
+        handle_exception(e, opts)
+      end
+      sleep(1)
+    end
+
     def handle_exception(e, opts = {})
       lines = [
         opts[:message],
         "#{e.class.name}: #{e.message}",
-        e.backtrace
+        (opts[:notrace] ? nil : e.backtrace),
       ].compact.flatten
 
       Disquo.logger.error lines.join("\n")

data/spec/disquo/worker_spec.rb

@@ -3,11 +3,12 @@ require 'spec_helper'
 RSpec.describe Disquo::Worker do
 
   subject do
-    described_class.new Disquo.connect, wait_time: 0.1, wait_count: 11, queues: Disquo::TEST::QUEUE
+    described_class.new Disquo.connect, wait_time: 0.1, queues: Disquo::TEST::QUEUE
   end
 
   it "should run/process/shutdown" do
     runner = Thread.new { subject.run }
+    runner.abort_on_exception = true
 
     # seed 200 jobs
     200.times {|n| TestJob.enqueue(n) }
@@ -33,12 +34,13 @@ RSpec.describe Disquo::Worker do
       queue: "__disquo_test__",
     )
     expect(Disquo::TEST::PERFORMED.last[:job_id]).to match(/^D\w+/)
+    expect(Disquo::TEST::PERFORMED.map {|e| e[:args].first }).to match_array(0..199)
   end
 
   def wait_for
     20.times do
       break if yield
-      sleep(0.01)
+      sleep(0.1)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disquo
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Dimitrij Denissenko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-23 00:00:00.000000000 Z
+date: 2017-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: disque
@@ -87,6 +87,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - ".travis.yml"
 - Gemfile
 - Gemfile.lock