sidekiq-reliable-fetch 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35c020c4d73f87e39a4b5fa52930e42505c8e866
-  data.tar.gz: 1d344483e74a309efe9db2355450f20a0d5f4487
+  metadata.gz: 803136d2fb9cf9eb4f60b0b54618e489616d763a
+  data.tar.gz: 8371867f1f2ce011d710d91dd0d2a3a09ab1c739
 SHA512:
-  metadata.gz: 755c4623df1d7d672bb4c857ca53cb4b2e12d9d3e0f1cbbb37b7eb42a4f93dbba639bf91f8588607e7570eef27d50996e637f3a71d6920a8aaacdb62b9d273f5
-  data.tar.gz: 6c76f3015bfc44f03285d151625cf3ee88024b97c8df6c970b72140290948b8625a550fccbf93f90f3291483175dd382d63f8527742269b54046271ac3be62e6
+  metadata.gz: ac47ebe55522819896e610f2b8bc59f69026d23967fe5611c32e6383c13ac14732ebd4f7d1884368ac43e967f958ff4b3441178407e2bdedbb8cecb00f9d55f8
+  data.tar.gz: 2a8f9651f266ddb272fbba06f149b7879b9f406f92a90ea74b09f9772048f900c147a48832dc7d47374093742281073065b72aeb2a660165e067b3eb0170d7c8

data/README.md

@@ -0,0 +1,32 @@
+sidekiq-reliable-fetch
+======================
+
+sidekiq-reliable-fetch is an extension to Sidekiq that adds support for reliable
+fetches from Redis.
+
+It implements in Sidekiq the reliable queue pattern using [Redis' rpoplpush
+command](http://redis.io/commands/rpoplpush#pattern-reliable-queue).
+
+## Installation
+
+Add the following to your `Gemfile`:
+
+```ruby
+gem 'sidekiq-reliable-fetch'
+```
+
+## Configuration
+
+Enable reliable fetches by calling this gem from your Sidekiq configuration:
+
+```ruby
+Sidekiq.configure_server do |config|
+  Sidekiq::ReliableFetcher.setup_reliable_fetch!(config)
+
+  # …
+end
+```
+
+## License
+
+LGPL-3.0, see the LICENSE file.

data/lib/sidekiq/reliable_fetcher.rb

@@ -2,7 +2,8 @@ module Sidekiq
   class ReliableFetcher
     WORKING_QUEUE = 'working'
     DEFAULT_DEAD_AFTER = 60 * 60 * 24 # 24 hours
-    DEFAULT_CLEANING_INTERVAL = 60 * 30 # 30 minutes
+    DEFAULT_CLEANING_INTERVAL = 5000 # clean each N processed jobs
+    IDLE_TIMEOUT = 5 # seconds
 
     def self.setup_reliable_fetch!(config)
       config.options[:fetch] = Sidekiq::ReliableFetcher
@@ -12,20 +13,35 @@ module Sidekiq
     end
 
     def initialize(options)
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
-      @unique_queues = @queues.uniq
-      @last_clean = Time.now.to_i
+      queues = options[:queues].map { |q| "queue:#{q}" }
+
+      @unique_queues = queues.uniq
+      @queues_iterator = queues.shuffle.cycle
+      @queues_size  = queues.size
+
+      @nb_fetched_jobs = 0
       @cleaning_interval = options[:cleaning_interval] || DEFAULT_CLEANING_INTERVAL
       @consider_dead_after = options[:consider_dead_after] || DEFAULT_DEAD_AFTER
     end
 
     def retrieve_work
-      clean_working_queues! if @cleaning_interval != -1 && Time.now.to_i - @last_clean > @cleaning_interval
+      clean_working_queues! if @cleaning_interval != -1 && @nb_fetched_jobs >= @cleaning_interval
+
+      for i in 0..@queues_size
+        queue = @queues_iterator.next
+        work = Sidekiq.redis { |conn| conn.rpoplpush(queue, "#{queue}:#{WORKING_QUEUE}") }
+
+        if work
+          @nb_fetched_jobs += 1
+          return UnitOfWork.new(queue, work)
+        end
+      end
 
-      queue = pick_queue
-      work = Sidekiq.redis { |conn| conn.brpoplpush(queue, "#{queue}:#{WORKING_QUEUE}", Sidekiq::Fetcher::TIMEOUT) }
-      UnitOfWork.new(queue, work) if work
+      # We didn't find a job in any of the configured queues. Let's sleep a bit
+      # to avoid uselessly burning too much CPU
+      sleep(IDLE_TIMEOUT)
+
+      nil
     end
 
     def self.requeue_on_startup!(queues)
@@ -90,16 +106,6 @@ module Sidekiq
 
     private
 
-    # Creating the Redis#brpoplpush command takes into account any
-    # configured queue weights. By default Redis#brpoplpush returns
-    # data from the first queue that has pending elements. We
-    # choose the queue each time we invoke Redis#brpoplpush to honor weights
-    # and avoid queue starvation.
-    def pick_queue
-      queues = @strictly_ordered_queues ? @unique_queues.dup : @queues.shuffle.uniq
-      queues.first
-    end
-
     # Detect "old" jobs and requeue them because the worker they were assigned
     # to probably failed miserably.
     # NOTE Potential problem here if a specific job always make a worker
@@ -111,7 +117,7 @@ module Sidekiq
         clean_working_queue!(queue)
       end
 
-      @last_clean = Time.now.to_i
+      @nb_fetched_jobs = 0
     end
 
     def clean_working_queue!(queue)
@@ -121,12 +127,12 @@ module Sidekiq
           enqueued_at = Sidekiq.load_json(job)['enqueued_at'].to_i
           job_duration = Time.now.to_i - enqueued_at
 
-          if job_duration > @consider_dead_after
-            Sidekiq.logger.info "Requeued a dead job found in #{queue}:#{WORKING_QUEUE}"
+          next if job_duration < @consider_dead_after
 
-            conn.rpush("#{queue}", job)
-            conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job)
-          end
+          Sidekiq.logger.info "Requeued a dead job from #{queue}:#{WORKING_QUEUE}"
+
+          conn.rpush("#{queue}", job)
+          conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job)
         end
       end
     end

data/lib/sidekiq-reliable-fetch/web.rb

@@ -7,7 +7,6 @@ module SidekiqReliableFetch
 
     def self.registered(app)
       app.get '/working' do
-        puts VIEW_PATH.inspect
         @queues = SidekiqReliableFetch::WorkingQueue.all
         erb File.read(File.join(VIEW_PATH, 'working_queues.erb'))
       end

data/sidekiq-reliable-fetch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'sidekiq-reliable-fetch'
-  s.version     = '0.1.0'
+  s.version     = '0.2.0'
   s.authors     = ['TEA']
   s.email       = 'technique@tea-ebook.com'
   s.license     = 'LGPL-3.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-reliable-fetch
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - TEA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-26 00:00:00.000000000 Z
+date: 2016-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -32,6 +32,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - LICENSE
+- README.md
 - lib/sidekiq-reliable-fetch.rb
 - lib/sidekiq-reliable-fetch/api.rb
 - lib/sidekiq-reliable-fetch/web.rb