atomic-sidekiq 1.1.4 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4cfc4a556c16ba7797344d6090c0e1fdd095ccab9e45e3b167c58e690ec4d31
-  data.tar.gz: ba1fdea411cf2e600bb5aef2843a9be83f36dbff421248dc94977fa85e391cc2
+  metadata.gz: aeb25fc1520ae4298c7643e5e012bd08afa767332db6422f1cbf5a9b2e2569af
+  data.tar.gz: d4588254c282b2e07c9c35858c351436d848be87d2b94b2bf295b489f71b4f5b
 SHA512:
-  metadata.gz: 46f3d12bb8320fc29f0fe546f43fd804445c48908e61daf400b3ec489ee174b52cd34c48a9da1daf9edcd846b7349c94fb2845118b5ba466a90aa453a91969fe
-  data.tar.gz: c4f0561f8441d8bddeab82fa4e10c8b26b8de6e16e77ec20caab0b416d479db618dfc2475be3db589f4cb2704fc39c58995c97d39a3ee7bc1b5971090abc6453
+  metadata.gz: ced404ab64bc0598c8bcabbcf9e75268f4351e0900de85468e655178f4cb0f2cda9481f93725529fbccfdc6d25cd1c9d66052a16f5f54b0ccda496e85c1deee6
+  data.tar.gz: 05cdd96f7a3ab054a8a55128913738ed8baa2ef3377d5a85b6fa87d1422322d92a1b7a80ccae96bf6fcb3e4ec31026ebdb516373cbede0a98dfbfc442966753d

data/.rubocop.yml

@@ -22,6 +22,9 @@ Style/Documentation:
 Metrics/AbcSize:
   Max: 20
 
+Metrics/MethodLength:
+  Max: 20
+
 Metrics/BlockLength:
   Exclude:
     - spec/**/*.rb

data/README.md

@@ -21,12 +21,15 @@ end
 ```
 
 ## Configuration
-By default, jobs will expire and be re-queued after 1 hour if not acknowledged, and the "Collector" will check if for expired jobs every 60 seconds. This can be reconfigured as desired: _(Note that collection adds some overhead)_
+By default, jobs will expire and be re-queued after 1 hour if not acknowledged, and the "Collector" will check if for expired jobs every 60 seconds. This can be reconfigured as desired: _(Note that collection adds some overhead)_. You
+may also configure which queues shouldn't be recovered using the property `ignored_queues`. Jobs in those queues will
+be expired without being pushed into the queue again.
 ```ruby
 Sidekiq.configure_server do |config|
   config.atomic_fetch!({
-    collection_interval: 5, # Unit: seconds
-    expiration_time: 1800   # Unit: seconds (30 minutes)
+    collection_interval: 5,   # Unit: seconds
+    expiration_time: 1800,    # Unit: seconds (30 minutes)
+    ignored_queues: ['cron'],
   })
 end
 ```

data/atomic-sidekiq.gemspec

@@ -2,8 +2,8 @@
 
 Gem::Specification.new do |s|
   s.name          = "atomic-sidekiq"
-  s.version       = "1.1.4"
-  s.date          = "2018-12-18"
+  s.version       = "1.2.0"
+  s.date          = "2018-12-20"
   s.summary       = "Reliable fetcher for Sidekiq"
   s.description   = "Reliable fetcher for Sidekiq"
   s.homepage      = "https://github.com/Colex/atomic-sidekiq"

data/lib/atomic_sidekiq/atomic_fetch.rb

@@ -25,14 +25,16 @@ module AtomicSidekiq
       collect_dead_jobs!
       work = retrieve_op.perform(ordered_queues, expire_at)
       return UnitOfWork.new(*work, in_flight_keymaker: keymaker) if work
+
       sleep(poll_interval)
       nil
     end
 
     private
 
-    attr_reader :retrieve_op, :queues, :strictly_ordered_queues,
-                :collection_interval, :poll_interval, :expiration_time,
+    attr_reader :retrieve_op, :queues, :ignored_queues,
+                :strictly_ordered_queues, :collection_interval,
+                :poll_interval, :expiration_time,
                 :keymaker
 
     def configure_atomic_fetch(options)
@@ -40,6 +42,8 @@ module AtomicSidekiq
       @collection_interval = options[:collection_wait_time] ||
                              DEFAULT_COLLECTION_INTERVAL
       @poll_interval = options[:poll_interval] || DEFAULT_POLL_INTERVAL
+      @ignored_queues = (options[:ignored_queues] || [])
+                        .map { |q| "queue:#{q}" }
     end
 
     def ordered_queues
@@ -52,8 +56,13 @@ module AtomicSidekiq
 
     def collect_dead_jobs!
       return if @@next_collection > Time.now
+
       @@next_collection = Time.now + collection_interval
-      DeadJobCollector.collect!(ordered_queues, in_flight_keymaker: keymaker)
+      DeadJobCollector.collect!(
+        ordered_queues,
+        in_flight_keymaker: keymaker,
+        skip_recovery_queues: ignored_queues
+      )
     end
 
     def expire_at

data/lib/atomic_sidekiq/atomic_operation/expire.rb

@@ -10,9 +10,19 @@ module AtomicSidekiq
         super(in_flight_keymaker: nil)
       end
 
-      def perform(queue, in_flight_key)
+      def perform(queue, in_flight_key, recover:)
         redis do |conn|
-          conn.eval(EXPIRE_SCRIPT, [queue, in_flight_key], [Time.now.utc.to_i])
+          conn.eval(
+            EXPIRE_SCRIPT,
+            [
+              queue, # Queue Name
+              in_flight_key, # Key of the inflight job being expired
+            ],
+            [
+              Time.now.utc.to_i, # Current time
+              recover, # Boolean flag: should it be recovered if expired
+            ]
+          )
         end
       end
     end

data/lib/atomic_sidekiq/atomic_operation/heartbeat.rb

@@ -9,6 +9,7 @@ module AtomicSidekiq
       def perform(jid:, timeout:)
         key = in_flight_job_key(jid)
         return unless key
+
         redis do |conn|
           conn.eval(HEARTBEAT_SCRIPT, [key], [expiration_date(timeout)])
         end
@@ -26,6 +27,7 @@ module AtomicSidekiq
         loop do
           it, keys = redis { |conn| conn.scan(it, match: matcher) }
           return keys[0] if keys.count > 0
+
           it = it.to_i
           return if it.zero?
         end

data/lib/atomic_sidekiq/atomic_operation/lua_scripts/expire.lua

@@ -1,6 +1,7 @@
 local queue = KEYS[1]
 local in_flight_key = KEYS[2]
 local now = tonumber(ARGV[1])
+local recover = ARGV[2] == 'true'
 
 local job = redis.call('get', in_flight_key)
 if (not job) then return nil end
@@ -9,7 +10,9 @@ local expiration = tonumber(string.match(job, '"expire_at":([0-9]*)'))
 if expiration > now then return nil end
 job = string.gsub(job, ',?"expire_at":[0-9]*', '')
 
-redis.call('lpush', queue, job)
+if recover then
+  redis.call('lpush', queue, job)
+end
 redis.call('del', in_flight_key)
 
 return { queue, job }

data/lib/atomic_sidekiq/dead_job_collector.rb

@@ -1,9 +1,10 @@
 module AtomicSidekiq
   class DeadJobCollector
     class << self
-      def collect!(queues, in_flight_keymaker:)
+      def collect!(queues, in_flight_keymaker:, skip_recovery_queues: [])
         queues.each do |q|
-          new(q, in_flight_keymaker: in_flight_keymaker).collect!
+          new(q, in_flight_keymaker: in_flight_keymaker)
+            .collect!(skip_recovery: skip_recovery_queues.include?(q))
         end
       end
     end
@@ -15,17 +16,18 @@ module AtomicSidekiq
       @expire_op          = AtomicOperation::Expire.new
     end
 
-    def collect!
-      each_keys { |job_key| expire!(job_key) }
+    def collect!(skip_recovery: false)
+      each_keys { |job_key| expire!(job_key, skip_recovery: skip_recovery) }
     end
 
     private
 
     attr_reader :queue, :in_flight_keymaker, :expire_op, :recovered_stats
 
-    def expire!(job_key)
-      recovered = expire_op.perform(queue, job_key)
+    def expire!(job_key, skip_recovery:)
+      recovered = expire_op.perform(queue, job_key, recover: !skip_recovery)
       return if recovered.nil?
+
       job = JSON.parse(recovered[1])
       recovered_stats.increment!(job)
       job

data/lib/atomic_sidekiq/recovered_stats.rb

@@ -1,7 +1,6 @@
 module AtomicSidekiq
   class RecoveredStats
     def increment!(job)
-      puts "INCREMENTING"
       increment_by_job!(job["class"])
       increment_by_queue!(job["queue"])
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: atomic-sidekiq
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.2.0
 platform: ruby
 authors:
 - Alex Correia Santos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-18 00:00:00.000000000 Z
+date: 2018-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler