sidekiq-unique-jobs 7.0.0.beta4 → 7.0.0.beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c73d16cc7119474898230003931d398ee6f0e6b925117cbbf763cbb9b740edf1
-  data.tar.gz: 8371f1cf3bb123a5df3a603c3c67abb584d51cbb646b088d9c2e9593343f4672
+  metadata.gz: d70a897c3f1aa180a2ec8d966218836038210bdf294f3dc2be955f3e2f4915f9
+  data.tar.gz: d57e756148fe58b0b72c8d1e32208bb6db61f1121a54b80ecbd275ee79a46828
 SHA512:
-  metadata.gz: 4671e078b75e70a2f427229b2d8dbcf259a32a3f8b13ab61ed9963b9d63327b31834082724a5abd4b46a79867c51f8578a36178027f690cac442550b8fed5bfe
-  data.tar.gz: b0bda233fddf2ff36394702b6b074a09290690802a7dcea8fe4be3ff56d95f0a7641b75287ea71a3417e11c7df6578f2403d77a57a1d24187fc3c6cac0bc5890
+  metadata.gz: c5c94eb9bc3b4ce1dd5ebcd7f7af5ae4456ff6783a47783ef5fe96b6bdf5c045d8adfff3e5abfe4390f2ee01d20e8e47a4cc50132cb2f31e17947dccbc0caf82
+  data.tar.gz: e0d9b47ba7930b254ce7e6741577de17a02607f8d89a5ff3aa313ed978bfefdb518d61f62c4437f75a93d267e3ffd3b5cc58bfd003160fc552d0bb91df21ec76

data/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## [v7.0.0.beta4](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.0.0.beta4) (2019-11-25)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.0.0.beta3...v7.0.0.beta4)
+
+**Fixed bugs:**
+
+- Fix ruby reaper [\#444](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/444) ([mhenrixon](https://github.com/mhenrixon))
+
+## [v7.0.0.beta3](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.0.0.beta3) (2019-11-24)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.0.0.beta2...v7.0.0.beta3)
+
+**Implemented enhancements:**
+
+- Brpoplpush redis script [\#434](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/434) ([mhenrixon](https://github.com/mhenrixon))
+- Drop support for almost EOL ruby 2.4 [\#433](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/433) ([mhenrixon](https://github.com/mhenrixon))
+
+**Fixed bugs:**
+
+- Redis is busy running script and script never terminates [\#441](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/441)
+- Make the ruby reaper plain ruby [\#443](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/443) ([mhenrixon](https://github.com/mhenrixon))
+
+**Closed issues:**
+
+- Some jobs seem to be treated as duplicate despite empty queue [\#440](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/440)
+
+**Merged pull requests:**
+
+- Fix typo and some formatting issues in README [\#442](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/442) ([ajkerr](https://github.com/ajkerr))
+
 ## [v7.0.0.beta2](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.0.0.beta2) (2019-10-08)
 
 [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.0.0.beta1...v7.0.0.beta2)

data/README.md

@@ -355,7 +355,7 @@ With this lock type it is possible to put any number of these jobs on the queue,
 
 **NOTE** Unless this job is configured with a `lock_timeout: nil` or `lock_timeout: > 0` then all jobs that are attempted to be executed will just be dropped without waiting.
 
-There is an example of this to try it out in the `my_app` application. Run `foreman start` in the root of the directory and open the url: `localhost:5000/work/duplicate_while_executing`.
+There is an example of this to try it out in the `myapp` application. Run `foreman start` in the root of the directory and open the url: `localhost:5000/work/duplicate_while_executing`.
 
 In the console you should see something like:
 
@@ -601,7 +601,8 @@ For sidekiq versions before 5.1 a `sidekiq_retries_exhausted` block is required
 ```ruby
 class MyWorker
   sidekiq_retries_exhausted do |msg, _ex|
-    SidekiqUniqueJobs::Digests.del(digest: msg['unique_digest']) if msg['unique_digest']
+    digest = msg['unique_digest']
+    SidekiqUniqueJobs::Digests.delete_by_digest(digest) if digest
   end
 end
 ```
@@ -612,7 +613,8 @@ Starting in v5.1, Sidekiq can also fire a global callback when a job dies:
 # this goes in your initializer
 Sidekiq.configure_server do |config|
   config.death_handlers << ->(job, _ex) do
-    SidekiqUniqueJobs::Digests.del(digest: job['unique_digest']) if job['unique_digest']
+    digest = msg['unique_digest']
+    SidekiqUniqueJobs::Digests.delete_by_digest(digest) if digest
   end
 end
 ```

data/lib/sidekiq_unique_jobs/cli.rb

@@ -16,9 +16,9 @@ module SidekiqUniqueJobs
     desc "list PATTERN", "list all unique digests and their expiry time"
     option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
     def list(pattern = "*")
-      digests = SidekiqUniqueJobs::Digests.new.entries(pattern: pattern, count: options[:count])
-      say "Found #{digests.size} digests matching '#{pattern}':"
-      print_in_columns(digests.sort) if digests.any?
+      entries = digests.entries(pattern: pattern, count: options[:count])
+      say "Found #{entries.size} digests matching '#{pattern}':"
+      print_in_columns(entries.sort) if entries.any?
     end
 
     desc "del PATTERN", "deletes unique digests from redis by pattern"
@@ -27,10 +27,10 @@ module SidekiqUniqueJobs
     def del(pattern)
       max_count = options[:count]
       if options[:dry_run]
-        digests = SidekiqUniqueJobs::Digests.new.entries(pattern: pattern, count: max_count)
-        say "Would delete #{digests.size} digests matching '#{pattern}'"
+        result = digests.entries(pattern: pattern, count: max_count)
+        say "Would delete #{result.size} digests matching '#{pattern}'"
       else
-        deleted_count = SidekiqUniqueJobs::Digests.new.del(pattern: pattern, count: max_count)
+        deleted_count = digests.delete_by_pattern(pattern, count: max_count)
         say "Deleted #{deleted_count} digests matching '#{pattern}'"
       end
     end
@@ -46,6 +46,10 @@ module SidekiqUniqueJobs
     end
 
     no_commands do
+      def digests
+        @digests ||= SidekiqUniqueJobs::Digests.new
+      end
+
       def console_class
         require "pry"
         Pry

data/lib/sidekiq_unique_jobs/constants.rb

@@ -41,5 +41,6 @@ module SidekiqUniqueJobs
   UNIQUE_ARGS           ||= "unique_args"
   UNIQUE_DIGEST         ||= "unique_digest"
   UNIQUE_PREFIX         ||= "unique_prefix"
+  UNIQUE_REAPER         ||= "uniquejobs:reaper"
   WORKER                ||= "worker"
 end

data/lib/sidekiq_unique_jobs/digests.rb

@@ -27,26 +27,44 @@ module SidekiqUniqueJobs
       redis { |conn| conn.zadd(key, now_f, digest) }
     end
 
+    # Deletes unique digests by pattern
     #
-    # Deletes unique digest either by a digest or pattern
-    #
-    # @overload call_script(digest: "abcdefab")
-    #   Call script with digest
-    #   @param [String] digest: a digest to delete
-    # @overload call_script(pattern: "*", count: 1_000)
-    #   Call script with pattern
-    #   @param [String] pattern: "*" a pattern to match
-    #   @param [String] count: DEFAULT_COUNT the number of keys to delete
-    #
-    # @raise [ArgumentError] when given neither pattern nor digest
-    #
+    # @param [String] pattern a key pattern to match with
+    # @param [Integer] count the maximum number
     # @return [Array<String>] with unique digests
+    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
+      result, elapsed = timed do
+        digests = entries(pattern: pattern, count: count).keys
+        redis { |conn| BatchDelete.call(digests, conn) }
+      end
+
+      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
+
+      result
+    end
+
+    # Delete unique digests by digest
+    #   Also deletes the :AVAILABLE, :EXPIRED etc keys
     #
-    def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
-      return delete_by_pattern(pattern, count: count) if pattern
-      return delete_by_digest(digest) if digest
+    # @param [String] digest a unique digest to delete
+    def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
+      result, elapsed = timed do
+        call_script(:delete_by_digest, [
+                      digest,
+                      "#{digest}:QUEUED",
+                      "#{digest}:PRIMED",
+                      "#{digest}:LOCKED",
+                      "#{digest}:RUN",
+                      "#{digest}:RUN:QUEUED",
+                      "#{digest}:RUN:PRIMED",
+                      "#{digest}:RUN:LOCKED",
+                      key,
+                    ])
+      end
+
+      log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
 
-      raise ArgumentError, "##{__method__} requires either a :digest or a :pattern"
+      result
     end
 
     #
@@ -92,37 +110,5 @@ module SidekiqUniqueJobs
         ]
       end
     end
-
-    private
-
-    # Deletes unique digests by pattern
-    #
-    # @param [String] pattern a key pattern to match with
-    # @param [Integer] count the maximum number
-    # @return [Array<String>] with unique digests
-    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
-      result, elapsed = timed do
-        digests = entries(pattern: pattern, count: count).keys
-        redis { |conn| BatchDelete.call(digests, conn) }
-      end
-
-      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
-
-      result
-    end
-
-    # Delete unique digests by digest
-    #   Also deletes the :AVAILABLE, :EXPIRED etc keys
-    #
-    # @param [String] digest a unique digest to delete
-    def delete_by_digest(digest)
-      result, elapsed = timed do
-        call_script(:delete_by_digest, [digest, key])
-      end
-
-      log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
-
-      result
-    end
   end
 end

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -17,18 +17,24 @@ module SidekiqUniqueJobs
       # @yield to the worker class perform method
       def execute
         if unlock
-          runtime_lock.execute { return yield }
+          lock_on_failure do
+            runtime_lock.execute { return yield }
+          end
         else
           log_warn "couldn't unlock digest: #{item[UNIQUE_DIGEST]} #{item[JID]}"
         end
       end
 
-      #
-      # Lock only when the server is processing the job
-      #
-      #
-      # @return [SidekiqUniqueJobs::Lock::WhileExecuting] an instance of a lock
-      #
+      private
+
+      def lock_on_failure
+        yield
+      rescue Exception # rubocop:disable Lint/RescueException
+        log_error("Runtime lock failed to execute job, restoring server lock")
+        lock
+        raise
+      end
+
       def runtime_lock
         @runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, callback, redis_pool)
       end

data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua

@@ -1,6 +1,13 @@
 -------- BEGIN keys ---------
-local digest  = KEYS[1]
-local digests = KEYS[2]
+local digest     = KEYS[1]
+local queued     = KEYS[2]
+local primed     = KEYS[3]
+local locked     = KEYS[4]
+local run_digest = KEYS[5]
+local run_queued = KEYS[6]
+local run_primed = KEYS[7]
+local run_locked = KEYS[8]
+local digests    = KEYS[9]
 --------  END keys  ---------
 
 --------  BEGIN injected arguments --------
@@ -15,17 +22,6 @@ local redisversion = tostring(ARGV[5])
 <%= include_partial "shared/_common.lua" %>
 ----------  END local functions ----------
 
---------  BEGIN Variables  --------
-local queued     = digest .. ":QUEUED"
-local primed     = digest .. ":PRIMED"
-local locked     = digest .. ":LOCKED"
-local run_digest = digest .. ":RUN"
-local run_queued = digest .. ":RUN:QUEUED"
-local run_primed = digest .. ":RUN:PRIMED"
-local run_locked = digest .. ":RUN:LOCKED"
---------   END Variables   --------
-
-
 --------  BEGIN delete_by_digest.lua --------
 local counter       = 0
 local redis_version = toversion(redisversion)

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -64,7 +64,11 @@ module SidekiqUniqueJobs
       # @return [Integer] the number of keys deleted
       #
       def delete_lock
-        call_script(:delete_by_digest, keys: [unique_digest, DIGESTS])
+        digests.delete_by_digest(unique_digest)
+      end
+
+      def digests
+        @digests ||= SidekiqUniqueJobs::Digests.new
       end
     end
   end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -7,7 +7,9 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@zoolutions.se>
     #
-    class Manager
+    module Manager
+      module_function
+
       include SidekiqUniqueJobs::Connection
       include SidekiqUniqueJobs::Logging
 
@@ -17,8 +19,11 @@ module SidekiqUniqueJobs
       #
       # @return [Concurrent::TimerTask] the task that was started
       #
-      def self.start
+      def start # rubocop:disable
+        return if registered?
+
         with_logging_context do
+          register_reaper_process
           log_info("Starting Reaper")
           task.add_observer(Observer.new)
           task.execute
@@ -32,9 +37,10 @@ module SidekiqUniqueJobs
       #
       # @return [Boolean]
       #
-      def self.stop
+      def stop
         with_logging_context do
           log_info("Stopping Reaper")
+          unregister_reaper_process
           task.shutdown
         end
       end
@@ -45,7 +51,7 @@ module SidekiqUniqueJobs
       #
       # @return [<type>] <description>
       #
-      def self.task
+      def task
         @task ||= Concurrent::TimerTask.new(timer_task_options) do
           with_logging_context do
             redis do |conn|
@@ -61,7 +67,7 @@ module SidekiqUniqueJobs
       #
       # @return [Hash]
       #
-      def self.timer_task_options
+      def timer_task_options
         { run_now: true,
           execution_interval: reaper_interval,
           timeout_interval: reaper_timeout }
@@ -70,14 +76,14 @@ module SidekiqUniqueJobs
       #
       # @see SidekiqUniqueJobs::Config#reaper_interval
       #
-      def self.reaper_interval
+      def reaper_interval
         SidekiqUniqueJobs.config.reaper_interval
       end
 
       #
       # @see SidekiqUniqueJobs::Config#reaper_timeout
       #
-      def self.reaper_timeout
+      def reaper_timeout
         SidekiqUniqueJobs.config.reaper_timeout
       end
 
@@ -88,13 +94,43 @@ module SidekiqUniqueJobs
       # @return [Hash] when logger responds to `:with_context`
       # @return [String] when logger does not responds to `:with_context`
       #
-      def self.logging_context
+      def logging_context
         if logger_context_hash?
           { "uniquejobs" => "reaper" }
         else
           "uniquejobs=orphan-reaper"
         end
       end
+
+      #
+      # Checks if a reaper is already registered
+      #
+      #
+      # @return [true, false]
+      #
+      def registered?
+        redis { |conn| conn.get(UNIQUE_REAPER) }.to_i == 1
+      end
+
+      #
+      # Writes a mutex key to redis
+      #
+      #
+      # @return [void]
+      #
+      def register_reaper_process
+        redis { |conn| conn.set(UNIQUE_REAPER, 1) }
+      end
+
+      #
+      # Removes mutex key from redis
+      #
+      #
+      # @return [void]
+      #
+      def unregister_reaper_process
+        redis { |conn| conn.del(UNIQUE_REAPER) }
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/version.rb

@@ -3,5 +3,5 @@
 module SidekiqUniqueJobs
   #
   # @return [String] the current SidekiqUniqueJobs version
-  VERSION = "7.0.0.beta4"
+  VERSION = "7.0.0.beta5"
 end

data/lib/sidekiq_unique_jobs/web.rb

@@ -32,7 +32,7 @@ module SidekiqUniqueJobs
       end
 
       app.get "/locks/delete_all" do
-        digests.del(pattern: "*", count: digests.count)
+        digests.delete_by_pattern("*", count: digests.count)
         redirect_to :locks
       end
 
@@ -44,7 +44,7 @@ module SidekiqUniqueJobs
       end
 
       app.get "/locks/:digest/delete" do
-        digests.del(digest: params[:digest])
+        digests.delete_by_digest(params[:digest])
         redirect_to :locks
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 7.0.0.beta4
+  version: 7.0.0.beta5
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-25 00:00:00.000000000 Z
+date: 2019-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: brpoplpush-redis_script
@@ -371,7 +371,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Sidekiq middleware that prevents duplicates jobs