sidekiq-unique-jobs 7.0.0.beta2 → 7.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3b55e52f9a512b127403e97a0bfe243f24174db6d31d03da0876c2cb285ed57
-  data.tar.gz: 60968aa5ffa0125fad0a35f2dadf2372b1f3b2cb0b529a26c036f08617526250
+  metadata.gz: f5f537750acf3b6a7b8f25ede43519210cc05dc77069417539a54c0c2d0c114c
+  data.tar.gz: 367d2a7f27d10200d4eb2b148b2ea34fbdffe16eb52ae71cfe73b5b20d2822d8
 SHA512:
-  metadata.gz: 41441c9e7280b8f3bda892879e29077778c0f99e0573dd478ee1f65c462c1b1a13b1573b7e3d74cdcf60c67cfa6e86f39ee366693b9c9396cfc9d8d54e2cbffe
-  data.tar.gz: 8e9b5d486a9f2714fdeff5968dfd9964c68bcf6e9d07af40488f800f76f40e56dc6040b48e0aa0369e178e3678f2d4692ab4221615b6a41cefff830cbc8f0e10
+  metadata.gz: 8fe2296811e1d1d8350c51d232261e61879317a6a4c46a86a302dfac9a0ad341191086ac96db74430d51ead315e370f1073f6015098fff09d5995abcc40d5146
+  data.tar.gz: 6fd909cfb0c8960cc1cb14bbd336493897121d24801ad1c2b678733666fba76a3ea0a5d6868e814c9f426080ff027fdf6102ffee08a2108e2ee840496a493717

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [v7.0.0.beta2](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.0.0.beta2) (2019-10-08)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.0.0.beta1...v7.0.0.beta2)
+
+**Fixed bugs:**
+
+- Pass redis\_version into scripts [\#431](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/431) ([mhenrixon](https://github.com/mhenrixon))
+
+**Closed issues:**
+
+- incorrect `:until\_and\_while\_executing` behavior [\#424](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/424)
+
 ## [v7.0.0.beta1](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.0.0.beta1) (2019-10-07)
 
 [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v6.0.15...v7.0.0.beta1)

data/README.md

@@ -418,7 +418,7 @@ The last one is log which can be be used with the lock `UntilExecuted` and `Unti
 ### log
 
 ```ruby
-sidekiq_options on_conflict: :log`
+sidekiq_options on_conflict: :log
 ```
 
 This strategy is intended to be used with `UntilExecuted` and `UntilExpired`. It will log a line about that this is job is a duplicate of another.
@@ -426,7 +426,7 @@ This strategy is intended to be used with `UntilExecuted` and `UntilExpired`. It
 ### raise
 
 ```ruby
-sidekiq_options on_conflict: :raise`
+sidekiq_options on_conflict: :raise
 ```
 
 This strategy is intended to be used with `WhileExecuting`. Basically it will allow us to let the server process crash with a specific error message and be retried without messing up the Sidekiq stats.
@@ -434,7 +434,7 @@ This strategy is intended to be used with `WhileExecuting`. Basically it will al
 ### reject
 
 ```ruby
-sidekiq_options on_conflict: :reject`
+sidekiq_options on_conflict: :reject
 ```
 
 This strategy is intended to be used with `WhileExecuting` and will push the job to the dead queue on conflict.
@@ -442,7 +442,7 @@ This strategy is intended to be used with `WhileExecuting` and will push the job
 ### replace
 
 ```ruby
-sidekiq_options on_conflict: :replace`
+sidekiq_options on_conflict: :replace
 ```
 
 This strategy is intended to be used with client locks like `UntilExecuted`.
@@ -455,7 +455,7 @@ always scheduled in the future. Currently only attempting to retry one time.
 ### Reschedule
 
 ```ruby
-sidekiq_options on_conflict: :reschedule`
+sidekiq_options on_conflict: :reschedule
 ```
 
 This strategy is intended to be used with `WhileExecuting` and will delay the job to be tried again in 5 seconds. This will mess up the sidekiq stats but will prevent exceptions from being logged and confuse your sysadmins.
@@ -475,7 +475,7 @@ module Strategies
 end
 ```
 
-You can refer on all the startegies defined in `lib/sidekiq_unique_jobs/on_conflict`.
+You can refer to all the strategies defined in `lib/sidekiq_unique_jobs/on_conflict`.
 
 In order to make it available, you should call in your project startup:
 

data/lib/sidekiq_unique_jobs.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "brpoplpush/redis_script"
 require "concurrent/future"
 require "concurrent/promises"
 require "concurrent/timer_task"
@@ -25,7 +26,6 @@ require "sidekiq_unique_jobs/connection"
 require "sidekiq_unique_jobs/exceptions"
 require "sidekiq_unique_jobs/script"
 require "sidekiq_unique_jobs/script/caller"
-require "sidekiq_unique_jobs/script/template"
 require "sidekiq_unique_jobs/json"
 require "sidekiq_unique_jobs/normalizer"
 require "sidekiq_unique_jobs/job"

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -82,89 +82,6 @@ module SidekiqUniqueJobs
     end
   end
 
-  # Error raised from {OnConflict::Raise}
-  #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class ScriptError < UniqueJobsError
-    # Reformats errors raised by redis representing failures while executing
-    # a lua script. The default errors have confusing messages and backtraces,
-    # and a type of +RuntimeError+. This class improves the message and
-    # modifies the backtrace to include the lua script itself in a reasonable
-    # way.
-
-    PATTERN  = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/.freeze
-    LIB_PATH = File.expand_path("..", __dir__)
-    CONTEXT_LINE_NUMBER = 3
-
-    attr_reader :error, :file, :content
-
-    # Is this error one that should be reformatted?
-    #
-    # @param error [StandardError] the original error raised by redis
-    # @return [Boolean] is this an error that should be reformatted?
-    def self.intercepts?(error)
-      error.message =~ PATTERN
-    end
-
-    # Initialize a new {ScriptError} from an existing redis error, adjusting
-    # the message and backtrace in the process.
-    #
-    # @param error [StandardError] the original error raised by redis
-    # @param file [Pathname] full path to the lua file the error ocurred in
-    # @param content [String] lua file content the error ocurred in
-    # :nocov:
-    def initialize(error, file, content)
-      @error        = error
-      @file         = file
-      @content      = content
-      @backtrace    = @error.backtrace
-
-      @error.message.match(PATTERN) do |regexp_match|
-        line_number   = regexp_match[2].to_i
-        message       = regexp_match[3]
-        error_context = generate_error_context(content, line_number)
-
-        super("#{message}\n\n#{error_context}\n\n")
-        set_backtrace(generate_backtrace(file, line_number))
-      end
-    end
-
-    private
-
-    # :nocov:
-    def generate_error_context(content, line_number)
-      lines                 = content.lines.to_a
-      beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
-      ending_line_number    = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
-      line_number_width     = ending_line_number.to_s.length
-
-      (beginning_line_number..ending_line_number).map do |number|
-        indicator = (number == line_number) ? "=>" : "  "
-        formatted_number = format("%#{line_number_width}d", number)
-        " #{indicator} #{formatted_number}: #{lines[number - 1]}"
-      end.join.chomp
-    end
-
-    # :nocov:
-    def generate_backtrace(file, line_number)
-      pre_gem                         = backtrace_before_entering_gem(@backtrace)
-      index_of_first_unique_jobs_line = (@backtrace.size - pre_gem.size - 1)
-      pre_gem.unshift(@backtrace[index_of_first_unique_jobs_line])
-      pre_gem.unshift("#{file}:#{line_number}")
-      pre_gem
-    end
-
-    # :nocov:
-    def backtrace_before_entering_gem(backtrace)
-      backtrace.reverse.take_while { |line| !line_from_gem(line) }.reverse
-    end
-
-    # :nocov:
-    def line_from_gem(line)
-      line.split(":").first.include?(LIB_PATH)
-    end
-  end
-
   # Error raised from {OptionsWithFallback#lock_class}
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -2,6 +2,11 @@
 
 module SidekiqUniqueJobs
   class Lock
+    #
+    # UntilExpired locks until the job expires
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    #
     class UntilExpired < UntilExecuted
     end
   end

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -19,7 +19,7 @@ module SidekiqUniqueJobs
       #
       def self.start
         with_logging_context do
-          logger.info("Starting Reaper")
+          log_info("Starting Reaper")
           task.add_observer(Observer.new)
           task.execute
           task
@@ -34,7 +34,7 @@ module SidekiqUniqueJobs
       #
       def self.stop
         with_logging_context do
-          logger.info("Stopping Reaper")
+          log_info("Stopping Reaper")
           task.shutdown
         end
       end

data/lib/sidekiq_unique_jobs/orphans/reaper.rb

@@ -175,11 +175,43 @@ module SidekiqUniqueJobs
       #
       #
       def enqueued?(digest)
-        if (result = call_script(:find_digest_in_queues, conn, keys: [digest]))
-          log_debug("#{digest} found in #{result}")
-          true
-        else
-          log_debug("#{digest} NOT found in any queues")
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            entries(conn, queue) do |entry|
+              if entry.include?(digest)
+                log_info("#{digest} found in #{queue}")
+                return true
+              end
+            end
+          end
+
+          log_info("#{digest} not enqueued")
+          false
+        end
+      end
+
+      def queues(conn, &block)
+        conn.sscan_each("queues", &block)
+      end
+
+      def entries(conn, queue) # rubocop:disable Metrics/MethodLength
+        queue_key    = "queue:#{queue}"
+        initial_size = conn.llen(queue_key)
+        deleted_size = 0
+        page         = 0
+        page_size    = 50
+
+        loop do
+          range_start = page * page_size - deleted_size
+          range_end   = range_start + page_size - 1
+          entries     = conn.lrange(queue_key, range_start, range_end)
+          page       += 1
+
+          entries.each do |entry|
+            yield entry
+          end
+
+          deleted_size = initial_size - size
         end
       end
 

data/lib/sidekiq_unique_jobs/script.rb

@@ -5,128 +5,11 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
   module Script
-    LUA_PATHNAME ||= Pathname.new(__FILE__).dirname.join("lua").freeze
-    SCRIPT_SHAS ||= Concurrent::Map.new
+    include Brpoplpush::RedisScript::DSL
 
-    extend SidekiqUniqueJobs::Connection
-    extend SidekiqUniqueJobs::Logging
-    extend SidekiqUniqueJobs::Timing
-
-    module_function
-
-    #
-    # Call a lua script with the provided file_name
-    #
-    # @note this method is recursive if we need to load a lua script
-    #   that wasn't previously loaded.
-    #
-    # @param [Symbol] file_name the name of the lua script
-    # @param [Array<String>] keys script keys
-    # @param [Array<Object>] argv script arguments
-    # @param [Redis] conn the redis connection to use
-    #
-    # @return value from script
-    #
-    def call(file_name, conn, keys: [], argv: [])
-      result, elapsed = timed do
-        execute_script(file_name, conn, keys, argv)
-      end
-
-      log_debug("Executed #{file_name}.lua in #{elapsed}ms")
-      result
-    rescue ::Redis::CommandError => ex
-      handle_error(ex, file_name, conn) do
-        call(file_name, conn, keys: keys, argv: argv)
-      end
-    end
-
-    #
-    # Execute the script file
-    #
-    # @param [Symbol] file_name the name of the lua script
-    # @param [Redis] conn the redis connection to use
-    # @param [Array] keys the array of keys to pass to the script
-    # @param [Array] argv the array of arguments to pass to the script
-    #
-    # @return value from script (evalsha)
-    #
-    def execute_script(file_name, conn, keys, argv)
-      conn.evalsha(
-        script_sha(conn, file_name),
-        keys,
-        argv,
-      )
-    end
-
-    #
-    # Return sha of already loaded lua script or load it and return the sha
-    #
-    # @param [Sidekiq::RedisConnection] conn the redis connection
-    # @param [Symbol] file_name the name of the lua script
-    # @return [String] sha of the script file
-    #
-    # @return [String] the sha of the script
-    #
-    def script_sha(conn, file_name)
-      if (sha = SCRIPT_SHAS.get(file_name))
-        return sha
-      end
-
-      sha = conn.script(:load, script_source(file_name))
-      SCRIPT_SHAS.put(file_name, sha)
-      sha
-    end
-
-    #
-    # Handle errors to allow retrying errors that need retrying
-    #
-    # @param [Redis::CommandError] ex exception to handle
-    # @param [Symbol] file_name the name of the lua script
-    # @param [Redis] conn the redis connection to use
-    #
-    # @return [void]
-    #
-    # @yieldreturn [void] yields back to the caller when NOSCRIPT is raised
-    def handle_error(ex, file_name, conn) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
-      case ex.message
-      when "NOSCRIPT No matching script. Please use EVAL."
-        SCRIPT_SHAS.delete(file_name)
-        return yield if block_given?
-      when "BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE."
-        begin
-          conn.script(:kill)
-          return yield if block_given?
-        rescue ::Redis::CommandError => ex
-          log_warn(ex)
-          return yield if block_given?
-        end
-      end
-
-      raise unless ScriptError.intercepts?(ex)
-
-      raise ScriptError.new(ex, script_path(file_name).to_s, script_source(file_name))
-    end
-
-    #
-    # Reads the lua file from disk
-    #
-    # @param [Symbol] file_name the name of the lua script
-    #
-    # @return [String] the content of the lua file
-    #
-    def script_source(file_name)
-      Template.new(LUA_PATHNAME).render(script_path(file_name))
-    end
-
-    #
-    # Construct a Pathname to a lua script
-    #
-    # @param [Symbol] file_name the name of the lua script
-    #
-    # @return [Pathname] the full path to the gems lua script
-    #
-    def script_path(file_name)
-      LUA_PATHNAME.join("#{file_name}.lua")
+    configure do |config|
+      config.scripts_path = Pathname.new(__FILE__).dirname.join("lua")
+      config.logger       = Sidekiq.logger # TODO: This becomes a little weird
     end
   end
 end

data/lib/sidekiq_unique_jobs/script/caller.rb

@@ -59,7 +59,7 @@ module SidekiqUniqueJobs
                                  file_name,
                                  redis_version,
                                ])
-        Script.call(file_name, conn, keys: keys, argv: argv)
+        Script.execute(file_name, conn, keys: keys, argv: argv)
       end
 
       #

data/lib/sidekiq_unique_jobs/version.rb

@@ -3,5 +3,5 @@
 module SidekiqUniqueJobs
   #
   # @return [String] the current SidekiqUniqueJobs version
-  VERSION = "7.0.0.beta2"
+  VERSION = "7.0.0.beta3"
 end

metadata

@@ -1,15 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 7.0.0.beta2
+  version: 7.0.0.beta3
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-08 00:00:00.000000000 Z
+date: 2019-11-24 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: brpoplpush-redis_script
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -309,7 +329,6 @@ files:
 - lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb
 - lib/sidekiq_unique_jobs/script.rb
 - lib/sidekiq_unique_jobs/script/caller.rb
-- lib/sidekiq_unique_jobs/script/template.rb
 - lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb
 - lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb
 - lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb
@@ -345,14 +364,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Sidekiq middleware that prevents duplicates jobs

data/lib/sidekiq_unique_jobs/script/template.rb

@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-module SidekiqUniqueJobs
-  # Interface to dealing with .lua files
-  #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  module Script
-    #
-    # Class Template provides LUA script partial template rendering
-    #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
-    #
-    class Template
-      def initialize(script_path)
-        @script_path = script_path
-      end
-
-      #
-      # Renders a Lua script and includes any partials in that file
-      #  all `<%= include_partial '' %>` replaced with the actual contents of the partial
-      #
-      # @param [Pathname] pathname the path to the
-      #
-      # @return [String] the rendered Luascript
-      #
-      def render(pathname)
-        @partial_templates ||= {}
-        ERB.new(File.read(pathname)).result(binding)
-      end
-
-      # helper method to include a lua partial within another lua script
-      #
-      def include_partial(relative_path)
-        return if @partial_templates.key?(relative_path)
-
-        @partial_templates[relative_path] = nil
-        render(Pathname.new("#{@script_path}/#{relative_path}"))
-      end
-    end
-  end
-end