sidekiq-unique-jobs 3.0.11 → 8.0.10

data/lib/sidekiq_unique_jobs/script/lua_error.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    #
+    # Misconfiguration is raised when gem is misconfigured
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    #
+    class Misconfiguration < RuntimeError
+    end
+
+    # LuaError raised on errors in Lua scripts
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class LuaError < RuntimeError
+      # Reformats errors raised by redis representing failures while executing
+      # a lua script. The default errors have confusing messages and backtraces,
+      # and a type of +RuntimeError+. This class improves the message and
+      # modifies the backtrace to include the lua script itself in a reasonable
+      # way.
+
+      PATTERN  = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/.freeze
+      LIB_PATH = File.expand_path("..", __dir__).freeze
+      CONTEXT_LINE_NUMBER = 2
+
+      attr_reader :error, :file, :content
+
+      # Is this error one that should be reformatted?
+      #
+      # @param error [StandardError] the original error raised by redis
+      # @return [Boolean] is this an error that should be reformatted?
+      def self.intercepts?(error)
+        PATTERN.match?(error.message)
+      end
+
+      # Initialize a new {LuaError} from an existing redis error, adjusting
+      # the message and backtrace in the process.
+      #
+      # @param error [StandardError] the original error raised by redis
+      # @param script [Script] a DTO with information about the script
+      #
+      def initialize(error, script)
+        @error        = error
+        @file         = script.path
+        @content      = script.source
+        @backtrace    = @error.backtrace
+
+        @error.message.match(PATTERN) do |regexp_match|
+          line_number   = regexp_match[2].to_i
+          message       = regexp_match[3]
+          error_context = generate_error_context(content, line_number)
+
+          super("#{message}\n\n#{error_context}\n\n")
+          set_backtrace(generate_backtrace(file, line_number))
+        end
+      end
+
+      private
+
+      # :nocov:
+      def generate_error_context(content, line_number)
+        lines                 = content.lines.to_a
+        beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+        ending_line_number    = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+        line_number_width     = ending_line_number.to_s.length
+
+        (beginning_line_number..ending_line_number).map do |number|
+          indicator = (number == line_number) ? "=>" : "  "
+          formatted_number = format("%#{line_number_width}d", number)
+          " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+        end.join.chomp
+      end
+
+      # :nocov:
+      def generate_backtrace(file, line_number)
+        pre_gem                 = backtrace_before_entering_gem(@backtrace)
+        index_of_first_gem_line = (@backtrace.size - pre_gem.size - 1)
+
+        pre_gem.unshift(@backtrace[index_of_first_gem_line])
+        pre_gem.unshift("#{file}:#{line_number}")
+        pre_gem
+      end
+
+      # :nocov:
+      def backtrace_before_entering_gem(backtrace)
+        backtrace.reverse.take_while { |line| !line_from_gem(line) }.reverse
+      end
+
+      # :nocov:
+      def line_from_gem(line)
+        line.split(":").first.include?(LIB_PATH)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/script.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Interface to dealing with .lua files
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Script
+      def self.load(name, root_path, conn)
+        script = new(name: name, root_path: root_path)
+        script.load(conn)
+      end
+
+      #
+      # @!attribute [r] script_name
+      #   @return [Symbol, String] the name of the script without extension
+      attr_reader :name
+      #
+      # @!attribute [r] script_path
+      #   @return [String] the path to the script on disk
+      attr_reader :path
+      #
+      # @!attribute [r] root_path
+      #   @return [Pathname]
+      attr_reader :root_path
+      #
+      # @!attribute [r] source
+      #   @return [String] the source code of the lua script
+      attr_reader :source
+      #
+      # @!attribute [rw] sha
+      #   @return [String] the sha of the script
+      attr_reader :sha
+      #
+      # @!attribute [rw] call_count
+      #   @return [Integer] the number of times the script was called/executed
+      attr_reader :call_count
+
+      def initialize(name:, root_path:)
+        @name      = name
+        @root_path = root_path
+        @path      = root_path.join("#{name}.lua").to_s
+        @source    = render_file
+        @sha       = compiled_sha
+        @call_count = 0
+      end
+
+      def ==(other)
+        sha == compiled_sha && compiled_sha == other.sha
+      end
+
+      def increment_call_count
+        @call_count += 1
+      end
+
+      def changed?
+        compiled_sha != sha
+      end
+
+      def render_file
+        Template.new(root_path).render(path)
+      end
+
+      def compiled_sha
+        Digest::SHA1.hexdigest(source)
+      end
+
+      def load(conn)
+        @sha = conn.script(:load, source)
+
+        self
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/scripts.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Interface to dealing with .lua files
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Scripts
+      #
+      # @return [Concurrent::Map] a map with configured script paths
+      SCRIPT_PATHS = Concurrent::Map.new
+
+      #
+      # Fetch a scripts configuration for path
+      #
+      # @param [Pathname] root_path the path to scripts
+      #
+      # @return [Scripts] a collection of scripts
+      #
+      def self.fetch(root_path)
+        if (scripts = SCRIPT_PATHS.get(root_path))
+          return scripts
+        end
+
+        create(root_path)
+      end
+
+      #
+      # Create a new scripts collection based on path
+      #
+      # @param [Pathname] root_path the path to scripts
+      #
+      # @return [Scripts] a collection of scripts
+      #
+      def self.create(root_path)
+        scripts = new(root_path)
+        store(scripts)
+      end
+
+      #
+      # Store the scripts collection in memory
+      #
+      # @param [Scripts] scripts the path to scripts
+      #
+      # @return [Scripts] the scripts instance that was stored
+      #
+      def self.store(scripts)
+        SCRIPT_PATHS.put(scripts.root_path, scripts)
+        scripts
+      end
+
+      #
+      # @!attribute [r] scripts
+      #   @return [Concurrent::Map] a collection of loaded scripts
+      attr_reader :scripts
+
+      #
+      # @!attribute [r] root_path
+      #   @return [Pathname] the path to the directory with lua scripts
+      attr_reader :root_path
+
+      def initialize(path)
+        raise ArgumentError, "path needs to be a Pathname" unless path.is_a?(Pathname)
+
+        @scripts   = Concurrent::Map.new
+        @root_path = path
+      end
+
+      def fetch(name, conn)
+        if (script = scripts.get(name.to_sym))
+          return script
+        end
+
+        load(name, conn)
+      end
+
+      def load(name, conn)
+        script = Script.load(name, root_path, conn)
+        scripts.put(name.to_sym, script)
+
+        script
+      end
+
+      def delete(script)
+        if script.is_a?(Script)
+          scripts.delete(script.name)
+        else
+          scripts.delete(script.to_sym)
+        end
+      end
+
+      def kill(conn)
+        if conn.respond_to?(:namespace)
+          conn.redis.script(:kill)
+        else
+          conn.script(:kill)
+        end
+      end
+
+      #
+      # Execute a lua script with given name
+      #
+      # @note this method is recursive if we need to load a lua script
+      #   that wasn't previously loaded.
+      #
+      # @param [Symbol] name the name of the script to execute
+      # @param [Redis] conn the redis connection to use for execution
+      # @param [Array<String>] keys script keys
+      # @param [Array<Object>] argv script arguments
+      #
+      # @return value from script
+      #
+      def execute(name, conn, keys: [], argv: [])
+        script = fetch(name, conn)
+        conn.evalsha(script.sha, keys, argv)
+      end
+
+      def count
+        scripts.keys.size
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/template.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Interface to dealing with .lua files
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module Script
+    #
+    # Class Template provides LUA script partial template rendering
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class Template
+      def initialize(script_path)
+        @script_path = script_path
+      end
+
+      #
+      # Renders a Lua script and includes any partials in that file
+      #  all `<%= include_partial '' %>` replaced with the actual contents of the partial
+      #
+      # @param [Pathname] pathname the path to the
+      #
+      # @return [String] the rendered Luascript
+      #
+      def render(pathname)
+        @partial_templates ||= {}
+        ::ERB.new(File.read(pathname)).result(binding)
+      end
+
+      # helper method to include a lua partial within another lua script
+      #
+      def include_partial(relative_path)
+        return if @partial_templates.key?(relative_path)
+
+        @partial_templates[relative_path] = nil
+        render(Pathname.new("#{@script_path}/#{relative_path}"))
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script/timing.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Script
+    # Handles timing> of things
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    module Timing
+      module_function
+
+      #
+      # Used for timing method calls
+      #
+      #
+      # @return [yield return, Float]
+      #
+      def timed
+        start_time = now
+
+        [yield, now - start_time]
+      end
+
+      #
+      # Returns a float representation of the current time.
+      #   Either from Process or Time
+      #
+      #
+      # @return [Float]
+      #
+      def now
+        (Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/script.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "sidekiq_unique_jobs/script/template"
+require "sidekiq_unique_jobs/script/lua_error"
+require "sidekiq_unique_jobs/script/script"
+require "sidekiq_unique_jobs/script/scripts"
+require "sidekiq_unique_jobs/script/config"
+require "sidekiq_unique_jobs/script/timing"
+require "sidekiq_unique_jobs/script/logging"
+require "sidekiq_unique_jobs/script/dsl"
+require "sidekiq_unique_jobs/script/client"
+
+module SidekiqUniqueJobs
+  # Interface to dealing with .lua files
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module Script
+    include SidekiqUniqueJobs::Script::DSL
+
+    configure do |config|
+      config.scripts_path = Pathname.new(__FILE__).dirname.join("lua")
+      config.logger       = Sidekiq.logger # TODO: This becomes a little weird
+    end
+
+    #
+    # The current logger
+    #
+    #
+    # @return [Logger] the configured logger
+    #
+    def self.logger
+      config.logger
+    end
+
+    #
+    # Set a new logger
+    #
+    # @param [Logger] other another logger
+    #
+    # @return [Logger] the new logger
+    #
+    def self.logger=(other)
+      config.logger = other
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/server.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # The unique sidekiq middleware for the server processor
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Server
+    #
+    # @return [Proc] returns a default death handler for the gem to cleanup dead locks
+    DEATH_HANDLER = (lambda do |job, _ex|
+      return unless (digest = job["lock_digest"])
+
+      SidekiqUniqueJobs::Digests.new.delete_by_digest(digest)
+    end).freeze
+    #
+    # Configure the server middleware
+    #
+    #
+    # @return [Sidekiq] the sidekiq configuration
+    #
+    def self.configure(config)
+      config.on(:startup)  { start }
+      config.on(:shutdown) { stop }
+
+      return unless config.respond_to?(:death_handlers)
+
+      config.death_handlers << death_handler
+    end
+
+    #
+    # Start the sidekiq unique jobs server process
+    #
+    #
+    # @return [void]
+    #
+    def self.start
+      SidekiqUniqueJobs::UpdateVersion.call
+      SidekiqUniqueJobs::Orphans::Manager.start
+      SidekiqUniqueJobs::Orphans::ReaperResurrector.start
+    end
+
+    #
+    # Stop the sidekiq unique jobs server process
+    #
+    #
+    # @return [void]
+    #
+    def self.stop
+      SidekiqUniqueJobs::Orphans::Manager.stop
+    end
+
+    #
+    # A death handler for dead jobs
+    #
+    #
+    # @return [lambda]
+    #
+    def self.death_handler
+      DEATH_HANDLER
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb

@@ -1,68 +1,141 @@
-require 'sidekiq/api'
+# frozen_string_literal: true
 
+require "sidekiq/api"
+
+#
+# @api private
+#
 module Sidekiq
-  class Job
+  # See Sidekiq::Api
+  class SortedEntry
+    #
+    # Provides extensions for unlocking jobs that are removed and deleted
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
     module UniqueExtension
-      def self.included(base)
-        base.class_eval do
-          alias_method :delete_orig, :delete
-          alias_method :delete, :delete_ext
-        end
+      #
+      # Wraps the original method to ensure locks for the job are deleted
+      #
+      #
+      # @return [Hash] the deleted sidekiq job hash
+      #
+      def delete
+        SidekiqUniqueJobs::Unlockable.delete!(item) if super
+        item
       end
 
-      def delete_ext
-        unlock(payload_hash(item))
-        delete_orig
+      private
+
+      #
+      # Wraps the original method to ensure locks for the job are deleted
+      #
+      #
+      # @yieldparam [Hash] message the sidekiq job hash
+      def remove_job
+        super do |message|
+          SidekiqUniqueJobs::Unlockable.delete!(Sidekiq.load_json(message))
+          yield message
+        end
       end
+    end
 
-      protected
+    prepend UniqueExtension
+  end
 
-      def payload_hash(item)
-        SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args'])
+  # See Sidekiq::Api
+  class ScheduledSet
+    #
+    # Provides extensions for unlocking jobs that are removed and deleted
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    module UniqueExtension
+      #
+      # Wraps the original method to ensure locks for the job are deleted
+      #
+      #
+      # @param [Integer, Float] score the score in the scheduled set
+      # @param [String] job_id the Sidekiq JID
+      #
+      def delete(score, job_id)
+        entry = find_job(job_id)
+        SidekiqUniqueJobs::Unlockable.delete!(entry.item) if super(score, job_id)
+        entry
       end
+    end
 
-      def unlock(payload_hash)
-        Sidekiq.redis { |conn| conn.del(payload_hash) }
+    prepend UniqueExtension
+  end
+
+  # See Sidekiq::Api
+  class JobRecord
+    #
+    # Provides extensions for unlocking jobs that are removed and deleted
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    module UniqueExtension
+      #
+      # Wraps the original method to ensure locks for the job are deleted
+      #
+      def delete
+        SidekiqUniqueJobs::Unlockable.delete!(item)
+        super
       end
     end
-    include UniqueExtension
+
+    prepend UniqueExtension
   end
 
+  # See Sidekiq::Api
   class Queue
+    #
+    # Provides extensions for unlocking jobs that are removed and deleted
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
     module UniqueExtension
-      def self.included(base)
-        base.class_eval do
-          alias_method :clear_orig, :clear
-          alias_method :clear, :clear_ext
-        end
-      end
-
-      def clear_ext
+      #
+      # Wraps the original method to ensure locks for the job are deleted
+      #
+      def clear
         each(&:delete)
-        clear_orig
+        super
       end
     end
 
-    include UniqueExtension
+    prepend UniqueExtension
   end
 
+  # See Sidekiq::Api
   class JobSet
+    #
+    # Provides extensions for unlocking jobs that are removed and deleted
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
     module UniqueExtension
-      def self.included(base)
-        base.class_eval do
-          if base.method_defined?(:clear)
-            alias_method :clear_orig, :clear
-            alias_method :clear, :clear_ext
-          end
-        end
+      #
+      # Wraps the original method to ensure locks for the job are deleted
+      #
+      def clear
+        each(&:delete)
+        super
       end
 
-      def clear_ext
-        each(&:delete_ext)
-        clear_orig
+      #
+      # Wraps the original method to ensure locks for the job are deleted
+      #
+      #
+      # @param [String] name the name of the key
+      # @param [String] value a sidekiq job hash
+      #
+      def delete_by_value(name, value)
+        SidekiqUniqueJobs::Unlockable.delete!(Sidekiq.load_json(value)) if super(name, value)
       end
     end
 
-    include UniqueExtension
+    prepend UniqueExtension
   end
 end