sidekiq-unique-jobs 6.0.24 → 7.0.4

data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua

@@ -0,0 +1,3 @@
+local function upgrade_v6_to_v7()
+
+end

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -0,0 +1,95 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id    = ARGV[1]
+local pttl      = tonumber(ARGV[2])
+local lock_type = ARGV[3]
+local limit     = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+
+--------  BEGIN Variables --------
+local queued_count = redis.call("LLEN", queued)
+local primed_count = redis.call("LLEN", primed)
+local locked_count = redis.call("HLEN", locked)
+---------  END Variables ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+---------  Begin unlock.lua ---------
+log_debug("BEGIN unlock digest:", digest, "(job_id: " .. job_id ..")")
+
+log_debug("HEXISTS", locked, job_id)
+if redis.call("HEXISTS", locked, job_id) == 0 then
+  -- TODO: Improve orphaned lock detection
+  if queued_count == 0 and primed_count == 0 and locked_count == 0 then
+    log_debug("Orphaned lock")
+  else
+    local result = ""
+    for i,v in ipairs(redis.call("HKEYS", locked)) do
+      result = result .. v .. ","
+    end
+    result = locked .. " (" .. result .. ")"
+    log("Yielding to: " .. result)
+    log_debug("Yielding to", result, locked, "by job", job_id)
+    return nil
+  end
+end
+
+-- Just in case something went wrong
+log_debug("LREM", queued, -1, job_id)
+redis.call("LREM", queued, -1, job_id)
+
+log_debug("LREM", primed, -1, job_id)
+redis.call("LREM", primed, -1, job_id)
+
+if limit and limit <= 1 and locked_count and locked_count <= 1 then
+  log_debug("ZREM", digests, digest)
+  redis.call("ZREM", digests, digest)
+end
+
+local redis_version = toversion(redisversion)
+local del_cmd       = "DEL"
+
+if tonumber(redis_version["major"]) >= 4 then del_cmd =  "UNLINK"; end
+
+if lock_type ~= "until_expired" then
+  log_debug(del_cmd, digest, info)
+  redis.call(del_cmd, digest, info)
+
+  log_debug("HDEL", locked, job_id)
+  redis.call("HDEL", locked, job_id)
+end
+
+log_debug("LPUSH", queued, "1")
+redis.call("LPUSH", queued, "1")
+
+log_debug("PEXPIRE", queued, 5000)
+redis.call("PEXPIRE", queued, 5000)
+
+log("Unlocked")
+log_debug("END unlock digest:", digest, "(job_id: " .. job_id ..")")
+return job_id
+---------  END unlock.lua ---------

data/lib/sidekiq_unique_jobs/lua/update_version.lua

@@ -0,0 +1,40 @@
+-------- BEGIN keys ---------
+local live_key = KEYS[1]
+local dead_key = KEYS[2]
+-------- END keys ---------
+
+-------- BEGIN lock arguments ---------
+local version  = ARGV[1]
+-------- END lock arguments -----------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN set_version.lua --------
+log_debug("BEGIN setting version:", version)
+
+local updated     = false
+local old_version = redis.call("GETSET", live_key, version)
+
+if not old_version then
+  log_debug("No previous version found")
+  updated = true
+elseif old_version ~= version then
+  log_debug("Old version:", old_version, "differs from:", version)
+  redis.call("SET", dead_key, old_version)
+  updated = true
+end
+
+return updated
+--------  END delete.lua --------
+
+

data/lib/sidekiq_unique_jobs/lua/upgrade.lua

@@ -0,0 +1,68 @@
+redis.replicate_commands()
+-------- BEGIN keys ---------
+local live_version = KEYS[1]
+local dead_version = KEYS[2]
+-------- END keys ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_upgrades.lua" %>
+----------  END local functions ----------
+
+
+local new_version   = redis.call("GET", live_version)
+local old_version   = redis.call("GET", dead_version)
+local redis_version = toversion(redisversion)
+local upgraded      = 0
+local del_cmd       = "DEL"
+
+if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
+--------  BEGIN delete.lua --------
+
+log_debug("BEGIN upgrading from: ", old_version, "to:", new_version)
+
+-- 1. Loop through all uniquejobs:jajshdahdas:GRABBED
+local cursor = "0"
+local per   = 50
+repeat
+  local pagination   = redis.call("SCAN", cursor, "MATCH", "*:GRABBED", "COUNT", per)
+  local next_cursor  = pagination[1]
+  local grabbed_keys = pagination[2]
+
+  for _, grabbed in ipairs(grabbed_keys) do
+    local locked_key = grabbed.gsub(":GRABBED", ":LOCKED")
+    local locks      = redis.call("HGETALL", grabbed)
+
+    if #locks == 0 then
+      log_debug("No entries in:", grabbed)
+    else
+      log_debug("HMSET", locked_key, unpack(locks))
+      redis.call("HMSET", locked_key, unpack(locks))
+    end
+
+    log_debug("DEL", grabbed)
+    redis.call("DEL", grabbed)
+
+    upgraded = upgraded + 1
+  end
+
+  cursor = next_cursor
+  if cursor == "0" then
+    log_debug("Looped through all grabbed keys, stopping iteration")
+  end
+until cursor == "0"
+
+
+
+log_debug("END upgrading from: ", old_version, "to:", new_version)
+
+return
+--------  END delete.lua --------

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -1,47 +1,45 @@
 # frozen_string_literal: true
 
-require "sidekiq"
-
 module SidekiqUniqueJobs
   #
   # Provides the sidekiq middleware that makes the gem work
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   module Middleware
-    def self.extended(base)
-      base.class_eval do
-        configure_middleware
-      end
-    end
+    include SidekiqUniqueJobs::Logging::Middleware
+    include SidekiqUniqueJobs::OptionsWithFallback
+    include SidekiqUniqueJobs::JSON
 
-    def configure_middleware
-      configure_server_middleware
-      configure_client_middleware
-    end
+    # The sidekiq job hash
+    # @return [Hash] the Sidekiq job hash
+    attr_reader :item
 
-    def configure_server_middleware
-      Sidekiq.configure_server do |config|
-        config.client_middleware do |chain|
-          require "sidekiq_unique_jobs/client/middleware"
-          chain.add SidekiqUniqueJobs::Client::Middleware
-        end
+    #
+    # This method runs before (prepended) the actual middleware implementation.
+    #   This is done to reduce duplication
+    #
+    # @param [Sidekiq::Worker] worker_class
+    # @param [Hash] item a sidekiq job hash
+    # @param [String] queue name of the queue
+    # @param [ConnectionPool] redis_pool only used for compatility reasons
+    #
+    # @return [yield<super>] <description>
+    #
+    # @yieldparam [<type>] if <description>
+    # @yieldreturn [<type>] <describe what yield should return>
+    def call(worker_class, item, queue, redis_pool = nil)
+      @worker_class = worker_class
+      @item         = item
+      @queue        = queue
+      @redis_pool   = redis_pool
+      return yield if unique_disabled?
 
-        config.server_middleware do |chain|
-          require "sidekiq_unique_jobs/server/middleware"
-          chain.add SidekiqUniqueJobs::Server::Middleware
-        end
-      end
-    end
+      SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]
 
-    def configure_client_middleware
-      Sidekiq.configure_client do |config|
-        config.client_middleware do |chain|
-          require "sidekiq_unique_jobs/client/middleware"
-          chain.add SidekiqUniqueJobs::Client::Middleware
-        end
+      with_logging_context do
+        super
       end
     end
   end
 end
-SidekiqUniqueJobs.extend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Middleware
+    # The unique sidekiq middleware for the client push
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Client
+      prepend SidekiqUniqueJobs::Middleware
+
+      # Calls this client middleware
+      #   Used from Sidekiq.process_single
+      #
+      # @see SidekiqUniqueJobs::Middleware#call
+      #
+      # @see https://github.com/mperham/sidekiq/wiki/Job-Format
+      # @see https://github.com/mperham/sidekiq/wiki/Middleware
+      #
+      # @yield when uniqueness is disable
+      # @yield when the lock is successful
+      def call(*, &block)
+        lock(&block)
+      end
+
+      private
+
+      def lock
+        if (_token = lock_instance.lock)
+          yield
+        else
+          warn_about_duplicate
+        end
+      end
+
+      def warn_about_duplicate
+        return unless log_duplicate?
+
+        log_warn "Already locked with another job_id (#{dump_json(item)})"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Middleware
+    # The unique sidekiq middleware for the server processor
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Server
+      prepend SidekiqUniqueJobs::Middleware
+
+      #
+      #
+      # Runs the server middleware (used from Sidekiq::Processor#process)
+      #
+      # @see SidekiqUniqueJobs::Middleware#call
+      #
+      # @see https://github.com/mperham/sidekiq/wiki/Job-Format
+      # @see https://github.com/mperham/sidekiq/wiki/Middleware
+      #
+      # @yield when uniqueness is disabled
+      # @yield when owning the lock
+      def call(*, &block)
+        lock_instance.execute(&block)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/normalizer.rb

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 
-require "json"
-
 module SidekiqUniqueJobs
   # Normalizes hashes by dumping them to json and loading them from json
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Normalizer
+    extend SidekiqUniqueJobs::JSON
+
     # Changes hash to a json compatible hash
     # @param [Hash] args
     # @return [Hash] a json compatible hash
     def self.jsonify(args)
-      Sidekiq.load_json(Sidekiq.dump_json(args))
+      load_json(dump_json(args))
     end
   end
 end

data/lib/sidekiq_unique_jobs/on_conflict.rb

@@ -12,20 +12,33 @@ module SidekiqUniqueJobs
   #
   # Provides lock conflict resolutions
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   module OnConflict
-    STRATEGIES = {
-      log: OnConflict::Log,
-      raise: OnConflict::Raise,
-      reject: OnConflict::Reject,
-      replace: OnConflict::Replace,
-      reschedule: OnConflict::Reschedule,
-    }.freeze
+    # A convenience method for using the configured strategies
+    def self.strategies
+      SidekiqUniqueJobs.strategies
+    end
 
-    # returns OnConflict::NullStrategy when no other could be found
+    #
+    # Find a strategy to use for conflicting locks
+    #
+    # @param [Symbol] strategy the key for the strategy
+    #
+    # @return [OnConflict::Strategy] when found
+    # @return [OnConflict::NullStrategy] when no other could be found
+    #
     def self.find_strategy(strategy)
-      STRATEGIES.fetch(strategy.to_s.to_sym) { OnConflict::NullStrategy }
+      return OnConflict::NullStrategy unless strategy
+
+      strategies.fetch(strategy.to_sym) do
+        SidekiqUniqueJobs.logger.warn(
+          "No matching implementation for strategy: #{strategy}, returning OnConflict::NullStrategy." \
+          " Available strategies are (#{strategies.inspect})",
+        )
+
+        OnConflict::NullStrategy
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/on_conflict/log.rb

@@ -4,16 +4,20 @@ module SidekiqUniqueJobs
   module OnConflict
     # Strategy to log information about conflict
     #
-    # @author Mikael Henriksson <mikael@zoolutions.se>
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Log < OnConflict::Strategy
       include SidekiqUniqueJobs::Logging
 
+      #
       # Logs an informational message about that the job was not unique
+      #
+      #
+      # @return [void]
+      #
       def call
-        log_info(
-          "skipping job with id (#{item[JID_KEY]}) " \
-          "because unique_digest: (#{item[UNIQUE_DIGEST_KEY]}) already exists",
-        )
+        log_info(<<~MESSAGE.chomp)
+          Skipping job with id (#{item[JID]}) because lock_digest: (#{item[LOCK_DIGEST]}) already exists
+        MESSAGE
       end
     end
   end