sidekiq-unique-jobs 6.0.25 → 7.0.0.beta2

data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua

@@ -0,0 +1,13 @@
+local hgetall = function (key)
+  local bulk = redis.call("HGETALL", key)
+  local result = {}
+  local nextkey
+  for i, v in ipairs(bulk) do
+    if i % 2 == 1 then
+      nextkey = v
+    else
+      result[nextkey] = v
+    end
+  end
+  return result
+end

data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua

@@ -0,0 +1,3 @@
+local function upgrade_v6_to_v7()
+
+end

data/lib/sidekiq_unique_jobs/lua/shared/find_digest_in_sorted_set.lua

@@ -0,0 +1,24 @@
+local function find_digest_in_sorted_set(name, digest)
+  local cursor  = 0
+  local count   = 5
+  local pattern = "*" .. digest .. "*"
+  local found   = false
+
+  log_debug("searching in:", name,
+            "for digest:", digest,
+            "cursor:", cursor)
+  repeat
+    local pagination  = redis.call("ZSCAN", name, cursor, "MATCH", pattern, "COUNT", count)
+    local next_cursor = pagination[1]
+    local items       = pagination[2]
+
+    if #items > 0 then
+      log_debug("Found digest", digest, "in sorted set:", name)
+      found = true
+    end
+
+    cursor = next_cursor
+  until found == true or cursor == "0"
+
+  return found
+end

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -0,0 +1,99 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+-------- BEGIN lock arguments ---------
+local job_id = ARGV[1]
+local pttl   = tonumber(ARGV[2])
+local type   = ARGV[3]
+local limit  = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+--------  BEGIN Variables --------
+local queued_count = redis.call("LLEN", queued)
+local primed_count = redis.call("LLEN", primed)
+local locked_count = redis.call("HLEN", locked)
+---------  END Variables ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+---------  Begin unlock.lua ---------
+log_debug("BEGIN unlock digest:", digest, "(job_id: " .. job_id ..")")
+
+log_debug("HEXISTS", locked, job_id)
+if redis.call("HEXISTS", locked, job_id) == 0 then
+  -- TODO: Improve orphaned lock detection
+  if queued_count == 0 and primed_count == 0 and locked_count == 0 then
+    log_debug("Orphaned lock")
+  else
+    local result = ""
+    for i,v in ipairs(redis.call("HKEYS", locked)) do
+      result = result .. v .. ","
+    end
+    result = locked .. " (" .. result .. ")"
+    log("Yielding to: " .. result)
+    log_debug("Yielding to", result, locked, "by job", job_id)
+    return nil
+  end
+end
+
+-- Just in case something went wrong
+log_debug("LREM", queued, -1, job_id)
+redis.call("LREM", queued, -1, job_id)
+
+log_debug("LREM", primed, -1, job_id)
+redis.call("LREM", primed, -1, job_id)
+
+-- TODO: Check if there are other locks before removing
+log_debug("ZREM", digests, digest)
+redis.call("ZREM", digests, digest)
+
+if pttl and pttl > 0 then
+  log_debug("PEXPIRE", digest, pttl)
+  redis.call("PEXPIRE", digest, pttl)
+
+  log_debug("PEXPIRE", locked, pttl)
+  redis.call("PEXPIRE", locked, pttl)
+
+  log_debug("PEXPIRE", info, pttl)
+  redis.call("PEXPIRE", info, pttl)
+else
+  local redis_version = toversion(redisversion)
+  local del_cmd       = "DEL"
+
+  if tonumber(redis_version["major"]) >= 4 then del_cmd =  "UNLINK"; end
+
+  log_debug(del_cmd, digest, info)
+  redis.call(del_cmd, digest, info)
+
+  log_debug("HDEL", locked, job_id)
+  redis.call("HDEL", locked, job_id)
+end
+
+log_debug("LPUSH", queued, "1")
+redis.call("LPUSH", queued, "1")
+
+log_debug("PEXPIRE", queued, 500)
+redis.call("PEXPIRE", queued, 500)
+
+log("Unlocked")
+log_debug("END unlock digest:", digest, "(job_id: " .. job_id ..")")
+return job_id
+---------  END unlock.lua ---------

data/lib/sidekiq_unique_jobs/lua/update_version.lua

@@ -0,0 +1,40 @@
+-------- BEGIN keys ---------
+local live_key = KEYS[1]
+local dead_key = KEYS[2]
+-------- END keys ---------
+
+-------- BEGIN lock arguments ---------
+local version  = ARGV[1]
+-------- END lock arguments -----------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN set_version.lua --------
+log_debug("BEGIN setting version:", version)
+
+local updated     = false
+local old_version = redis.call("GETSET", live_key, version)
+
+if not old_version then
+  log_debug("No previous version found")
+  updated = true
+elseif old_version ~= version then
+  log_debug("Old version:", old_version, "differs from:", version)
+  redis.call("SET", dead_key, old_version)
+  updated = true
+end
+
+return updated
+--------  END delete.lua --------
+
+

data/lib/sidekiq_unique_jobs/lua/upgrade.lua

@@ -0,0 +1,68 @@
+redis.replicate_commands()
+-------- BEGIN keys ---------
+local live_version = KEYS[1]
+local dead_version = KEYS[2]
+-------- END keys ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_upgrades.lua" %>
+----------  END local functions ----------
+
+
+local new_version   = redis.call("GET", live_version)
+local old_version   = redis.call("GET", dead_version)
+local redis_version = toversion(redisversion)
+local upgraded      = 0
+local del_cmd       = "DEL"
+
+if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
+--------  BEGIN delete.lua --------
+
+log_debug("BEGIN upgrading from: ", old_version, "to:", new_version)
+
+-- 1. Loop through all uniquejobs:jajshdahdas:GRABBED
+local cursor = "0"
+local per   = 50
+repeat
+  local pagination   = redis.call("SCAN", cursor, "MATCH", "*:GRABBED", "COUNT", per)
+  local next_cursor  = pagination[1]
+  local grabbed_keys = pagination[2]
+
+  for _, grabbed in ipairs(grabbed_keys) do
+    local locked_key = grabbed.gsub(":GRABBED", ":LOCKED")
+    local locks      = redis.call("HGETALL", grabbed)
+
+    if #locks == 0 then
+      log_debug("No entries in:", grabbed)
+    else
+      log_debug("HMSET", locked_key, unpack(locks))
+      redis.call("HMSET", locked_key, unpack(locks))
+    end
+
+    log_debug("DEL", grabbed)
+    redis.call("DEL", grabbed)
+
+    upgraded = upgraded + 1
+  end
+
+  cursor = next_cursor
+  if cursor == "0" then
+    log_debug("Looped through all grabbed keys, stopping iteration")
+  end
+until cursor == "0"
+
+
+
+log_debug("END upgrading from: ", old_version, "to:", new_version)
+
+return
+--------  END delete.lua --------

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "sidekiq"
-
 module SidekiqUniqueJobs
   #
   # Provides the sidekiq middleware that makes the gem work
@@ -9,51 +7,84 @@ module SidekiqUniqueJobs
   # @author Mikael Henriksson <mikael@zoolutions.se>
   #
   module Middleware
-    def self.extended(base)
-      base.class_eval do
-        configure_middleware
-      end
-    end
+    include SidekiqUniqueJobs::Logging::Middleware
+    include SidekiqUniqueJobs::OptionsWithFallback
+    include SidekiqUniqueJobs::JSON
 
-    def configure_middleware
-      configure_server_middleware
-      configure_client_middleware
+    #
+    # Configure both server and client
+    #
+    def self.configure
+      configure_server
+      configure_client
     end
 
-    def configure_server_middleware # rubocop:disable Metrics/MethodLength
+    #
+    # Configures the Sidekiq server
+    #
+    def self.configure_server # rubocop:disable Metrics/MethodLength
       Sidekiq.configure_server do |config|
         config.client_middleware do |chain|
-          require "sidekiq_unique_jobs/client/middleware"
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Client::Middleware
-          else
-            chain.add SidekiqUniqueJobs::Client::Middleware
-          end
+          chain.add SidekiqUniqueJobs::Middleware::Client
         end
 
         config.server_middleware do |chain|
-          require "sidekiq_unique_jobs/server/middleware"
-          if defined?(Apartment::Sidekiq::Middleware::Server)
-            chain.insert_after Apartment::Sidekiq::Middleware::Server, SidekiqUniqueJobs::Server::Middleware
-          else
-            chain.add SidekiqUniqueJobs::Server::Middleware
-          end
+          chain.add SidekiqUniqueJobs::Middleware::Server
+        end
+
+        config.on(:startup) do
+          SidekiqUniqueJobs::UpdateVersion.call
+          SidekiqUniqueJobs::UpgradeLocks.call
+
+          SidekiqUniqueJobs::Orphans::Manager.start
+        end
+
+        config.on(:shutdown) do
+          SidekiqUniqueJobs::Orphans::Manager.stop
         end
       end
     end
 
-    def configure_client_middleware
+    #
+    # Configures the Sidekiq client
+    #
+    def self.configure_client
       Sidekiq.configure_client do |config|
         config.client_middleware do |chain|
-          require "sidekiq_unique_jobs/client/middleware"
-          if defined?(Apartment::Sidekiq::Middleware::Client)
-            chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Client::Middleware
-          else
-            chain.add SidekiqUniqueJobs::Client::Middleware
-          end
+          chain.add SidekiqUniqueJobs::Middleware::Client
         end
       end
     end
+
+    # The sidekiq job hash
+    # @return [Hash] the Sidekiq job hash
+    attr_reader :item
+
+    #
+    # This method runs before (prepended) the actual middleware implementation.
+    #   This is done to reduce duplication
+    #
+    # @param [Sidekiq::Worker] worker_class
+    # @param [Hash] item a sidekiq job hash
+    # @param [String] queue name of the queue
+    # @param [ConnectionPool] redis_pool only used for compatility reasons
+    #
+    # @return [yield<super>] <description>
+    #
+    # @yieldparam [<type>] if <description>
+    # @yieldreturn [<type>] <describe what yield should return>
+    def call(worker_class, item, queue, redis_pool = nil)
+      @worker_class = worker_class
+      @item         = item
+      @queue        = queue
+      @redis_pool   = redis_pool
+      return yield if unique_disabled?
+
+      SidekiqUniqueJobs::Job.add_uniqueness(item)
+
+      with_logging_context do
+        super
+      end
+    end
   end
 end
-SidekiqUniqueJobs.extend SidekiqUniqueJobs::Middleware

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Middleware
+    # The unique sidekiq middleware for the client push
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    class Client
+      prepend SidekiqUniqueJobs::Middleware
+
+      # Calls this client middleware
+      #   Used from Sidekiq.process_single
+      #
+      # @see SidekiqUniqueJobs::Middleware#call
+      #
+      # @see https://github.com/mperham/sidekiq/wiki/Job-Format
+      # @see https://github.com/mperham/sidekiq/wiki/Middleware
+      #
+      # @yield when uniqueness is disable
+      # @yield when the lock is successful
+      def call(*)
+        lock { yield }
+      end
+
+      private
+
+      def lock
+        if (token = lock_instance.lock)
+          yield token
+        else
+          warn_about_duplicate
+        end
+      end
+
+      def warn_about_duplicate
+        return unless log_duplicate?
+
+        log_warn "Already locked with another job_id (#{dump_json(item)})"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Middleware
+    # The unique sidekiq middleware for the server processor
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    class Server
+      prepend SidekiqUniqueJobs::Middleware
+
+      #
+      #
+      # Runs the server middleware (used from Sidekiq::Processor#process)
+      #
+      # @see SidekiqUniqueJobs::Middleware#call
+      #
+      # @see https://github.com/mperham/sidekiq/wiki/Job-Format
+      # @see https://github.com/mperham/sidekiq/wiki/Middleware
+      #
+      # @yield when uniqueness is disabled
+      # @yield when owning the lock
+      def call(*)
+        lock_instance.execute { yield }
+      end
+    end
+  end
+end