sidekiq-unique-jobs 6.0.23 → 7.1.12

data/lib/sidekiq_unique_jobs/logging.rb

@@ -3,65 +3,220 @@
 module SidekiqUniqueJobs
   # Utility module for reducing the number of uses of logger.
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Logging
-    # A convenience method for using the configured logger
+    def self.included(base)
+      base.send(:extend, self)
+    end
+
+    #
+    # A convenience method for using the configured gem logger
+    #
+    # @see SidekiqUniqueJobs#.logger
+    #
+    # @return [Logger]
+    #
     def logger
       SidekiqUniqueJobs.logger
     end
 
+    #
     # Logs a message at debug level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
-    def log_debug(message_or_exception = nil, &block)
-      logger.debug(message_or_exception, &block)
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
+    def log_debug(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.debug(message, &block)
       nil
     end
 
+    #
     # Logs a message at info level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
-    def log_info(message_or_exception = nil, &block)
-      logger.info(message_or_exception, &block)
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
+    def log_info(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.info(message, &block)
       nil
     end
 
+    #
     # Logs a message at warn level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
-    def log_warn(message_or_exception = nil, &block)
-      logger.warn(message_or_exception, &block)
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
+    def log_warn(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.warn(message, &block)
       nil
     end
 
+    #
     # Logs a message at error level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
-    def log_error(message_or_exception = nil, &block)
-      logger.error(message_or_exception, &block)
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
+    def log_error(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.error(message, &block)
       nil
     end
 
+    #
     # Logs a message at fatal level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
-    def log_fatal(message_or_exception = nil, &block)
-      logger.fatal(message_or_exception, &block)
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
+    def log_fatal(message_or_exception = nil, item = nil, &block)
+      message = build_message(message_or_exception, item)
+      logger.fatal(message, &block)
+
       nil
     end
 
-    def logging_context(middleware_class, job_hash)
-      digest = job_hash["unique_digest"]
-      if defined?(Sidekiq::Logging)
-        "#{middleware_class} #{"DIG-#{digest}" if digest}"
-      else
-        { middleware: middleware_class, unique_digest: digest }
+    #
+    # Build a log message
+    #
+    # @param [String, Exception] message_or_exception an entry to log
+    # @param [Hash] item the sidekiq job hash
+    #
+    # @return [String] a complete log entry
+    #
+    def build_message(message_or_exception, item = nil)
+      return nil if message_or_exception.nil?
+      return message_or_exception if item.nil?
+
+      message = message_or_exception.dup
+      details = item.slice(LOCK, QUEUE, CLASS, JID, LOCK_DIGEST).each_with_object([]) do |(key, value), memo|
+        memo << "#{key}=#{value}"
+      end
+      message << " ("
+      message << details.join(" ")
+      message << ")"
+
+      message
+    end
+
+    #
+    # Wraps the middleware logic with context aware logging
+    #
+    #
+    # @return [void]
+    #
+    # @yieldreturn [void] yield to the middleware instance
+    #
+    def with_logging_context
+      with_configured_loggers_context do
+        return yield
       end
+
+      nil # Need to make sure we don't return anything here
+    end
+
+    #
+    # Attempt to setup context aware logging for the given logger
+    #
+    #
+    # @return [void]
+    #
+    # @yield
+    #
+    def with_configured_loggers_context(&block)
+      logger_method.call(logging_context, &block)
+    end
+
+    #
+    # Setup some variables to add to each log line
+    #
+    #
+    # @return [Hash] the context to use for each log line
+    #
+    def logging_context
+      raise NotImplementedError, "#{__method__} needs to be implemented in #{self.class}"
+    end
+
+    private
+
+    #
+    # A memoized method to use for setting up a logging context
+    #
+    #
+    # @return [proc] the method to call
+    #
+    def logger_method
+      @logger_method ||= sidekiq_context_method
+      @logger_method ||= sidekiq_logger_context_method
+      @logger_method ||= sidekiq_logging_context_method
+      @logger_method ||= no_sidekiq_context_method
+    end
+
+    #
+    # Checks if the logger respond to `with_context`.
+    #
+    # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
+    #
+    #
+    # @return [true,false]
+    #
+    def logger_respond_to_with_context?
+      logger.respond_to?(:with_context)
+    end
+
+    #
+    # Checks if the logger context takes a hash argument
+    #
+    # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
+    #
+    #
+    # @return [true,false]
+    #
+    def logger_context_hash?
+      defined?(Sidekiq::Context) || logger_respond_to_with_context?
+    end
+
+    def sidekiq_context_method
+      Sidekiq::Context.method(:with) if defined?(Sidekiq::Context)
+    end
+
+    def sidekiq_logger_context_method
+      logger.method(:with_context) if logger_respond_to_with_context?
+    end
+
+    def sidekiq_logging_context_method
+      Sidekiq::Logging.method(:with_context) if defined?(Sidekiq::Logging)
+    end
+
+    def no_sidekiq_context_method
+      method(:fake_logger_context)
+    end
+
+    def fake_logger_context(_context)
+      logger.warn "Don't know how to setup the logging context. Please open a feature request:" \
+                  " https://github.com/mhenrixon/sidekiq-unique-jobs/issues/new?template=feature_request.md"
+
+      yield
     end
   end
 end

data/lib/sidekiq_unique_jobs/lua/delete.lua

@@ -0,0 +1,51 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+-------- BEGIN lock arguments ---------
+local job_id       = ARGV[1]
+local pttl         = tonumber(ARGV[2])
+local lock_type    = ARGV[3]
+local limit        = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = tostring(ARGV[6]) == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = tostring(ARGV[9])
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete.lua --------
+log_debug("BEGIN delete", digest)
+
+local redis_version  = toversion(redisversion)
+local count          = 0
+local del_cmd        = "DEL"
+
+log_debug("ZREM", digests, digest)
+count = count + redis.call("ZREM", digests, digest)
+
+if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
+
+log_debug(del_cmd, digest, queued, primed, locked, info)
+count = count + redis.call(del_cmd, digest, queued, primed, locked, info)
+
+
+log("Deleted (" .. count .. ") keys")
+log_debug("END delete (" .. count .. ") keys for:", digest)
+
+return count
+--------  END delete.lua --------

data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua

@@ -0,0 +1,42 @@
+-------- BEGIN keys ---------
+local digest     = KEYS[1]
+local queued     = KEYS[2]
+local primed     = KEYS[3]
+local locked     = KEYS[4]
+local run_digest = KEYS[5]
+local run_queued = KEYS[6]
+local run_primed = KEYS[7]
+local run_locked = KEYS[8]
+local digests    = KEYS[9]
+--------  END keys  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[1])
+local debug_lua    = ARGV[2] == "true"
+local max_history  = tonumber(ARGV[3])
+local script_name  = tostring(ARGV[4]) .. ".lua"
+local redisversion = tostring(ARGV[5])
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+--------  BEGIN delete_by_digest.lua --------
+local counter       = 0
+local redis_version = toversion(redisversion)
+local del_cmd       = "DEL"
+
+log_debug("BEGIN delete_by_digest:", digest)
+
+if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
+
+log_debug(del_cmd, digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
+counter = redis.call(del_cmd, digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
+
+log_debug("ZREM", digests, digest)
+redis.call("ZREM", digests, digest)
+
+log_debug("END delete_by_digest:", digest, "(deleted " ..  counter .. " keys)")
+return counter
+--------   END delete_by_digest.lua  --------

data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua

@@ -0,0 +1,38 @@
+-------- BEGIN keys ---------
+local queue        = KEYS[1]
+local schedule_set = KEYS[2]
+local retry_set    = KEYS[3]
+--------  END keys  ---------
+
+-------- BEGIN Arguments ---------
+local digest       = ARGV[1]
+--------  END Arguments  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_delete_from_queue.lua" %>
+<%= include_partial "shared/_delete_from_sorted_set.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete_job_by_digest.lua --------
+local result = delete_from_queue(queue, digest)
+if result then
+  return result
+end
+
+result = delete_from_sorted_set(schedule_set, digest)
+if result then
+  return result
+end
+
+result = delete_from_sorted_set(retry_set, digest)
+return result
+--------   END delete_job_by_digest.lua  --------

data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua

@@ -0,0 +1,26 @@
+-------- BEGIN keys ---------
+local digest = KEYS[1]
+--------  END keys  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_find_digest_in_queues.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete_orphaned.lua --------
+log_debug("BEGIN")
+local result = find_digest_in_queues(digest)
+log_debug("END")
+if result and result ~= nil then
+  return result
+end
+--------   END delete_orphaned.lua  --------

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -0,0 +1,93 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id       = ARGV[1]
+local pttl         = tonumber(ARGV[2])
+local lock_type    = ARGV[3]
+local limit        = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+---------  BEGIN lock.lua ---------
+log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
+
+if redis.call("HEXISTS", locked, job_id) == 1 then
+  log_debug(locked, "already locked with job_id:", job_id)
+  log("Duplicate")
+
+  log_debug("LREM", queued, -1, job_id)
+  redis.call("LREM", queued, -1, job_id)
+
+  log_debug("LREM", primed, 1, job_id)
+  redis.call("LREM", primed, 1, job_id)
+
+  return job_id
+end
+
+local locked_count   = redis.call("HLEN", locked)
+local within_limit   = limit > locked_count
+local limit_exceeded = not within_limit
+
+if limit_exceeded then
+  log_debug("Limit exceeded:", digest, "(",  locked_count, "of", limit, ")")
+  log("Limited")
+  return nil
+end
+
+log_debug("ZADD", digests, current_time, digest)
+redis.call("ZADD", digests, current_time, digest)
+
+log_debug("HSET", locked, job_id, current_time)
+redis.call("HSET", locked, job_id, current_time)
+
+log_debug("LREM", queued, -1, job_id)
+redis.call("LREM", queued, -1, job_id)
+
+log_debug("LREM", primed, 1, job_id)
+redis.call("LREM", primed, 1, job_id)
+
+-- The Sidekiq client sets pttl
+if pttl and pttl > 0 then
+  log_debug("PEXPIRE", digest, pttl)
+  redis.call("PEXPIRE", digest, pttl)
+
+  log_debug("PEXPIRE", locked, pttl)
+  redis.call("PEXPIRE", locked, pttl)
+
+  log_debug("PEXPIRE", info, pttl)
+  redis.call("PEXPIRE", info, pttl)
+end
+
+log_debug("PEXPIRE", queued, 1000)
+redis.call("PEXPIRE", queued, 1000)
+
+log_debug("PEXPIRE", primed, 1000)
+redis.call("PEXPIRE", primed, 1000)
+
+log("Locked")
+log_debug("END lock digest:", digest, "job_id:", job_id)
+return job_id
+----------  END lock.lua  ----------

data/lib/sidekiq_unique_jobs/lua/locked.lua

@@ -0,0 +1,35 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+-------- BEGIN lock arguments ---------
+local job_id = ARGV[1]
+-------- END lock arguments -----------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+---------  END local functions ---------
+
+
+--------  BEGIN locked.lua --------
+if redis.call("HEXISTS", locked, job_id) == 1 then
+  log_debug("Locked", digest, "job_id:", job_id)
+  return 1
+else
+  log_debug("NOT Locked", digest, "job_id:", job_id)
+  return -1
+end
+---------  END locked.lua ---------

data/lib/sidekiq_unique_jobs/lua/queue.lua

@@ -0,0 +1,87 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id    = ARGV[1]      -- The job_id that was previously primed
+local pttl      = tonumber(ARGV[2])
+local lock_type = ARGV[3]
+local limit     = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+---------  END injected arguments ---------
+
+
+--------  BEGIN Variables --------
+local queued_count = redis.call("LLEN", queued)
+local locked_count = redis.call("HLEN", locked)
+local within_limit = limit > locked_count
+local limit_exceeded = not within_limit
+--------   END Variables  --------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN queue.lua --------
+log_debug("BEGIN queue with key:", digest, "for job:", job_id)
+
+if redis.call("HEXISTS", locked, job_id) == 1 then
+  log_debug("HEXISTS", locked, job_id, "== 1")
+  log("Duplicate")
+  return job_id
+end
+
+local prev_jid = redis.call("GET", digest)
+log_debug("job_id:", job_id, "prev_jid:", prev_jid)
+if not prev_jid or prev_jid == false then
+  log_debug("SET", digest, job_id)
+  redis.call("SET", digest, job_id)
+elseif prev_jid == job_id then
+  log_debug(digest, "already queued with job_id:", job_id)
+  log("Duplicate")
+  return job_id
+else
+  -- TODO: Consider constraining the total count of both locked and queued?
+  if within_limit and queued_count < limit then
+    log_debug("Within limit:", digest, "(",  locked_count, "of", limit, ")", "queued (", queued_count, "of", limit, ")")
+    log_debug("SET", digest, job_id, "(was", prev_jid, ")")
+    redis.call("SET", digest, job_id)
+  else
+    log_debug("Limit exceeded:", digest, "(",  locked_count, "of", limit, ")")
+    log("Limit exceeded", prev_jid)
+    return prev_jid
+  end
+end
+
+log_debug("LPUSH", queued, job_id)
+redis.call("LPUSH", queued, job_id)
+
+-- The Sidekiq client should only set pttl for until_expired
+-- The Sidekiq server should set pttl for all other jobs
+if pttl and pttl > 0 then
+  log_debug("PEXPIRE", digest, pttl)
+  redis.call("PEXPIRE", digest, pttl)
+  log_debug("PEXPIRE", queued, pttl)
+  redis.call("PEXPIRE", queued, pttl)
+end
+
+log("Queued")
+log_debug("END queue with key:", digest, "for job:", job_id)
+return job_id
+--------  END queue.lua --------