sidekiq-unique-jobs 6.0.24 → 7.0.4

data/lib/sidekiq_unique_jobs/logging.rb

@@ -3,65 +3,191 @@
 module SidekiqUniqueJobs
   # Utility module for reducing the number of uses of logger.
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Logging
-    # A convenience method for using the configured logger
+    def self.included(base)
+      base.send(:extend, self)
+    end
+
+    #
+    # A convenience method for using the configured gem logger
+    #
+    # @see SidekiqUniqueJobs#.logger
+    #
+    # @return [Logger]
+    #
     def logger
       SidekiqUniqueJobs.logger
     end
 
+    #
     # Logs a message at debug level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_debug(message_or_exception = nil, &block)
       logger.debug(message_or_exception, &block)
       nil
     end
 
+    #
     # Logs a message at info level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_info(message_or_exception = nil, &block)
       logger.info(message_or_exception, &block)
       nil
     end
 
+    #
     # Logs a message at warn level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_warn(message_or_exception = nil, &block)
       logger.warn(message_or_exception, &block)
       nil
     end
 
+    #
     # Logs a message at error level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_error(message_or_exception = nil, &block)
       logger.error(message_or_exception, &block)
       nil
     end
 
+    #
     # Logs a message at fatal level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_fatal(message_or_exception = nil, &block)
       logger.fatal(message_or_exception, &block)
       nil
     end
 
-    def logging_context(middleware_class, job_hash)
-      digest = job_hash["unique_digest"]
-      if defined?(Sidekiq::Logging)
-        "#{middleware_class} #{"DIG-#{digest}" if digest}"
-      else
-        { middleware: middleware_class, unique_digest: digest }
+    #
+    # Wraps the middleware logic with context aware logging
+    #
+    #
+    # @return [void]
+    #
+    # @yieldreturn [void] yield to the middleware instance
+    #
+    def with_logging_context
+      with_configured_loggers_context do
+        return yield
       end
+
+      nil # Need to make sure we don't return anything here
+    end
+
+    #
+    # Attempt to setup context aware logging for the given logger
+    #
+    #
+    # @return [void]
+    #
+    # @yield
+    #
+    def with_configured_loggers_context(&block)
+      logger_method.call(logging_context, &block)
+    end
+
+    #
+    # Setup some variables to add to each log line
+    #
+    #
+    # @return [Hash] the context to use for each log line
+    #
+    def logging_context
+      raise NotImplementedError, "#{__method__} needs to be implemented in #{self.class}"
+    end
+
+    private
+
+    #
+    # A memoized method to use for setting up a logging context
+    #
+    #
+    # @return [proc] the method to call
+    #
+    def logger_method
+      @logger_method ||= sidekiq_context_method
+      @logger_method ||= sidekiq_logger_context_method
+      @logger_method ||= sidekiq_logging_context_method
+      @logger_method ||= no_sidekiq_context_method
+    end
+
+    #
+    # Checks if the logger respond to `with_context`.
+    #
+    # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
+    #
+    #
+    # @return [true,false]
+    #
+    def logger_respond_to_with_context?
+      logger.respond_to?(:with_context)
+    end
+
+    #
+    # Checks if the logger context takes a hash argument
+    #
+    # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
+    #
+    #
+    # @return [true,false]
+    #
+    def logger_context_hash?
+      defined?(Sidekiq::Context) || logger_respond_to_with_context?
+    end
+
+    def sidekiq_context_method
+      Sidekiq::Context.method(:with) if defined?(Sidekiq::Context)
+    end
+
+    def sidekiq_logger_context_method
+      logger.method(:with_context)           if logger_respond_to_with_context?
+    end
+
+    def sidekiq_logging_context_method
+      Sidekiq::Logging.method(:with_context) if defined?(Sidekiq::Logging)
+    end
+
+    def no_sidekiq_context_method
+      method(:fake_logger_context)
+    end
+
+    def fake_logger_context(_context)
+      logger.warn "Don't know how to setup the logging context. Please open a feature request:" \
+                  " https://github.com/mhenrixon/sidekiq-unique-jobs/issues/new?template=feature_request.md"
+
+      yield
     end
   end
 end

data/lib/sidekiq_unique_jobs/logging/middleware_context.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Provides the sidekiq middleware that makes the gem work
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  module Logging
+    #
+    # Context aware logging for Sidekiq Middlewares
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    module Middleware
+      include Logging
+
+      def self.included(base)
+        base.class_eval do
+          extend Logging::Middleware
+        end
+      end
+
+      #
+      # Provides a logging context for Sidekiq Middlewares
+      #
+      #
+      # @return [Hash] when logger responds to `:with_context`
+      # @return [String] when logger does not responds to `:with_context`
+      #
+      def logging_context
+        middleware = is_a?(SidekiqUniqueJobs::Middleware::Client) ? :client : :server
+        digest     = item[LOCK_DIGEST]
+        lock_type  = item[LOCK]
+
+        if logger_context_hash?
+          { "uniquejobs" => middleware, lock_type => digest }
+        else
+          "uniquejobs-#{middleware} #{"DIG-#{digest}" if digest}"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lua/delete.lua

@@ -0,0 +1,51 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+-------- BEGIN lock arguments ---------
+local job_id       = ARGV[1]
+local pttl         = tonumber(ARGV[2])
+local lock_type    = ARGV[3]
+local limit        = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = tostring(ARGV[6]) == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = tostring(ARGV[9])
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete.lua --------
+log_debug("BEGIN delete", digest)
+
+local redis_version  = toversion(redisversion)
+local count          = 0
+local del_cmd        = "DEL"
+
+log_debug("ZREM", digests, digest)
+count = count + redis.call("ZREM", digests, digest)
+
+if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
+
+log_debug(del_cmd, digest, queued, primed, locked, info)
+count = count + redis.call(del_cmd, digest, queued, primed, locked, info)
+
+
+log("Deleted (" .. count .. ") keys")
+log_debug("END delete (" .. count .. ") keys for:", digest)
+
+return count
+--------  END delete.lua --------

data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua

@@ -0,0 +1,42 @@
+-------- BEGIN keys ---------
+local digest     = KEYS[1]
+local queued     = KEYS[2]
+local primed     = KEYS[3]
+local locked     = KEYS[4]
+local run_digest = KEYS[5]
+local run_queued = KEYS[6]
+local run_primed = KEYS[7]
+local run_locked = KEYS[8]
+local digests    = KEYS[9]
+--------  END keys  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[1])
+local debug_lua    = ARGV[2] == "true"
+local max_history  = tonumber(ARGV[3])
+local script_name  = tostring(ARGV[4]) .. ".lua"
+local redisversion = tostring(ARGV[5])
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+--------  BEGIN delete_by_digest.lua --------
+local counter       = 0
+local redis_version = toversion(redisversion)
+local del_cmd       = "DEL"
+
+log_debug("BEGIN delete_by_digest:", digest)
+
+if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
+
+log_debug(del_cmd, digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
+counter = redis.call(del_cmd, digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
+
+log_debug("ZREM", digests, digest)
+redis.call("ZREM", digests, digest)
+
+log_debug("END delete_by_digest:", digest, "(deleted " ..  counter .. " keys)")
+return counter
+--------   END delete_by_digest.lua  --------

data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua

@@ -0,0 +1,38 @@
+-------- BEGIN keys ---------
+local queue        = KEYS[1]
+local schedule_set = KEYS[2]
+local retry_set    = KEYS[3]
+--------  END keys  ---------
+
+-------- BEGIN Arguments ---------
+local digest       = ARGV[1]
+--------  END Arguments  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_delete_from_queue.lua" %>
+<%= include_partial "shared/_delete_from_sorted_set.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete_job_by_digest.lua --------
+local result = delete_from_queue(queue, digest)
+if result then
+  return result
+end
+
+result = delete_from_sorted_set(schedule_set, digest)
+if result then
+  return result
+end
+
+result = delete_from_sorted_set(retry_set, digest)
+return result
+--------   END delete_job_by_digest.lua  --------

data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua

@@ -0,0 +1,26 @@
+-------- BEGIN keys ---------
+local digest = KEYS[1]
+--------  END keys  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_find_digest_in_queues.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete_orphaned.lua --------
+log_debug("BEGIN")
+local result = find_digest_in_queues(digest)
+log_debug("END")
+if result and result ~= nil then
+  return result
+end
+--------   END delete_orphaned.lua  --------

data/lib/sidekiq_unique_jobs/lua/lock.lua

@@ -0,0 +1,93 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id       = ARGV[1]
+local pttl         = tonumber(ARGV[2])
+local lock_type    = ARGV[3]
+local limit        = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+---------  BEGIN lock.lua ---------
+log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
+
+if redis.call("HEXISTS", locked, job_id) == 1 then
+  log_debug(locked, "already locked with job_id:", job_id)
+  log("Duplicate")
+
+  log_debug("LREM", queued, -1, job_id)
+  redis.call("LREM", queued, -1, job_id)
+
+  log_debug("LREM", primed, 1, job_id)
+  redis.call("LREM", primed, 1, job_id)
+
+  return job_id
+end
+
+local locked_count   = redis.call("HLEN", locked)
+local within_limit   = limit > locked_count
+local limit_exceeded = not within_limit
+
+if limit_exceeded then
+  log_debug("Limit exceeded:", digest, "(",  locked_count, "of", limit, ")")
+  log("Limited")
+  return nil
+end
+
+log_debug("ZADD", digests, current_time, digest)
+redis.call("ZADD", digests, current_time, digest)
+
+log_debug("HSET", locked, job_id, current_time)
+redis.call("HSET", locked, job_id, current_time)
+
+log_debug("LREM", queued, -1, job_id)
+redis.call("LREM", queued, -1, job_id)
+
+log_debug("LREM", primed, 1, job_id)
+redis.call("LREM", primed, 1, job_id)
+
+-- The Sidekiq client sets pttl
+if pttl and pttl > 0 then
+  log_debug("PEXPIRE", digest, pttl)
+  redis.call("PEXPIRE", digest, pttl)
+
+  log_debug("PEXPIRE", locked, pttl)
+  redis.call("PEXPIRE", locked, pttl)
+end
+
+log_debug("PEXPIRE", queued, 1000)
+redis.call("PEXPIRE", queued, 1000)
+
+log_debug("PEXPIRE", primed, 1000)
+redis.call("PEXPIRE", primed, 1000)
+
+log_debug("PEXPIRE", info, 1000)
+redis.call("PEXPIRE", info, 1000)
+
+log("Locked")
+log_debug("END lock digest:", digest, "job_id:", job_id)
+return job_id
+----------  END lock.lua  ----------