sidekiq-unique-jobs 6.0.25 → 7.0.0.beta2

data/lib/sidekiq_unique_jobs/logging.rb

@@ -5,63 +5,189 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
   module Logging
-    # A convenience method for using the configured logger
+    def self.included(base)
+      base.send(:extend, self)
+    end
+
+    #
+    # A convenience method for using the configured gem logger
+    #
+    # @see SidekiqUniqueJobs#.logger
+    #
+    # @return [Logger]
+    #
     def logger
       SidekiqUniqueJobs.logger
     end
 
+    #
     # Logs a message at debug level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_debug(message_or_exception = nil, &block)
       logger.debug(message_or_exception, &block)
       nil
     end
 
+    #
     # Logs a message at info level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_info(message_or_exception = nil, &block)
       logger.info(message_or_exception, &block)
       nil
     end
 
+    #
     # Logs a message at warn level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_warn(message_or_exception = nil, &block)
       logger.warn(message_or_exception, &block)
       nil
     end
 
+    #
     # Logs a message at error level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_error(message_or_exception = nil, &block)
       logger.error(message_or_exception, &block)
       nil
     end
 
+    #
     # Logs a message at fatal level
-    # @param message_or_exception [String, Exception] the message or exception to log
-    # @yield the message or exception to use for log message
-    #   Used for compatibility with logger
+    #
+    # @param [String, Exception] message_or_exception the message or exception to log
+    #
+    # @return [void]
+    #
+    # @yield [String, Exception] the message or exception to use for log message
+    #
     def log_fatal(message_or_exception = nil, &block)
       logger.fatal(message_or_exception, &block)
       nil
     end
 
-    def logging_context(middleware_class, job_hash)
-      digest = job_hash["unique_digest"]
-      if defined?(Sidekiq::Logging)
-        "#{middleware_class} #{"DIG-#{digest}" if digest}"
-      else
-        { middleware: middleware_class, unique_digest: digest }
+    #
+    # Wraps the middleware logic with context aware logging
+    #
+    #
+    # @return [void]
+    #
+    # @yieldreturn [void] yield to the middleware instance
+    #
+    def with_logging_context
+      with_configured_loggers_context do
+        return yield
       end
+
+      nil # Need to make sure we don't return anything here
+    end
+
+    #
+    # Attempt to setup context aware logging for the given logger
+    #
+    #
+    # @return [void]
+    #
+    # @yield
+    #
+    def with_configured_loggers_context
+      logger_method.call(logging_context) { yield }
+    end
+
+    #
+    # Setup some variables to add to each log line
+    #
+    #
+    # @return [Hash] the context to use for each log line
+    #
+    def logging_context
+      raise NotImplementedError, "#{__method__} needs to be implemented in #{self.class}"
+    end
+
+    private
+
+    #
+    # A memoized method to use for setting up a logging context
+    #
+    #
+    # @return [proc] the method to call
+    #
+    def logger_method
+      @logger_method ||= sidekiq_context_method
+      @logger_method ||= sidekiq_logger_context_method
+      @logger_method ||= sidekiq_logging_context_method
+      @logger_method ||= no_sidekiq_context_method
+    end
+
+    #
+    # Checks if the logger respond to `with_context`.
+    #
+    # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
+    #
+    #
+    # @return [true,false]
+    #
+    def logger_respond_to_with_context?
+      logger.respond_to?(:with_context)
+    end
+
+    #
+    # Checks if the logger context takes a hash argument
+    #
+    # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
+    #
+    #
+    # @return [true,false]
+    #
+    def logger_context_hash?
+      defined?(Sidekiq::Context) || logger_respond_to_with_context?
+    end
+
+    def sidekiq_context_method
+      Sidekiq::Context.method(:with) if defined?(Sidekiq::Context)
+    end
+
+    def sidekiq_logger_context_method
+      logger.method(:with_context)           if logger_respond_to_with_context?
+    end
+
+    def sidekiq_logging_context_method
+      Sidekiq::Logging.method(:with_context) if defined?(Sidekiq::Logging)
+    end
+
+    def no_sidekiq_context_method
+      method(:fake_logger_context)
+    end
+
+    def fake_logger_context(_context)
+      logger.warn "Don't know how to setup the logging context. Please open a feature request:" \
+                  " https://github.com/mhenrixon/sidekiq-unique-jobs/issues/new?template=feature_request.md"
+
+      yield
     end
   end
 end

data/lib/sidekiq_unique_jobs/logging/middleware_context.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Provides the sidekiq middleware that makes the gem work
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  module Logging
+    #
+    # Context aware logging for Sidekiq Middlewares
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    #
+    module Middleware
+      include Logging
+
+      def self.included(base)
+        base.class_eval do
+          extend Logging::Middleware
+        end
+      end
+
+      #
+      # Provides a logging context for Sidekiq Middlewares
+      #
+      #
+      # @return [Hash] when logger responds to `:with_context`
+      # @return [String] when logger does not responds to `:with_context`
+      #
+      def logging_context
+        middleware = is_a?(SidekiqUniqueJobs::Middleware::Client) ? :client : :server
+        digest     = item[UNIQUE_DIGEST]
+        lock_type  = item[LOCK]
+
+        if logger_context_hash?
+          { "uniquejobs" => middleware, lock_type => digest }
+        else
+          "uniquejobs-#{middleware} #{"DIG-#{digest}" if digest}"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lua/delete.lua

@@ -0,0 +1,51 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+-------- BEGIN lock arguments ---------
+local job_id       = ARGV[1]
+local pttl         = tonumber(ARGV[2])
+local lock_type    = ARGV[3]
+local limit        = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = tostring(ARGV[6]) == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = tostring(ARGV[9])
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete.lua --------
+log_debug("BEGIN delete", digest)
+
+local redis_version  = toversion(redisversion)
+local count          = 0
+local del_cmd        = "DEL"
+
+log_debug("ZREM", digests, digest)
+count = count + redis.call("ZREM", digests, digest)
+
+if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
+
+log_debug(del_cmd, digest, queued, primed, locked, info)
+count = count + redis.call(del_cmd, digest, queued, primed, locked, info)
+
+
+log("Deleted (" .. count .. ") keys")
+log_debug("END delete (" .. count .. ") keys for:", digest)
+
+return count
+--------  END delete.lua --------

data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua

@@ -0,0 +1,46 @@
+-------- BEGIN keys ---------
+local digest  = KEYS[1]
+local digests = KEYS[2]
+--------  END keys  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[1])
+local debug_lua    = ARGV[2] == "true"
+local max_history  = tonumber(ARGV[3])
+local script_name  = tostring(ARGV[4]) .. ".lua"
+local redisversion = tostring(ARGV[5])
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+--------  BEGIN Variables  --------
+local queued     = digest .. ":QUEUED"
+local primed     = digest .. ":PRIMED"
+local locked     = digest .. ":LOCKED"
+local run_digest = digest .. ":RUN"
+local run_queued = digest .. ":RUN:QUEUED"
+local run_primed = digest .. ":RUN:PRIMED"
+local run_locked = digest .. ":RUN:LOCKED"
+--------   END Variables   --------
+
+
+--------  BEGIN delete_by_digest.lua --------
+local counter       = 0
+local redis_version = toversion(redisversion)
+local del_cmd   = "DEL"
+
+log_debug("BEGIN delete_by_digest:", digest)
+
+if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
+
+log_debug(del_cmd, digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
+counter = redis.call(del_cmd, digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
+
+log_debug("ZREM", digests, digest)
+redis.call("ZREM", digests, digest)
+
+log_debug("END delete_by_digest:", digest, "(deleted " ..  counter .. " keys)")
+return counter
+--------   END delete_by_digest.lua  --------

data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua

@@ -0,0 +1,38 @@
+-------- BEGIN keys ---------
+local queue        = KEYS[1]
+local schedule_set = KEYS[2]
+local retry_set    = KEYS[3]
+--------  END keys  ---------
+
+-------- BEGIN Arguments ---------
+local digest       = ARGV[1]
+--------  END Arguments  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_delete_from_queue.lua" %>
+<%= include_partial "shared/_delete_from_sorted_set.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete_job_by_digest.lua --------
+local result = delete_from_queue(queue, digest)
+if result then
+  return result
+end
+
+result = delete_from_sorted_set(schedule_set, digest)
+if result then
+  return result
+end
+
+result = delete_from_sorted_set(retry_set, digest)
+return result
+--------   END delete_job_by_digest.lua  --------

data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua

@@ -0,0 +1,26 @@
+-------- BEGIN keys ---------
+local digest = KEYS[1]
+--------  END keys  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_find_digest_in_queues.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete_orphaned.lua --------
+log_debug("BEGIN")
+local result = find_digest_in_queues(digest)
+log_debug("END")
+if result and result ~= nil then
+  return result
+end
+--------   END delete_orphaned.lua  --------

data/lib/sidekiq_unique_jobs/lua/find_digest_in_sorted_set.lua

@@ -0,0 +1,24 @@
+local function find_digest_in_sorted_set(name, digest)
+  local cursor  = 0
+  local count   = 5
+  local pattern = "*" .. digest .. "*"
+  local found   = false
+
+  log_debug("searching in:", name,
+            "for digest:", digest,
+            "cursor:", cursor)
+  repeat
+    local pagination  = redis.call("ZSCAN", name, cursor, "MATCH", pattern, "COUNT", count)
+    local next_cursor = pagination[1]
+    local items       = pagination[2]
+
+    if #items > 0 then
+      log_debug("Found digest", digest, "in zset:", name)
+      found = true
+    end
+
+    cursor = next_cursor
+  until found == true or cursor == "0"
+
+  return found
+end