sidekiq 6.5.3 → 6.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46b428cecc373a6b11d27e12622dcfc8661b138e244c16575ad49362c672a60e
-  data.tar.gz: 64fc302563e615eec63e8701b581949cb90738c799714257d98bbd6f359cff45
+  metadata.gz: 7e12ddc46413e3e476d620e04428a86476f4826d7802575110b5c3697a096137
+  data.tar.gz: 802bdbc7c0d45a9b793222baca250e292ce852d67902446bc75a0961a36f5677
 SHA512:
-  metadata.gz: 6b62cefbc386b0113ffe47ca4e10b6752cfd86d973c0a8143e116ad45e2e6c680f0de686c9e2750760c14c71f17e3c13375145ae16a98f6cceab96908ca4cf61
-  data.tar.gz: 0dfc564242453b670e13e656098a3294e0c9831efa9520e025b08d3c265ef6cb148fb4519af78406a43d9972b9e83db1c9d8271ee7288f57401a5e7bb30d401e
+  metadata.gz: 6180da0e7aefa1573d6b283c56ad365063b290afb5c10a15dc83fb367cb981391d8304acc9ebb1219279a749da71e0af8b37979baea770b0b1efeb04627e0e4c
+  data.tar.gz: 35c51c58b73450c6bbd3479cc55d873ed54a6a1ce6a5b7414ca78fc807984acea11d4697ce8c05bdb2e9eb799c30dc7b377872b4c48a3de60bee5d6d71e07b3a

data/Changes.md

@@ -2,6 +2,24 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
+6.5.6
+----------
+
+- Fix deprecation warnings with redis-rb 4.8.0 [#5484]
+- Lock redis-rb to < 5.0 as we are moving to redis-client in Sidekiq 7.0
+
+6.5.5
+----------
+
+- Fix require issue with job_retry.rb [#5462]
+- Improve Sidekiq::Web compatibility with Rack 3.x
+
+6.5.4
+----------
+
+- Fix invalid code on Ruby 2.5 [#5460]
+- Fix further metrics dependency issues [#5457]
+
 6.5.3
 ----------
 

data/lib/sidekiq/api.rb

@@ -5,8 +5,11 @@ require "sidekiq"
 require "zlib"
 require "set"
 require "base64"
-require "sidekiq/metrics/deploy"
-require "sidekiq/metrics/query"
+
+if ENV["SIDEKIQ_METRICS_BETA"]
+  require "sidekiq/metrics/deploy"
+  require "sidekiq/metrics/query"
+end
 
 module Sidekiq
   # Retrieve runtime statistics from Redis regarding
@@ -313,7 +316,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.multi do |transaction|
           transaction.unlink(@rname)
-          transaction.srem("queues", name)
+          transaction.srem("queues", [name])
         end
       end
       true
@@ -483,7 +486,7 @@ module Sidekiq
       # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
       # memory yet so the YAML can't be loaded.
       # TODO is this still necessary? Zeitwerk reloader should handle?
-      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.config[:environment] == "development"
+      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
       default
     end
 

data/lib/sidekiq/client.rb

@@ -231,7 +231,7 @@ module Sidekiq
           entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
         }
-        conn.sadd("queues", queue)
+        conn.sadd("queues", [queue])
         conn.lpush("queue:#{queue}", to_push)
       end
     end

data/lib/sidekiq/fetch.rb

@@ -33,7 +33,7 @@ module Sidekiq # :nodoc:
       @queues = @config[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
         @queues.uniq!
-        @queues << TIMEOUT
+        @queues << {timeout: TIMEOUT}
       end
     end
 
@@ -83,7 +83,7 @@ module Sidekiq # :nodoc:
       else
         permute = @queues.shuffle
         permute.uniq!
-        permute << TIMEOUT
+        permute << {timeout: TIMEOUT}
         permute
       end
     end

data/lib/sidekiq/job_retry.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
-require "sidekiq/scheduled"
-require "sidekiq/api"
-
 require "zlib"
 require "base64"
+require "sidekiq/component"
 
 module Sidekiq
   ##
@@ -203,14 +201,13 @@ module Sidekiq
         nil
       end
 
-      delay = if Integer === rv && rv > 0
-        rv
+      delay = (count**4) + 15
+      if Integer === rv && rv > 0
+        delay = rv
       elsif rv == :discard
         return [:discard, nil] # do nothing, job goes poof
       elsif rv == :kill
         return [:kill, nil]
-      else
-        (count**4) + 15
       end
 
       [:default, delay]
@@ -236,7 +233,15 @@ module Sidekiq
     def send_to_morgue(msg)
       logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
       payload = Sidekiq.dump_json(msg)
-      DeadSet.new.kill(payload, notify_failure: false)
+      now = Time.now.to_f
+
+      config.redis do |conn|
+        conn.multi do |xa|
+          xa.zadd("dead", now.to_s, payload)
+          xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
+          xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
+        end
+      end
     end
 
     def retry_attempts_from(msg_retry, default)

data/lib/sidekiq/launcher.rb

@@ -86,7 +86,7 @@ module Sidekiq
       # doesn't actually exit, it'll reappear in the Web UI.
       redis do |conn|
         conn.pipelined do |pipeline|
-          pipeline.srem("processes", identity)
+          pipeline.srem("processes", [identity])
           pipeline.unlink("#{identity}:work")
         end
       end
@@ -165,7 +165,7 @@ module Sidekiq
 
         _, exists, _, _, msg = redis { |conn|
           conn.multi { |transaction|
-            transaction.sadd("processes", key)
+            transaction.sadd("processes", [key])
             transaction.exists?(key)
             transaction.hmset(key, "info", to_json,
               "busy", curstate.size,

data/lib/sidekiq/metrics/deploy.rb

@@ -1,5 +1,5 @@
 require "sidekiq"
-require "date"
+require "time"
 
 # This file is designed to be required within the user's
 # deployment script; it should need a bare minimum of dependencies.
@@ -32,7 +32,7 @@ module Sidekiq
         key = "#{datecode}-marks"
         @pool.with do |c|
           c.pipelined do |pipe|
-            pipe.hsetnx(key, floor.rfc3339, label)
+            pipe.hsetnx(key, floor.iso8601, label)
             pipe.expire(key, MARK_TTL)
           end
         end

data/lib/sidekiq/metrics/query.rb

@@ -13,111 +13,140 @@ module Sidekiq
     # NB: all metrics and times/dates are UTC only. We specifically do not
     # support timezones.
     class Query
-      # :hour, :day, :month
-      attr_accessor :period
-
-      # a specific job class, e.g. "App::OrderJob"
-      attr_accessor :klass
-
-      # the date specific to the period
-      # for :day or :hour, something like Date.today or Date.new(2022, 7, 13)
-      # for :month, Date.new(2022, 7, 1)
-      attr_accessor :date
-
-      # for period = :hour, the specific hour, integer e.g. 1 or 18
-      # note that hours and minutes do not have a leading zero so minute-specific
-      # keys will look like "j|20220718|7:3" for data at 07:03.
-      attr_accessor :hour
-
       def initialize(pool: Sidekiq.redis_pool, now: Time.now)
         @time = now.utc
         @pool = pool
         @klass = nil
       end
 
-      # Get metric data from the last hour and roll it up
-      # into top processed count and execution time based on class.
-      def top_jobs
-        resultset = {}
-        resultset[:date] = @time.to_date
-        resultset[:period] = :hour
-        resultset[:ends_at] = @time
-        time = @time
+      # Get metric data for all jobs from the last hour
+      def top_jobs(minutes: 60)
+        result = Result.new
 
-        results = @pool.with do |conn|
+        time = @time
+        redis_results = @pool.with do |conn|
           conn.pipelined do |pipe|
-            resultset[:size] = 60
-            60.times do |idx|
+            minutes.times do |idx|
               key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
               pipe.hgetall key
+              result.prepend_bucket time
               time -= 60
             end
-            resultset[:starts_at] = time
           end
         end
 
-        t = Hash.new(0)
-        klsset = Set.new
-        # merge the per-minute data into a totals hash for the hour
-        results.each do |hash|
-          hash.each { |k, v| t[k] = t[k] + v.to_i }
-          klsset.merge(hash.keys.map { |k| k.split("|")[0] })
-        end
-        resultset[:job_classes] = klsset.delete_if { |item| item.size < 3 }
-        resultset[:totals] = t
-        top = t.each_with_object({}) do |(k, v), memo|
-          (kls, metric) = k.split("|")
-          memo[metric] ||= Hash.new(0)
-          memo[metric][kls] = v
+        time = @time
+        redis_results.each do |hash|
+          hash.each do |k, v|
+            kls, metric = k.split("|")
+            result.job_results[kls].add_metric metric, time, v.to_i
+          end
+          time -= 60
         end
 
-        sorted = {}
-        top.each_pair do |metric, hash|
-          sorted[metric] = hash.sort_by { |k, v| v }.reverse.to_h
-        end
-        resultset[:top_classes] = sorted
-        resultset
+        result.marks = fetch_marks(result.starts_at..result.ends_at)
+
+        result
       end
 
-      def for_job(klass)
-        resultset = {}
-        resultset[:date] = @time.to_date
-        resultset[:period] = :hour
-        resultset[:ends_at] = @time
-        marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
+      def for_job(klass, minutes: 60)
+        result = Result.new
 
         time = @time
-        initial = @pool.with do |conn|
+        redis_results = @pool.with do |conn|
           conn.pipelined do |pipe|
-            resultset[:size] = 60
-            60.times do |idx|
-              key = "j|#{time.strftime("%Y%m%d|%-H:%-M")}"
+            minutes.times do |idx|
+              key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
               pipe.hmget key, "#{klass}|ms", "#{klass}|p", "#{klass}|f"
+              result.prepend_bucket time
               time -= 60
             end
           end
         end
 
         time = @time
-        hist = Histogram.new(klass)
-        results = @pool.with do |conn|
-          initial.map do |(ms, p, f)|
-            tm = Time.utc(time.year, time.month, time.mday, time.hour, time.min, 0)
-            {
-              time: tm.iso8601,
-              epoch: tm.to_i,
-              ms: ms.to_i, p: p.to_i, f: f.to_i, hist: hist.fetch(conn, time)
-            }.tap { |x|
-              x[:mark] = marks[x[:time]] if marks[x[:time]]
-              time -= 60
-            }
+        @pool.with do |conn|
+          redis_results.each do |(ms, p, f)|
+            result.job_results[klass].add_metric "ms", time, ms.to_i if ms
+            result.job_results[klass].add_metric "p", time, p.to_i if p
+            result.job_results[klass].add_metric "f", time, f.to_i if f
+            result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time)
+            time -= 60
+          end
+        end
+
+        result.marks = fetch_marks(result.starts_at..result.ends_at)
+
+        result
+      end
+
+      class Result < Struct.new(:starts_at, :ends_at, :size, :buckets, :job_results, :marks)
+        def initialize
+          super
+          self.buckets = []
+          self.marks = []
+          self.job_results = Hash.new { |h, k| h[k] = JobResult.new }
+        end
+
+        def prepend_bucket(time)
+          buckets.unshift time.strftime("%H:%M")
+          self.ends_at ||= time
+          self.starts_at = time
+        end
+      end
+
+      class JobResult < Struct.new(:series, :hist, :totals)
+        def initialize
+          super
+          self.series = Hash.new { |h, k| h[k] = Hash.new(0) }
+          self.hist = Hash.new { |h, k| h[k] = [] }
+          self.totals = Hash.new(0)
+        end
+
+        def add_metric(metric, time, value)
+          totals[metric] += value
+          series[metric][time.strftime("%H:%M")] += value
+
+          # Include timing measurements in seconds for convenience
+          add_metric("s", time, value / 1000.0) if metric == "ms"
+        end
+
+        def add_hist(time, hist_result)
+          hist[time.strftime("%H:%M")] = hist_result
+        end
+
+        def total_avg(metric = "ms")
+          completed = totals["p"] - totals["f"]
+          totals[metric].to_f / completed
+        end
+
+        def series_avg(metric = "ms")
+          series[metric].each_with_object(Hash.new(0)) do |(bucket, value), result|
+            completed = series.dig("p", bucket) - series.dig("f", bucket)
+            result[bucket] = completed == 0 ? 0 : value.to_f / completed
           end
         end
+      end
+
+      class MarkResult < Struct.new(:time, :label)
+        def bucket
+          time.strftime("%H:%M")
+        end
+      end
 
-        resultset[:marks] = marks
-        resultset[:starts_at] = time
-        resultset[:data] = results
-        resultset
+      private
+
+      def fetch_marks(time_range)
+        [].tap do |result|
+          marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
+
+          marks.each do |timestamp, label|
+            time = Time.parse(timestamp)
+            if time_range.cover? time
+              result << MarkResult.new(time, label)
+            end
+          end
+        end
       end
     end
   end

data/lib/sidekiq/processor.rb

@@ -152,8 +152,14 @@ module Sidekiq
         job_hash = Sidekiq.load_json(jobstr)
       rescue => ex
         handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
-        # we can't notify because the job isn't a valid hash payload.
-        DeadSet.new.kill(jobstr, notify_failure: false)
+        now = Time.now.to_f
+        config.redis do |conn|
+          conn.multi do |xa|
+            xa.zadd("dead", now.to_s, jobstr)
+            xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
+            xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
+          end
+        end
         return uow.acknowledge
       end
 

data/lib/sidekiq/redis_connection.rb

@@ -46,8 +46,6 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
-
         # Issue #3303, redis-rb will silently retry an operation.
         # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
         # is performed twice but I believe this is much, much rarer

data/lib/sidekiq/scheduled.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "sidekiq"
-require "sidekiq/api"
 require "sidekiq/component"
 
 module Sidekiq
@@ -183,13 +182,8 @@ module Sidekiq
       end
 
       def process_count
-        # The work buried within Sidekiq::ProcessSet#cleanup can be
-        # expensive at scale. Cut it down by 90% with this counter.
-        # NB: This method is only called by the scheduler thread so we
-        # don't need to worry about the thread safety of +=.
-        pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
+        pcount = Sidekiq.redis { |conn| conn.scard("processes") }
         pcount = 1 if pcount == 0
-        @count_calls += 1
         pcount
       end
 

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.5.3"
+  VERSION = "6.5.6"
 end

data/lib/sidekiq/web/action.rb

@@ -15,11 +15,11 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, [res, {"Content-Type" => "text/plain"}, [res.to_s]]
+      throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
-      throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
+      throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
     end
 
     def params
@@ -68,7 +68,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
+      [200, {"content-type" => "application/json", "cache-control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)

data/lib/sidekiq/web/application.rb

@@ -62,14 +62,14 @@ module Sidekiq
 
     get "/metrics" do
       q = Sidekiq::Metrics::Query.new
-      @resultset = q.top_jobs
+      @query_result = q.top_jobs
       erb(:metrics)
     end
 
     get "/metrics/:name" do
       @name = route_params[:name]
       q = Sidekiq::Metrics::Query.new
-      @resultset = q.for_job(@name)
+      @query_result = q.for_job(@name)
       erb(:metrics_for_job)
     end
 
@@ -312,7 +312,7 @@ module Sidekiq
 
     def call(env)
       action = self.class.match(env)
-      return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
+      return [404, {"content-type" => "text/plain", "x-cascade" => "pass"}, ["Not Found"]] unless action
 
       app = @klass
       resp = catch(:halt) do
@@ -329,10 +329,10 @@ module Sidekiq
       else
         # rendered content goes here
         headers = {
-          "Content-Type" => "text/html",
-          "Cache-Control" => "private, no-store",
-          "Content-Language" => action.locale,
-          "Content-Security-Policy" => CSP_HEADER
+          "content-type" => "text/html",
+          "cache-control" => "private, no-store",
+          "content-language" => action.locale,
+          "content-security-policy" => CSP_HEADER
         }
         # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]

data/lib/sidekiq/web.rb

@@ -148,7 +148,7 @@ module Sidekiq
       m = middlewares
 
       rules = []
-      rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
+      rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],

data/sidekiq.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
     "source_code_uri" => "https://github.com/mperham/sidekiq"
   }
 
-  gem.add_dependency "redis", ">= 4.5.0"
-  gem.add_dependency "connection_pool", ">= 2.2.2"
+  gem.add_dependency "redis", "<5", ">= 4.5.0"
+  gem.add_dependency "connection_pool", ">= 2.2.5"
   gem.add_dependency "rack", "~> 2.0"
 end