sidekiq 6.2.2 → 6.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b0da541124d097e05936cc860a93009ba170d714cbd8dbc760c1245818e8ed7
-  data.tar.gz: 63995ccbb57fa16e4954cda4ab9610506b0a8ff35084492b520fc18608face35
+  metadata.gz: b72b92d3d281aaea2b4781cb7743d95a8c7f42708526383bf85fbed420b4b82c
+  data.tar.gz: 9546107658b47c8f68497730e2ee6689881c1388ba96db3f2415b0fda7b7f668
 SHA512:
-  metadata.gz: f3b864db74530840437c0af43475ed0e12a876d6d2cbbd808ecc72722adddbd8d5548c9b777d4e34af5deddf94871567717518f70d787ea7f6a089f2f5b2f4ed
-  data.tar.gz: 4569eed5baa10134e99a0b1404c213e3b35c08ec158e2fa8b006918298750919e8d34074aa10d3b4dfb621bb18cbe68be467d1e7822fb855012cff0f64dcdc7b
+  metadata.gz: 78f0af1f4ed41c5b4ad01636d9808d409d1cedc0bc6146be278412e87f9d826fad2cec9c89b31acbc6da9bab59bff44e826418991b10ce27b34631bcc3a00cf2
+  data.tar.gz: a586b9ec613cf3da47ecb77eb4e0f5d6e497f234369d958cdfa3d8c8c7418806383e022a9b08db4c3b5d08a1c2aac3aaecdf851744234f4c42055e1a7a6ebf6c

data/Changes.md

@@ -1,6 +1,42 @@
 # Sidekiq Changes
 
-[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
+[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
+
+HEAD
+---------
+
+- **BREAK**: The Web UI has been refactored to remove jQuery. Any UI extensions
+  which use jQuery will break.
+- **FEATURE**: Sidekiq.logger has been enhanced so any `Rails.logger`
+  output in jobs now shows up in the Sidekiq console. Remove any logger
+  hacks in your initializer and see if it Just Works™ now. [#5021]
+- **FEATURE**: Add `Sidekiq::Job` alias for `Sidekiq::Worker`, to better
+  reflect industry standard terminology. You can now do this:
+```ruby
+class MyJob
+  include Sidekiq::Job
+  sidekiq_options ...
+  def perform(args)
+  end
+end
+```
+- **FEATURE**: Support for serializing ActiveSupport::CurrentAttributes into each job. [#4982]
+```ruby
+# config/initializers/sidekiq.rb
+require "sidekiq/middleware/current_attributes"
+Sidekiq::CurrentAttributes.persist(Myapp::Current) # Your AS::CurrentAttributes singleton
+```
+- **FEATURE**: Add `Sidekiq::Worker.perform_bulk` for enqueuing jobs in bulk,
+  similar to `Sidekiq::Client.push_bulk` [#5042]
+```ruby
+MyJob.perform_bulk([[1], [2], [3]])
+```
+- Implement `queue_as`, `wait` and `wait_until` for ActiveJob compatibility [#5003]
+- Scheduler now uses Lua to reduce Redis load and network roundtrips [#5044]
+- Retry Redis operation if we get an `UNBLOCKED` Redis error [#4985]
+- Run existing signal traps, if any, before running Sidekiq's trap [#4991]
+- Fix fetch bug when using weighted queues which caused Sidekiq to stop
+  processing queues randomly [#5031]
 
 6.2.2
 ---------

data/README.md

@@ -46,7 +46,7 @@ See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Gett
 You can watch [this Youtube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
 Sidekiq and see its features in action.  Here's the Web UI:
 
-![Web UI](https://github.com/mperham/sidekiq/raw/master/examples/web-ui.png)
+![Web UI](https://github.com/mperham/sidekiq/raw/main/examples/web-ui.png)
 
 
 Want to Upgrade?
@@ -84,7 +84,7 @@ See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
 License
 -----------------
 
-Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
+Please see [LICENSE](https://github.com/mperham/sidekiq/blob/main/LICENSE) for licensing details.
 
 
 Author

data/lib/sidekiq/api.rb

@@ -113,6 +113,7 @@ module Sidekiq
 
       @stats[:workers_size] = workers_size
       @stats[:enqueued] = enqueued
+      @stats
     end
 
     def fetch_stats!

data/lib/sidekiq/cli.rb

@@ -46,7 +46,15 @@ module Sidekiq
       # USR1 and USR2 don't work on the JVM
       sigs << "USR2" if Sidekiq.pro? && !jruby?
       sigs.each do |sig|
-        trap sig do
+        old_handler = Signal.trap(sig) do
+          if old_handler.respond_to?(:call)
+            begin
+              old_handler.call
+            rescue Exception => exc
+              # signal handlers can't use Logger so puts only
+              puts ["Error in #{sig} handler", exc].inspect
+            end
+          end
           self_write.puts(sig)
         end
       rescue ArgumentError
@@ -396,7 +404,7 @@ module Sidekiq
       opts[:queues] ||= []
       opts[:strict] = true if opts[:strict].nil?
       raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
-      [weight.to_i, 1].max.times { opts[:queues] << queue }
+      [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
       opts[:strict] = false if weight.to_i > 0
     end
 

data/lib/sidekiq/client.rb

@@ -95,7 +95,7 @@ module Sidekiq
       return [] if args.empty? # no jobs to push
 
       at = items.delete("at")
-      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
+      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       normed = normalize_item(items)
@@ -186,7 +186,7 @@ module Sidekiq
 
     def raw_push(payloads)
       @redis_pool.with do |conn|
-        conn.multi do
+        conn.pipelined do
           atomic_push(conn, payloads)
         end
       end

data/lib/sidekiq/fetch.rb

@@ -79,9 +79,10 @@ module Sidekiq
       if @strictly_ordered_queues
         @queues
       else
-        queues = @queues.shuffle!.uniq
-        queues << TIMEOUT
-        queues
+        permute = @queues.shuffle
+        permute.uniq!
+        permute << TIMEOUT
+        permute
       end
     end
   end

data/lib/sidekiq/job.rb

@@ -1,8 +1,13 @@
 require "sidekiq/worker"
 
 module Sidekiq
-  # Sidekiq::Job is a new alias for Sidekiq::Worker, coming in 6.3.0.
-  # You can opt into this by requiring 'sidekiq/job' in your initializer
-  # and then using `include Sidekiq::Job` rather than `Sidekiq::Worker`.
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
   Job = Worker
 end

data/lib/sidekiq/launcher.rb

@@ -69,10 +69,12 @@ module Sidekiq
 
     private unless $TESTING
 
+    BEAT_PAUSE = 5
+
     def start_heartbeat
       loop do
         heartbeat
-        sleep 5
+        sleep BEAT_PAUSE
       end
       Sidekiq.logger.info("Heartbeat stopping...")
     end
@@ -211,6 +213,8 @@ module Sidekiq
           Your Redis network connection is performing extremely poorly.
           Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
           Ensure Redis is running in the same AZ or datacenter as Sidekiq.
+          If these values are close to 100,000, that means your Sidekiq process may be
+          CPU overloaded; see https://github.com/mperham/sidekiq/discussions/5039
         EOM
         RTT_READINGS.reset
       end

data/lib/sidekiq/middleware/current_attributes.rb

@@ -0,0 +1,48 @@
+require "active_support/current_attributes"
+
+module Sidekiq
+  ##
+  # Automatically save and load any current attributes in the execution context
+  # so context attributes "flow" from Rails actions into any associated jobs.
+  # This can be useful for multi-tenancy, i18n locale, timezone, any implicit
+  # per-request attribute. See +ActiveSupport::CurrentAttributes+.
+  #
+  # @example
+  #
+  #   # in your initializer
+  #   require "sidekiq/middleware/current_attributes"
+  #   Sidekiq::CurrentAttributes.persist(Myapp::Current)
+  #
+  module CurrentAttributes
+    class Save
+      def initialize(with:)
+        @klass = with
+      end
+
+      def call(_, job, _, _)
+        job["ctx"] = @klass.attributes
+        yield
+      end
+    end
+
+    class Load
+      def initialize(with:)
+        @klass = with
+      end
+
+      def call(_, job, _, &block)
+        @klass.set(job["ctx"], &block)
+      end
+    end
+
+    def self.persist(klass)
+      Sidekiq.configure_client do |config|
+        config.client_middleware.add Save, with: klass
+      end
+      Sidekiq.configure_server do |config|
+        config.client_middleware.add Save, with: klass
+        config.server_middleware.add Load, with: klass
+      end
+    end
+  end
+end

data/lib/sidekiq/rails.rb

@@ -37,6 +37,17 @@ module Sidekiq
       end
     end
 
+    initializer "sidekiq.rails_logger" do
+      Sidekiq.configure_server do |_|
+        # This is the integration code necessary so that if code uses `Rails.logger.info "Hello"`,
+        # it will appear in the Sidekiq console with all of the job context. See #5021 and
+        # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
+        unless ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
+        end
+      end
+    end
+
     # This hook happens after all initializers are run, just before returning
     # from config/environment.rb back to sidekiq/cli.rb.
     #

data/lib/sidekiq/redis_connection.rb

@@ -94,12 +94,10 @@ module Sidekiq
       def log_info(options)
         redacted = "REDACTED"
 
-        # deep clone so we can muck with these options all we want
-        #
-        # exclude SSL params from dump-and-load because some information isn't
-        # safely dumpable in current Rubies
-        keys = options.keys
-        keys.delete(:ssl_params)
+        # Deep clone so we can muck with these options all we want and exclude
+        # params from dump-and-load that may contain objects that Marshal is
+        # unable to safely dump.
+        keys = options.keys - [:logger, :ssl_params]
         scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted

data/lib/sidekiq/scheduled.rb

@@ -9,29 +9,48 @@ module Sidekiq
     SETS = %w[retry schedule]
 
     class Enq
+      LUA_ZPOPBYSCORE = <<~LUA
+        local key, now = KEYS[1], ARGV[1]
+        local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+        if jobs[1] then
+          redis.call("zrem", key, jobs[1])
+          return jobs[1]
+        end
+      LUA
+
+      def initialize
+        @lua_zpopbyscore_sha = nil
+      end
+
       def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = SETS)
         # A job's "score" in Redis is the time at which it should be processed.
         # Just check Redis for the set of jobs with a timestamp before now.
         Sidekiq.redis do |conn|
           sorted_sets.each do |sorted_set|
-            # Get next items in the queue with scores (time to execute) <= now.
-            until (jobs = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 100])).empty?
-              # We need to go through the list one at a time to reduce the risk of something
-              # going wrong between the time jobs are popped from the scheduled queue and when
-              # they are pushed onto a work queue and losing the jobs.
-              jobs.each do |job|
-                # Pop item off the queue and add it to the work queue. If the job can't be popped from
-                # the queue, it's because another process already popped it so we can move on to the
-                # next one.
-                if conn.zrem(sorted_set, job)
-                  Sidekiq::Client.push(Sidekiq.load_json(job))
-                  Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
-                end
-              end
+            # Get next item in the queue with score (time to execute) <= now.
+            # We need to go through the list one at a time to reduce the risk of something
+            # going wrong between the time jobs are popped from the scheduled queue and when
+            # they are pushed onto a work queue and losing the jobs.
+            while (job = zpopbyscore(conn, keys: [sorted_set], argv: [now]))
+              Sidekiq::Client.push(Sidekiq.load_json(job))
+              Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
             end
           end
         end
       end
+
+      private
+
+      def zpopbyscore(conn, keys: nil, argv: nil)
+        @lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE) if @lua_zpopbyscore_sha.nil?
+
+        conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?("NOSCRIPT")
+
+        @lua_zpopbyscore_sha = nil
+        retry
+      end
     end
 
     ##

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.2.2"
+  VERSION = "6.3.0"
 end

data/lib/sidekiq/web/application.rb

@@ -91,8 +91,8 @@ module Sidekiq
 
       @count = (params["count"] || 25).to_i
       @queue = Sidekiq::Queue.new(@name)
-      (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
-      @messages = @messages.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
+      (@current_page, @total_size, @jobs) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
+      @jobs = @jobs.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
 
       erb(:queue)
     end

data/lib/sidekiq/web/helpers.rb

@@ -70,17 +70,6 @@ module Sidekiq
       @head_html.join if defined?(@head_html)
     end
 
-    def poll_path
-      if current_path != "" && params["poll"]
-        path = root_path + current_path
-        query_string = to_query_string(params.slice(*params.keys - %w[page poll]))
-        path += "?#{query_string}" unless query_string.empty?
-        path
-      else
-        ""
-      end
-    end
-
     def text_direction
       get_locale["TextDirection"] || "ltr"
     end
@@ -203,7 +192,7 @@ module Sidekiq
       [score.to_f, jid]
     end
 
-    SAFE_QPARAMS = %w[page poll direction]
+    SAFE_QPARAMS = %w[page direction]
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)

data/lib/sidekiq/worker.rb

@@ -9,6 +9,7 @@ module Sidekiq
   #
   #   class HardWorker
   #     include Sidekiq::Worker
+  #     sidekiq_options queue: 'critical', retry: 5
   #
   #     def perform(*args)
   #       # do some work
@@ -20,6 +21,26 @@ module Sidekiq
   #   HardWorker.perform_async(1, 2, 3)
   #
   # Note that perform_async is a class method, perform is an instance method.
+  #
+  # Sidekiq::Worker also includes several APIs to provide compatibility with
+  # ActiveJob.
+  #
+  #   class SomeWorker
+  #     include Sidekiq::Worker
+  #     queue_as :critical
+  #
+  #     def perform(...)
+  #     end
+  #   end
+  #
+  #   SomeWorker.set(wait_until: 1.hour).perform_async(123)
+  #
+  # Note that arguments passed to the job must still obey Sidekiq's
+  # best practice for simple, JSON-native data types. Sidekiq will not
+  # implement ActiveJob's more complex argument serialization. For
+  # this reason, we don't implement `perform_later` as our call semantics
+  # are very different.
+  #
   module Worker
     ##
     # The Options module is extracted so we can include it in ActiveJob::Base
@@ -153,10 +174,16 @@ module Sidekiq
       def initialize(klass, opts)
         @klass = klass
         @opts = opts
+
+        # ActiveJob compatibility
+        interval = @opts.delete(:wait_until) || @opts.delete(:wait)
+        at(interval) if interval
       end
 
       def set(options)
+        interval = options.delete(:wait_until) || options.delete(:wait)
         @opts.merge!(options)
+        at(interval) if interval
         self
       end
 
@@ -164,19 +191,29 @@ module Sidekiq
         @klass.client_push(@opts.merge("args" => args, "class" => @klass))
       end
 
+      def perform_bulk(args, batch_size: 1_000)
+        args.each_slice(batch_size).flat_map do |slice|
+          Sidekiq::Client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
+        end
+      end
+
       # +interval+ must be a timestamp, numeric or something that acts
       #   numeric (like an activesupport time interval).
       def perform_in(interval, *args)
+        at(interval).perform_async(*args)
+      end
+      alias_method :perform_at, :perform_in
+
+      private
+
+      def at(interval)
         int = interval.to_f
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
-
-        payload = @opts.merge("class" => @klass, "args" => args)
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        payload["at"] = ts if ts > now
-        @klass.client_push(payload)
+        @opts["at"] = ts if ts > now
+        self
       end
-      alias_method :perform_at, :perform_in
     end
 
     module ClassMethods
@@ -192,6 +229,10 @@ module Sidekiq
         raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
       end
 
+      def queue_as(q)
+        sidekiq_options("queue" => q.to_s)
+      end
+
       def set(options)
         Setter.new(self, options)
       end
@@ -200,6 +241,32 @@ module Sidekiq
         client_push("class" => self, "args" => args)
       end
 
+      ##
+      # Push a large number of jobs to Redis, while limiting the batch of
+      # each job payload to 1,000. This method helps cut down on the number
+      # of round trips to Redis, which can increase the performance of enqueueing
+      # large numbers of jobs.
+      #
+      # +items+ must be an Array of Arrays.
+      #
+      # For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
+      #
+      # Example (3 Redis round trips):
+      #
+      #     SomeWorker.perform_async(1)
+      #     SomeWorker.perform_async(2)
+      #     SomeWorker.perform_async(3)
+      #
+      # Would instead become (1 Redis round trip):
+      #
+      #     SomeWorker.perform_bulk([[1], [2], [3]])
+      #
+      def perform_bulk(items, batch_size: 1_000)
+        items.each_slice(batch_size).flat_map do |slice|
+          Sidekiq::Client.push_bulk("class" => self, "args" => slice)
+        end
+      end
+
       # +interval+ must be a timestamp, numeric or something that acts
       #   numeric (like an activesupport time interval).
       def perform_in(interval, *args)

data/lib/sidekiq.rb

@@ -6,6 +6,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
 require "sidekiq/logger"
 require "sidekiq/client"
 require "sidekiq/worker"
+require "sidekiq/job"
 require "sidekiq/redis_connection"
 require "sidekiq/delay"
 
@@ -100,7 +101,8 @@ module Sidekiq
         # 2550 Failover can cause the server to become a replica, need
         # to disconnect and reopen the socket to get back to the primary.
         # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
-        if retryable && ex.message =~ /READONLY|NOREPLICAS/
+        # 4985 Use the same logic when a blocking command is force-unblocked
+        if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
           conn.disconnect!
           retryable = false
           retry

data/sidekiq.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
     "homepage_uri" => "https://sidekiq.org",
     "bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
     "documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
-    "changelog_uri" => "https://github.com/mperham/sidekiq/blob/master/Changes.md",
+    "changelog_uri" => "https://github.com/mperham/sidekiq/blob/main/Changes.md",
     "source_code_uri" => "https://github.com/mperham/sidekiq"
   }