sidekiq 6.3.1 → 6.4.2

data/lib/sidekiq/testing.rb

@@ -101,20 +101,20 @@ module Sidekiq
     ##
     # The Queues class is only for testing the fake queue implementation.
     # There are 2 data structures involved in tandem. This is due to the
-    # Rspec syntax of change(QueueWorker.jobs, :size). It keeps a reference
+    # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
     # to the array. Because the array was dervied from a filter of the total
     # jobs enqueued, it appeared as though the array didn't change.
     #
     # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
-    # on the queue, and another with keys of the worker names, so the array for
-    # QueueWorker.jobs is a straight reference to a real array.
+    # on the queue, and another with keys of the job type, so the array for
+    # HardJob.jobs is a straight reference to a real array.
     #
     # Queue-based hash:
     #
     # {
     #   "default"=>[
     #     {
-    #       "class"=>"TestTesting::QueueWorker",
+    #       "class"=>"TestTesting::HardJob",
     #       "args"=>[1, 2],
     #       "retry"=>true,
     #       "queue"=>"default",
@@ -124,12 +124,12 @@ module Sidekiq
     #   ]
     # }
     #
-    # Worker-based hash:
+    # Job-based hash:
     #
     # {
-    #   "TestTesting::QueueWorker"=>[
+    #   "TestTesting::HardJob"=>[
     #     {
-    #       "class"=>"TestTesting::QueueWorker",
+    #       "class"=>"TestTesting::HardJob",
     #       "args"=>[1, 2],
     #       "retry"=>true,
     #       "queue"=>"default",
@@ -144,14 +144,14 @@ module Sidekiq
     #   require 'sidekiq/testing'
     #
     #   assert_equal 0, Sidekiq::Queues["default"].size
-    #   HardWorker.perform_async(:something)
+    #   HardJob.perform_async(:something)
     #   assert_equal 1, Sidekiq::Queues["default"].size
     #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
     #
-    # You can also clear all workers' jobs:
+    # You can also clear all jobs:
     #
     #   assert_equal 0, Sidekiq::Queues["default"].size
-    #   HardWorker.perform_async(:something)
+    #   HardJob.perform_async(:something)
     #   Sidekiq::Queues.clear_all
     #   assert_equal 0, Sidekiq::Queues["default"].size
     #
@@ -170,35 +170,36 @@ module Sidekiq
 
       def push(queue, klass, job)
         jobs_by_queue[queue] << job
-        jobs_by_worker[klass] << job
+        jobs_by_class[klass] << job
       end
 
       def jobs_by_queue
         @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
       end
 
-      def jobs_by_worker
-        @jobs_by_worker ||= Hash.new { |hash, key| hash[key] = [] }
+      def jobs_by_class
+        @jobs_by_class ||= Hash.new { |hash, key| hash[key] = [] }
       end
+      alias_method :jobs_by_worker, :jobs_by_class
 
       def delete_for(jid, queue, klass)
         jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
-        jobs_by_worker[klass].delete_if { |job| job["jid"] == jid }
+        jobs_by_class[klass].delete_if { |job| job["jid"] == jid }
       end
 
       def clear_for(queue, klass)
         jobs_by_queue[queue].clear
-        jobs_by_worker[klass].clear
+        jobs_by_class[klass].clear
       end
 
       def clear_all
         jobs_by_queue.clear
-        jobs_by_worker.clear
+        jobs_by_class.clear
       end
     end
   end
 
-  module Worker
+  module Job
     ##
     # The Sidekiq testing infrastructure overrides perform_async
     # so that it does not actually touch the network.  Instead it
@@ -212,16 +213,16 @@ module Sidekiq
     #
     #   require 'sidekiq/testing'
     #
-    #   assert_equal 0, HardWorker.jobs.size
-    #   HardWorker.perform_async(:something)
-    #   assert_equal 1, HardWorker.jobs.size
-    #   assert_equal :something, HardWorker.jobs[0]['args'][0]
+    #   assert_equal 0, HardJob.jobs.size
+    #   HardJob.perform_async(:something)
+    #   assert_equal 1, HardJob.jobs.size
+    #   assert_equal :something, HardJob.jobs[0]['args'][0]
     #
     #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
     #   MyMailer.delay.send_welcome_email('foo@example.com')
     #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
     #
-    # You can also clear and drain all workers' jobs:
+    # You can also clear and drain all job types:
     #
     #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
     #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
@@ -241,14 +242,14 @@ module Sidekiq
     #
     #   RSpec.configure do |config|
     #     config.before(:each) do
-    #       Sidekiq::Worker.clear_all
+    #       Sidekiq::Job.clear_all
     #     end
     #   end
     #
     # or for acceptance testing, i.e. with cucumber:
     #
     #   AfterStep do
-    #     Sidekiq::Worker.drain_all
+    #     Sidekiq::Job.drain_all
     #   end
     #
     #   When I sign up as "foo@example.com"
@@ -262,7 +263,7 @@ module Sidekiq
 
       # Jobs queued for this worker
       def jobs
-        Queues.jobs_by_worker[to_s]
+        Queues.jobs_by_class[to_s]
       end
 
       # Clear all jobs for this worker
@@ -288,11 +289,11 @@ module Sidekiq
       end
 
       def process_job(job)
-        worker = new
-        worker.jid = job["jid"]
-        worker.bid = job["bid"] if worker.respond_to?(:bid=)
-        Sidekiq::Testing.server_middleware.invoke(worker, job, job["queue"]) do
-          execute_job(worker, job["args"])
+        inst = new
+        inst.jid = job["jid"]
+        inst.bid = job["bid"] if inst.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
+          execute_job(inst, job["args"])
         end
       end
 
@@ -306,18 +307,18 @@ module Sidekiq
         Queues.jobs_by_queue.values.flatten
       end
 
-      # Clear all queued jobs across all workers
+      # Clear all queued jobs
       def clear_all
         Queues.clear_all
       end
 
-      # Drain all queued jobs across all workers
+      # Drain (execute) all queued jobs
       def drain_all
         while jobs.any?
-          worker_classes = jobs.map { |job| job["class"] }.uniq
+          job_classes = jobs.map { |job| job["class"] }.uniq
 
-          worker_classes.each do |worker_class|
-            Sidekiq::Testing.constantize(worker_class).drain
+          job_classes.each do |job_class|
+            Sidekiq::Testing.constantize(job_class).drain
           end
         end
       end

data/lib/sidekiq/util.rb

@@ -39,6 +39,19 @@ module Sidekiq
   module Util
     include ExceptionHandler
 
+    # hack for quicker development / testing environment #2774
+    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
+
+    # Wait for the orblock to be true or the deadline passed.
+    def wait_for(deadline, &condblock)
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      while remaining > PAUSE_TIME
+        return if condblock.call
+        sleep PAUSE_TIME
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+
     def watchdog(last_words)
       yield
     rescue Exception => ex

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.3.1"
+  VERSION = "6.4.2"
 end

data/lib/sidekiq/web/application.rb

@@ -50,7 +50,10 @@ module Sidekiq
 
     get "/" do
       @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
-      stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
+      days = (params["days"] || 30).to_i
+      return halt(401) if days < 1 || days > 180
+
+      stats_history = Sidekiq::Stats::History.new(days)
       @processed_history = stats_history.processed
       @failed_history = stats_history.failed
 
@@ -299,7 +302,7 @@ module Sidekiq
       return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
 
       app = @klass
-      resp = catch(:halt) do # rubocop:disable Standard/SemanticBlocks
+      resp = catch(:halt) do
         self.class.run_befores(app, action)
         action.instance_exec env, &action.block
       ensure

data/lib/sidekiq/web/csrf_protection.rb

@@ -143,7 +143,7 @@ module Sidekiq
         one_time_pad = SecureRandom.random_bytes(token.length)
         encrypted_token = xor_byte_strings(one_time_pad, token)
         masked_token = one_time_pad + encrypted_token
-        Base64.strict_encode64(masked_token)
+        Base64.urlsafe_encode64(masked_token)
       end
 
       # Essentially the inverse of +mask_token+.
@@ -169,7 +169,7 @@ module Sidekiq
       end
 
       def decode_token(token)
-        Base64.strict_decode64(token)
+        Base64.urlsafe_decode64(token)
       end
 
       def xor_byte_strings(s1, s2)

data/lib/sidekiq/web/helpers.rb

@@ -140,8 +140,8 @@ module Sidekiq
       params[:direction] == "asc" ? "↑" : "↓"
     end
 
-    def workers
-      @workers ||= Sidekiq::Workers.new
+    def workset
+      @work ||= Sidekiq::WorkSet.new
     end
 
     def processes
@@ -175,7 +175,7 @@ module Sidekiq
     end
 
     def current_status
-      workers.size == 0 ? "idle" : "active"
+      workset.size == 0 ? "idle" : "active"
     end
 
     def relative_time(time)
@@ -242,7 +242,7 @@ module Sidekiq
       queue class args retry_count retried_at failed_at
       jid error_message error_class backtrace
       error_backtrace enqueued_at retry wrapped
-      created_at tags
+      created_at tags display_class
     ])
 
     def retry_extra_items(retry_job)

data/lib/sidekiq/web.rb

@@ -148,9 +148,9 @@ module Sidekiq
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
-                          root: ASSETS,
-                          cascade: true,
-                          header_rules: rules
+          root: ASSETS,
+          cascade: true,
+          header_rules: rules
         m.each { |middleware, block| use(*middleware, &block) }
         use Sidekiq::Web::CsrfProtection unless $TESTING
         run WebApplication.new(klass)

data/lib/sidekiq/worker.rb

@@ -82,7 +82,7 @@ module Sidekiq
         end
 
         def get_sidekiq_options # :nodoc:
-          self.sidekiq_options_hash ||= Sidekiq.default_worker_options
+          self.sidekiq_options_hash ||= Sidekiq.default_job_options
         end
 
         def sidekiq_class_attribute(*attrs)
@@ -171,30 +171,78 @@ module Sidekiq
     #     SomeWorker.set(queue: 'foo').perform_async(....)
     #
     class Setter
+      include Sidekiq::JobUtil
+
       def initialize(klass, opts)
         @klass = klass
-        @opts = opts
+        # NB: the internal hash always has stringified keys
+        @opts = opts.transform_keys(&:to_s)
 
         # ActiveJob compatibility
-        interval = @opts.delete(:wait_until) || @opts.delete(:wait)
+        interval = @opts.delete("wait_until") || @opts.delete("wait")
         at(interval) if interval
       end
 
       def set(options)
-        interval = options.delete(:wait_until) || options.delete(:wait)
-        @opts.merge!(options)
+        hash = options.transform_keys(&:to_s)
+        interval = hash.delete("wait_until") || @opts.delete("wait")
+        @opts.merge!(hash)
         at(interval) if interval
         self
       end
 
       def perform_async(*args)
-        @klass.client_push(@opts.merge("args" => args, "class" => @klass))
+        if @opts["sync"] == true
+          perform_inline(*args)
+        else
+          @klass.client_push(@opts.merge("args" => args, "class" => @klass))
+        end
+      end
+
+      # Explicit inline execution of a job. Returns nil if the job did not
+      # execute, true otherwise.
+      def perform_inline(*args)
+        raw = @opts.merge("args" => args, "class" => @klass)
+
+        # validate and normalize payload
+        item = normalize_item(raw)
+        queue = item["queue"]
+
+        # run client-side middleware
+        result = Sidekiq.client_middleware.invoke(item["class"], item, queue, Sidekiq.redis_pool) do
+          item
+        end
+        return nil unless result
+
+        # round-trip the payload via JSON
+        msg = Sidekiq.load_json(Sidekiq.dump_json(item))
+
+        # prepare the job instance
+        klass = msg["class"].constantize
+        job = klass.new
+        job.jid = msg["jid"]
+        job.bid = msg["bid"] if job.respond_to?(:bid)
+
+        # run the job through server-side middleware
+        result = Sidekiq.server_middleware.invoke(job, msg, msg["queue"]) do
+          # perform it
+          job.perform(*msg["args"])
+          true
+        end
+        return nil unless result
+        # jobs do not return a result. they should store any
+        # modified state.
+        true
       end
+      alias_method :perform_sync, :perform_inline
 
       def perform_bulk(args, batch_size: 1_000)
-        args.each_slice(batch_size).flat_map do |slice|
-          Sidekiq::Client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
+        client = @klass.build_client
+        result = args.each_slice(batch_size).flat_map do |slice|
+          client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
         end
+
+        result.is_a?(Enumerator::Lazy) ? result.force : result
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
@@ -238,8 +286,14 @@ module Sidekiq
       end
 
       def perform_async(*args)
-        client_push("class" => self, "args" => args)
+        Setter.new(self, {}).perform_async(*args)
+      end
+
+      # Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
+      def perform_inline(*args)
+        Setter.new(self, {}).perform_inline(*args)
       end
+      alias_method :perform_sync, :perform_inline
 
       ##
       # Push a large number of jobs to Redis, while limiting the batch of
@@ -261,10 +315,8 @@ module Sidekiq
       #
       #     SomeWorker.perform_bulk([[1], [2], [3]])
       #
-      def perform_bulk(items, batch_size: 1_000)
-        items.each_slice(batch_size).flat_map do |slice|
-          Sidekiq::Client.push_bulk("class" => self, "args" => slice)
-        end
+      def perform_bulk(*args, **kwargs)
+        Setter.new(self, {}).perform_bulk(*args, **kwargs)
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
@@ -301,10 +353,13 @@ module Sidekiq
       end
 
       def client_push(item) # :nodoc:
-        pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
-        stringified_item = item.transform_keys(&:to_s)
+        raise ArgumentError, "Job payloads should contain no Symbols: #{item}" if item.any? { |k, v| k.is_a?(::Symbol) }
+        build_client.push(item)
+      end
 
-        Sidekiq::Client.new(pool).push(stringified_item)
+      def build_client # :nodoc:
+        pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
+        Sidekiq::Client.new(pool)
       end
     end
   end

data/lib/sidekiq.rb

@@ -26,6 +26,7 @@ module Sidekiq
     timeout: 25,
     poll_interval_average: nil,
     average_scheduled_poll_interval: 5,
+    on_complex_arguments: :warn,
     error_handlers: [],
     death_handlers: [],
     lifecycle_events: {
@@ -39,11 +40,6 @@ module Sidekiq
     reloader: proc { |&block| block.call }
   }
 
-  DEFAULT_WORKER_OPTIONS = {
-    "retry" => true,
-    "queue" => "default"
-  }
-
   FAKE_INFO = {
     "redis_version" => "9.9.9",
     "uptime_in_days" => "9999",
@@ -102,6 +98,7 @@ module Sidekiq
         # to disconnect and reopen the socket to get back to the primary.
         # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
         # 4985 Use the same logic when a blocking command is force-unblocked
+        # The same retry logic is also used in client.rb
         if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
           conn.disconnect!
           retryable = false
@@ -156,13 +153,20 @@ module Sidekiq
     Middleware::Chain.new
   end
 
-  def self.default_worker_options=(hash)
-    # stringify
-    @default_worker_options = default_worker_options.merge(hash.transform_keys(&:to_s))
+  def self.default_worker_options=(hash) # deprecated
+    @default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
+  end
+
+  def self.default_job_options=(hash)
+    @default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
+  end
+
+  def self.default_worker_options # deprecated
+    @default_job_options ||= {"retry" => true, "queue" => "default"}
   end
 
-  def self.default_worker_options
-    defined?(@default_worker_options) ? @default_worker_options : DEFAULT_WORKER_OPTIONS
+  def self.default_job_options
+    @default_job_options ||= {"retry" => true, "queue" => "default"}
   end
 
   ##
@@ -200,12 +204,12 @@ module Sidekiq
   end
 
   def self.logger
-    @logger ||= Sidekiq::Logger.new($stdout, level: Logger::INFO)
+    @logger ||= Sidekiq::Logger.new($stdout, level: :info)
   end
 
   def self.logger=(logger)
     if logger.nil?
-      self.logger.level = Logger::FATAL
+      self.logger.fatal!
       return self.logger
     end
 
@@ -218,6 +222,10 @@ module Sidekiq
     defined?(Sidekiq::Pro)
   end
 
+  def self.ent?
+    defined?(Sidekiq::Enterprise)
+  end
+
   # How frequently Redis should be checked by a random Sidekiq process for
   # scheduled and retriable jobs. Each individual process will take turns by
   # waiting some multiple of this value.
@@ -252,12 +260,16 @@ module Sidekiq
     options[:lifecycle_events][event] << block
   end
 
-  # We are shutting down Sidekiq but what about workers that
+  def self.strict_args!(mode = :raise)
+    options[:on_complex_arguments] = mode
+  end
+
+  # We are shutting down Sidekiq but what about threads that
   # are working on some long job?  This error is
-  # raised in workers that have not finished within the hard
+  # raised in jobs that have not finished within the hard
   # timeout limit.  This is needed to rollback db transactions,
   # otherwise Ruby's Thread#kill will commit.  See #377.
-  # DO NOT RESCUE THIS ERROR IN YOUR WORKERS
+  # DO NOT RESCUE THIS ERROR IN YOUR JOBS
   class Shutdown < Interrupt; end
 end
 

data/web/assets/javascripts/application.js

@@ -9,7 +9,9 @@ var ready = (callback) => {
   else document.addEventListener("DOMContentLoaded", callback);
 }
 
-ready(() => {
+ready(addListeners)
+
+function addListeners() {
   document.querySelectorAll(".check_all").forEach(node => {
     node.addEventListener("click", event => {
       node.closest('table').querySelectorAll('input[type=checkbox]').forEach(inp => { inp.checked = !!node.checked; });
@@ -26,42 +28,48 @@ ready(() => {
   })
 
   document.querySelectorAll("[data-toggle]").forEach(node => {
-    node.addEventListener("click", event => {
-      var targName = node.getAttribute("data-toggle");
-      var full = document.getElementById(targName + "_full");
-      if (full.style.display == "block") {
-        full.style.display = 'none';
-      } else {
-        full.style.display = 'block';
-      }
-    })
+    node.addEventListener("click", addDataToggleListeners)
   })
 
   updateFuzzyTimes();
+  setLivePollFromUrl();
 
   var buttons = document.querySelectorAll(".live-poll");
   if (buttons.length > 0) {
     buttons.forEach(node => {
-      node.addEventListener("click", event => {
-        if (localStorage.sidekiqLivePoll == "enabled") {
-          localStorage.sidekiqLivePoll = "disabled";
-          clearTimeout(livePollTimer);
-          livePollTimer = null;
-        } else {
-          localStorage.sidekiqLivePoll = "enabled";
-          livePollCallback();
-        }
-
-        updateLivePollButton();
-      })
+      node.addEventListener("click", addPollingListeners)
     });
 
     updateLivePollButton();
-    if (localStorage.sidekiqLivePoll == "enabled") {
+    if (localStorage.sidekiqLivePoll == "enabled" && !livePollTimer) {
       scheduleLivePoll();
     }
   }
-})
+}
+
+function addPollingListeners(_event)  {
+  if (localStorage.sidekiqLivePoll == "enabled") {
+    localStorage.sidekiqLivePoll = "disabled";
+    clearTimeout(livePollTimer);
+    livePollTimer = null;
+  } else {
+    localStorage.sidekiqLivePoll = "enabled";
+    livePollCallback();
+  }
+
+  updateLivePollButton();
+}
+
+function addDataToggleListeners(event) {
+  var source = event.target || event.srcElement;
+  var targName = source.getAttribute("data-toggle");
+  var full = document.getElementById(targName + "_full");
+  if (full.style.display == "block") {
+    full.style.display = 'none';
+  } else {
+    full.style.display = 'block';
+  }
+}
 
 function updateFuzzyTimes() {
   var locale = document.body.getAttribute("data-locale");
@@ -76,6 +84,14 @@ function updateFuzzyTimes() {
   t.cancel();
 }
 
+function setLivePollFromUrl() {
+  var url_params = new URL(window.location.href).searchParams
+
+  if (url_params.get("poll") == "true") {
+    localStorage.sidekiqLivePoll = "enabled";
+  }
+}
+
 function updateLivePollButton() {
   if (localStorage.sidekiqLivePoll == "enabled") {
     document.querySelectorAll('.live-poll-stop').forEach(box => { box.style.display = "inline-block" })
@@ -89,7 +105,19 @@ function updateLivePollButton() {
 function livePollCallback() {
   clearTimeout(livePollTimer);
 
-  fetch(window.location.href).then(resp => resp.text()).then(replacePage).finally(scheduleLivePoll)
+  fetch(window.location.href)
+  .then(checkResponse)
+  .then(resp => resp.text())
+  .then(replacePage)
+  .catch(showError)
+  .finally(scheduleLivePoll)
+}
+
+function checkResponse(resp) {
+  if (!resp.ok) {
+    throw response.error();
+  }
+  return resp
 }
 
 function scheduleLivePoll() {
@@ -107,5 +135,9 @@ function replacePage(text) {
   var header_status = doc.querySelector('.status')
   document.querySelector('.status').replaceWith(header_status)
 
-  updateFuzzyTimes();
+  addListeners();
+}
+
+function showError(error) {
+  console.error(error)
 }