sidekiq 6.0.7 → 6.4.1

data/lib/sidekiq/web.rb

@@ -10,12 +10,11 @@ require "sidekiq/web/helpers"
 require "sidekiq/web/router"
 require "sidekiq/web/action"
 require "sidekiq/web/application"
+require "sidekiq/web/csrf_protection"
 
-require "rack/protection"
-
+require "rack/content_length"
 require "rack/builder"
-require "rack/file"
-require "rack/session/cookie"
+require "rack/static"
 
 module Sidekiq
   class Web
@@ -39,14 +38,6 @@ module Sidekiq
         self
       end
 
-      def middlewares
-        @middlewares ||= []
-      end
-
-      def use(*middleware_args, &block)
-        middlewares << [middleware_args, block]
-      end
-
       def default_tabs
         DEFAULT_TABS
       end
@@ -72,32 +63,45 @@ module Sidekiq
         opts.each { |key| set(key, false) }
       end
 
-      # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
+      def middlewares
+        @middlewares ||= []
+      end
+
+      def use(*args, &block)
+        middlewares << [args, block]
+      end
+
       def set(attribute, value)
         send(:"#{attribute}=", value)
       end
 
-      attr_accessor :app_url, :session_secret, :redis_pool, :sessions
+      def sessions=(val)
+        puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      def session_secret=(val)
+        puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      attr_accessor :app_url, :redis_pool
       attr_writer :locales, :views
     end
 
     def self.inherited(child)
       child.app_url = app_url
-      child.session_secret = session_secret
       child.redis_pool = redis_pool
-      child.sessions = sessions
     end
 
     def settings
       self.class.settings
     end
 
-    def use(*middleware_args, &block)
-      middlewares << [middleware_args, block]
+    def middlewares
+      @middlewares ||= self.class.middlewares
     end
 
-    def middlewares
-      @middlewares ||= Web.middlewares.dup
+    def use(*args, &block)
+      middlewares << [args, block]
     end
 
     def call(env)
@@ -125,18 +129,8 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    # Default values
-    set :sessions, true
-
-    attr_writer :sessions
-
-    def sessions
-      unless instance_variable_defined?("@sessions")
-        @sessions = self.class.sessions
-        @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
-      end
-
-      @sessions
+    def sessions=(val)
+      puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
     end
 
     def self.register(extension)
@@ -145,50 +139,20 @@ module Sidekiq
 
     private
 
-    def using?(middleware)
-      middlewares.any? do |(m, _)|
-        m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
-      end
-    end
-
-    def build_sessions
-      middlewares = self.middlewares
-
-      unless using?(::Rack::Protection) || ENV["RACK_ENV"] == "test"
-        middlewares.unshift [[::Rack::Protection, {use: :authenticity_token}], nil]
-      end
-
-      s = sessions
-      return unless s
-
-      unless using? ::Rack::Session::Cookie
-        unless (secret = Web.session_secret)
-          require "securerandom"
-          secret = SecureRandom.hex(64)
-        end
-
-        options = {secret: secret}
-        options = options.merge(s.to_hash) if s.respond_to? :to_hash
-
-        middlewares.unshift [[::Rack::Session::Cookie, options], nil]
-      end
-    end
-
     def build
-      build_sessions
-
-      middlewares = self.middlewares
       klass = self.class
+      m = middlewares
 
-      ::Rack::Builder.new do
-        %w[stylesheets javascripts images].each do |asset_dir|
-          map "/#{asset_dir}" do
-            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
-          end
-        end
-
-        middlewares.each { |middleware, block| use(*middleware, &block) }
+      rules = []
+      rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
+      ::Rack::Builder.new do
+        use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
+          root: ASSETS,
+          cascade: true,
+          header_rules: rules
+        m.each { |middleware, block| use(*middleware, &block) }
+        use Sidekiq::Web::CsrfProtection unless $TESTING
         run WebApplication.new(klass)
       end
     end

data/lib/sidekiq/worker.rb

@@ -9,6 +9,7 @@ module Sidekiq
   #
   #   class HardWorker
   #     include Sidekiq::Worker
+  #     sidekiq_options queue: 'critical', retry: 5
   #
   #     def perform(*args)
   #       # do some work
@@ -20,6 +21,26 @@ module Sidekiq
   #   HardWorker.perform_async(1, 2, 3)
   #
   # Note that perform_async is a class method, perform is an instance method.
+  #
+  # Sidekiq::Worker also includes several APIs to provide compatibility with
+  # ActiveJob.
+  #
+  #   class SomeWorker
+  #     include Sidekiq::Worker
+  #     queue_as :critical
+  #
+  #     def perform(...)
+  #     end
+  #   end
+  #
+  #   SomeWorker.set(wait_until: 1.hour).perform_async(123)
+  #
+  # Note that arguments passed to the job must still obey Sidekiq's
+  # best practice for simple, JSON-native data types. Sidekiq will not
+  # implement ActiveJob's more complex argument serialization. For
+  # this reason, we don't implement `perform_later` as our call semantics
+  # are very different.
+  #
   module Worker
     ##
     # The Options module is extracted so we can include it in ActiveJob::Base
@@ -150,33 +171,97 @@ module Sidekiq
     #     SomeWorker.set(queue: 'foo').perform_async(....)
     #
     class Setter
+      include Sidekiq::JobUtil
+
       def initialize(klass, opts)
         @klass = klass
         @opts = opts
+
+        # ActiveJob compatibility
+        interval = @opts.delete(:wait_until) || @opts.delete(:wait)
+        at(interval) if interval
       end
 
       def set(options)
+        interval = options.delete(:wait_until) || options.delete(:wait)
         @opts.merge!(options)
+        at(interval) if interval
         self
       end
 
       def perform_async(*args)
-        @klass.client_push(@opts.merge("args" => args, "class" => @klass))
+        if @opts["sync"] == true
+          perform_inline(*args)
+        else
+          @klass.client_push(@opts.merge("args" => args, "class" => @klass))
+        end
+      end
+
+      # Explicit inline execution of a job. Returns nil if the job did not
+      # execute, true otherwise.
+      def perform_inline(*args)
+        raw = @opts.merge("args" => args, "class" => @klass).transform_keys(&:to_s)
+
+        # validate and normalize payload
+        item = normalize_item(raw)
+        queue = item["queue"]
+
+        # run client-side middleware
+        result = Sidekiq.client_middleware.invoke(item["class"], item, queue, Sidekiq.redis_pool) do
+          item
+        end
+        return nil unless result
+
+        # round-trip the payload via JSON
+        msg = Sidekiq.load_json(Sidekiq.dump_json(item))
+
+        # prepare the job instance
+        klass = msg["class"].constantize
+        job = klass.new
+        job.jid = msg["jid"]
+        job.bid = msg["bid"] if job.respond_to?(:bid)
+
+        # run the job through server-side middleware
+        result = Sidekiq.server_middleware.invoke(job, msg, msg["queue"]) do
+          # perform it
+          job.perform(*msg["args"])
+          true
+        end
+        return nil unless result
+        # jobs do not return a result. they should store any
+        # modified state.
+        true
+      end
+      alias_method :perform_sync, :perform_inline
+
+      def perform_bulk(args, batch_size: 1_000)
+        hash = @opts.transform_keys(&:to_s)
+        pool = Thread.current[:sidekiq_via_pool] || @klass.get_sidekiq_options["pool"] || Sidekiq.redis_pool
+        client = Sidekiq::Client.new(pool)
+        result = args.each_slice(batch_size).flat_map do |slice|
+          client.push_bulk(hash.merge("class" => @klass, "args" => slice))
+        end
+
+        result.is_a?(Enumerator::Lazy) ? result.force : result
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
       #   numeric (like an activesupport time interval).
       def perform_in(interval, *args)
+        at(interval).perform_async(*args)
+      end
+      alias_method :perform_at, :perform_in
+
+      private
+
+      def at(interval)
         int = interval.to_f
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
-
-        payload = @opts.merge("class" => @klass, "args" => args)
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        payload["at"] = ts if ts > now
-        @klass.client_push(payload)
+        @opts["at"] = ts if ts > now
+        self
       end
-      alias_method :perform_at, :perform_in
     end
 
     module ClassMethods
@@ -192,12 +277,45 @@ module Sidekiq
         raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
       end
 
+      def queue_as(q)
+        sidekiq_options("queue" => q.to_s)
+      end
+
       def set(options)
         Setter.new(self, options)
       end
 
       def perform_async(*args)
-        client_push("class" => self, "args" => args)
+        Setter.new(self, {}).perform_async(*args)
+      end
+
+      # Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
+      def perform_inline(*args)
+        Setter.new(self, {}).perform_inline(*args)
+      end
+
+      ##
+      # Push a large number of jobs to Redis, while limiting the batch of
+      # each job payload to 1,000. This method helps cut down on the number
+      # of round trips to Redis, which can increase the performance of enqueueing
+      # large numbers of jobs.
+      #
+      # +items+ must be an Array of Arrays.
+      #
+      # For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
+      #
+      # Example (3 Redis round trips):
+      #
+      #     SomeWorker.perform_async(1)
+      #     SomeWorker.perform_async(2)
+      #     SomeWorker.perform_async(3)
+      #
+      # Would instead become (1 Redis round trip):
+      #
+      #     SomeWorker.perform_bulk([[1], [2], [3]])
+      #
+      def perform_bulk(*args, **kwargs)
+        Setter.new(self, {}).perform_bulk(*args, **kwargs)
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
@@ -235,12 +353,9 @@ module Sidekiq
 
       def client_push(item) # :nodoc:
         pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
-        # stringify
-        item.keys.each do |key|
-          item[key.to_s] = item.delete(key)
-        end
+        stringified_item = item.transform_keys(&:to_s)
 
-        Sidekiq::Client.new(pool).push(item)
+        Sidekiq::Client.new(pool).push(stringified_item)
       end
     end
   end

data/lib/sidekiq.rb

@@ -6,6 +6,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
 require "sidekiq/logger"
 require "sidekiq/client"
 require "sidekiq/worker"
+require "sidekiq/job"
 require "sidekiq/redis_connection"
 require "sidekiq/delay"
 
@@ -20,10 +21,12 @@ module Sidekiq
     labels: [],
     concurrency: 10,
     require: ".",
+    strict: true,
     environment: nil,
     timeout: 25,
     poll_interval_average: nil,
     average_scheduled_poll_interval: 5,
+    on_complex_arguments: :warn,
     error_handlers: [],
     death_handlers: [],
     lifecycle_events: {
@@ -95,10 +98,13 @@ module Sidekiq
       retryable = true
       begin
         yield conn
-      rescue Redis::CommandError => ex
+      rescue Redis::BaseError => ex
         # 2550 Failover can cause the server to become a replica, need
         # to disconnect and reopen the socket to get back to the primary.
-        if retryable && ex.message =~ /READONLY/
+        # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+        # 4985 Use the same logic when a blocking command is force-unblocked
+        # The same retry logic is also used in client.rb
+        if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
           conn.disconnect!
           retryable = false
           retry
@@ -196,7 +202,7 @@ module Sidekiq
   end
 
   def self.logger
-    @logger ||= Sidekiq::Logger.new(STDOUT, level: Logger::INFO)
+    @logger ||= Sidekiq::Logger.new($stdout, level: Logger::INFO)
   end
 
   def self.logger=(logger)
@@ -248,6 +254,10 @@ module Sidekiq
     options[:lifecycle_events][event] << block
   end
 
+  def self.strict_args!(mode = :raise)
+    options[:on_complex_arguments] = mode
+  end
+
   # We are shutting down Sidekiq but what about workers that
   # are working on some long job?  This error is
   # raised in workers that have not finished within the hard

data/sidekiq.gemspec

@@ -5,17 +5,24 @@ Gem::Specification.new do |gem|
   gem.email = ["mperham@gmail.com"]
   gem.summary = "Simple, efficient background processing for Ruby"
   gem.description = "Simple, efficient background processing for Ruby."
-  gem.homepage = "http://sidekiq.org"
+  gem.homepage = "https://sidekiq.org"
   gem.license = "LGPL-3.0"
 
   gem.executables = ["sidekiq", "sidekiqmon"]
-  gem.files = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
+  gem.files = ["sidekiq.gemspec", "README.md", "Changes.md", "LICENSE"] + `git ls-files | grep -E '^(bin|lib|web)'`.split("\n")
   gem.name = "sidekiq"
   gem.version = Sidekiq::VERSION
   gem.required_ruby_version = ">= 2.5.0"
 
-  gem.add_dependency "redis", ">= 4.1.0"
+  gem.metadata = {
+    "homepage_uri" => "https://sidekiq.org",
+    "bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
+    "documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
+    "changelog_uri" => "https://github.com/mperham/sidekiq/blob/main/Changes.md",
+    "source_code_uri" => "https://github.com/mperham/sidekiq"
+  }
+
+  gem.add_dependency "redis", ">= 4.2.0"
   gem.add_dependency "connection_pool", ">= 2.2.2"
   gem.add_dependency "rack", "~> 2.0"
-  gem.add_dependency "rack-protection", ">= 2.0.0"
 end