sidekiq 4.2.10 → 7.3.2

data/lib/sidekiq/web.rb

@@ -1,51 +1,59 @@
 # frozen_string_literal: true
-require 'erb'
 
-require 'sidekiq'
-require 'sidekiq/api'
-require 'sidekiq/paginator'
-require 'sidekiq/web/helpers'
+require "erb"
+require "securerandom"
 
-require 'sidekiq/web/router'
-require 'sidekiq/web/action'
-require 'sidekiq/web/application'
+require "sidekiq"
+require "sidekiq/api"
+require "sidekiq/paginator"
+require "sidekiq/web/helpers"
 
-require 'rack/protection'
+require "sidekiq/web/router"
+require "sidekiq/web/action"
+require "sidekiq/web/application"
+require "sidekiq/web/csrf_protection"
 
-require 'rack/builder'
-require 'rack/file'
-require 'rack/session/cookie'
+require "rack/content_length"
+require "rack/builder"
+require "rack/static"
 
 module Sidekiq
   class Web
     ROOT = File.expand_path("#{File.dirname(__FILE__)}/../../web")
-    VIEWS = "#{ROOT}/views".freeze
-    LOCALES = ["#{ROOT}/locales".freeze]
-    LAYOUT = "#{VIEWS}/layout.erb".freeze
-    ASSETS = "#{ROOT}/assets".freeze
+    VIEWS = "#{ROOT}/views"
+    LOCALES = ["#{ROOT}/locales"]
+    LAYOUT = "#{VIEWS}/layout.erb"
+    ASSETS = "#{ROOT}/assets"
 
     DEFAULT_TABS = {
-      "Dashboard" => '',
-      "Busy"      => 'busy',
-      "Queues"    => 'queues',
-      "Retries"   => 'retries',
-      "Scheduled" => 'scheduled',
-      "Dead"      => 'morgue',
+      "Dashboard" => "",
+      "Busy" => "busy",
+      "Queues" => "queues",
+      "Retries" => "retries",
+      "Scheduled" => "scheduled",
+      "Dead" => "morgue",
+      "Metrics" => "metrics"
     }
 
+    if Gem::Version.new(Rack::RELEASE) < Gem::Version.new("3")
+      CONTENT_LANGUAGE = "Content-Language"
+      CONTENT_SECURITY_POLICY = "Content-Security-Policy"
+      LOCATION = "Location"
+      X_CASCADE = "X-Cascade"
+      X_CONTENT_TYPE_OPTIONS = "X-Content-Type-Options"
+    else
+      CONTENT_LANGUAGE = "content-language"
+      CONTENT_SECURITY_POLICY = "content-security-policy"
+      LOCATION = "location"
+      X_CASCADE = "x-cascade"
+      X_CONTENT_TYPE_OPTIONS = "x-content-type-options"
+    end
+
     class << self
       def settings
         self
       end
 
-      def middlewares
-        @middlewares ||= []
-      end
-
-      def use(*middleware_args, &block)
-        middlewares << [middleware_args, block]
-      end
-
       def default_tabs
         DEFAULT_TABS
       end
@@ -55,6 +63,10 @@ module Sidekiq
       end
       alias_method :tabs, :custom_tabs
 
+      def custom_job_info_rows
+        @custom_job_info_rows ||= []
+      end
+
       def locales
         @locales ||= LOCALES
       end
@@ -64,42 +76,48 @@ module Sidekiq
       end
 
       def enable(*opts)
-        opts.each {|key| set(key, true) }
+        opts.each { |key| set(key, true) }
       end
 
       def disable(*opts)
-        opts.each {|key| set(key, false) }
+        opts.each { |key| set(key, false) }
+      end
+
+      def middlewares
+        @middlewares ||= []
+      end
+
+      def use(*args, &block)
+        middlewares << [args, block]
       end
 
-      # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
       def set(attribute, value)
         send(:"#{attribute}=", value)
       end
 
-      attr_accessor :app_url, :session_secret, :redis_pool, :sessions
+      attr_accessor :app_url, :redis_pool
       attr_writer :locales, :views
     end
 
     def self.inherited(child)
-      child.app_url = self.app_url
-      child.session_secret = self.session_secret
-      child.redis_pool = self.redis_pool
-      child.sessions = self.sessions
+      child.app_url = app_url
+      child.redis_pool = redis_pool
     end
 
     def settings
       self.class.settings
     end
 
-    def use(*middleware_args, &block)
-      middlewares << [middleware_args, block]
+    def middlewares
+      @middlewares ||= self.class.middlewares
     end
 
-    def middlewares
-      @middlewares ||= Web.middlewares.dup
+    def use(*args, &block)
+      middlewares << [args, block]
     end
 
     def call(env)
+      env[:csp_nonce] = SecureRandom.base64(16)
       app.call(env)
     end
 
@@ -113,81 +131,80 @@ module Sidekiq
     end
 
     def enable(*opts)
-      opts.each {|key| set(key, true) }
+      opts.each { |key| set(key, true) }
     end
 
     def disable(*opts)
-      opts.each {|key| set(key, false) }
+      opts.each { |key| set(key, false) }
     end
 
     def set(attribute, value)
       send(:"#{attribute}=", value)
     end
 
-    # Default values
-    set :sessions, true
-
-    attr_writer :sessions
-
-    def sessions
-      unless instance_variable_defined?("@sessions")
-        @sessions = self.class.sessions
-        @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
+    # Register a class as a Sidekiq Web UI extension. The class should
+    # provide one or more tabs which map to an index route. Options:
+    #
+    # @param extension [Class] Class which contains the HTTP actions, required
+    # @param name [String] the name of the extension, used to namespace assets
+    # @param tab [String | Array] labels(s) of the UI tabs
+    # @param index [String | Array] index route(s) for each tab
+    # @param root_dir [String] directory location to find assets, locales and views, typically `web/` within the gemfile
+    # @param asset_paths [Array] one or more directories under {root}/assets/{name} to be publicly served, e.g. ["js", "css", "img"]
+    # @param cache_for [Integer] amount of time to cache assets, default one day
+    #
+    # TODO name, tab and index will be mandatory in 8.0
+    #
+    # Web extensions will have a root `web/` directory with `locales/`, `assets/`
+    # and `views/` subdirectories.
+    def self.register(extension, name: nil, tab: nil, index: nil, root_dir: nil, cache_for: 86400, asset_paths: nil)
+      tab = Array(tab)
+      index = Array(index)
+      tab.zip(index).each do |tab, index|
+        tabs[tab] = index
+      end
+      if root_dir
+        locdir = File.join(root_dir, "locales")
+        locales << locdir if File.directory?(locdir)
+
+        if asset_paths && name
+          # if you have {root}/assets/{name}/js/scripts.js
+          # and {root}/assets/{name}/css/styles.css
+          # you would pass in:
+          #   asset_paths: ["js", "css"]
+          # See script_tag and style_tag in web/helpers.rb
+          assdir = File.join(root_dir, "assets")
+          assurls = Array(asset_paths).map { |x| "/#{name}/#{x}" }
+          assetprops = {
+            urls: assurls,
+            root: assdir,
+            cascade: true
+          }
+          assetprops[:header_rules] = [[:all, {Rack::CACHE_CONTROL => "private, max-age=#{cache_for.to_i}"}]] if cache_for
+          middlewares << [[Rack::Static, assetprops], nil]
+        end
       end
 
-      @sessions
-    end
-
-    def self.register(extension)
+      yield self if block_given?
       extension.registered(WebApplication)
     end
 
     private
 
-    def using?(middleware)
-      middlewares.any? do |(m,_)|
-        m.kind_of?(Array) && (m[0] == middleware || m[0].kind_of?(middleware))
-      end
-    end
-
-    def build_sessions
-      middlewares = self.middlewares
-
-      unless using?(::Rack::Protection) || ENV['RACK_ENV'] == 'test'
-        middlewares.unshift [[::Rack::Protection, { use: :authenticity_token }], nil]
-      end
-
-      s = sessions
-      return unless s
-
-      unless using? ::Rack::Session::Cookie
-        unless secret = Web.session_secret
-          require 'securerandom'
-          secret = SecureRandom.hex(64)
-        end
-
-        options = { secret: secret }
-        options = options.merge(s.to_hash) if s.respond_to? :to_hash
-
-        middlewares.unshift [[::Rack::Session::Cookie, options], nil]
-      end
-    end
-
     def build
-      build_sessions
-
-      middlewares = self.middlewares
       klass = self.class
+      m = middlewares
 
-      ::Rack::Builder.new do
-        %w(stylesheets javascripts images).each do |asset_dir|
-          map "/#{asset_dir}" do
-            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", { 'Cache-Control' => 'public, max-age=86400' })
-          end
-        end
-
-        middlewares.each {|middleware, block| use(*middleware, &block) }
+      rules = []
+      rules = [[:all, {Rack::CACHE_CONTROL => "private, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
+      ::Rack::Builder.new do
+        use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
+          root: ASSETS,
+          cascade: true,
+          header_rules: rules
+        m.each { |middleware, block| use(*middleware, &block) }
+        use Sidekiq::Web::CsrfProtection unless $TESTING
         run WebApplication.new(klass)
       end
     end
@@ -196,18 +213,9 @@ module Sidekiq
   Sidekiq::WebApplication.helpers WebHelpers
   Sidekiq::WebApplication.helpers Sidekiq::Paginator
 
-  Sidekiq::WebAction.class_eval "def _render\n#{ERB.new(File.read(Web::LAYOUT)).src}\nend"
-end
-
-if defined?(::ActionDispatch::Request::Session) &&
-    !::ActionDispatch::Request::Session.method_defined?(:each)
-  # mperham/sidekiq#2460
-  # Rack apps can't reuse the Rails session store without
-  # this monkeypatch, fixed in Rails 5.
-  class ActionDispatch::Request::Session
-    def each(&block)
-      hash = self.to_hash
-      hash.each(&block)
+  Sidekiq::WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+    def _render
+      #{ERB.new(File.read(Web::LAYOUT)).src}
     end
-  end
+  RUBY
 end

data/lib/sidekiq/worker_compatibility_alias.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
+  Worker = Job
+end

data/lib/sidekiq.rb

@@ -1,237 +1,148 @@
-# encoding: utf-8
 # frozen_string_literal: true
-require 'sidekiq/version'
-fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.0.0." if RUBY_PLATFORM != 'java' && RUBY_VERSION < '2.0.0'
 
-require 'sidekiq/logging'
-require 'sidekiq/client'
-require 'sidekiq/worker'
-require 'sidekiq/redis_connection'
+require "sidekiq/version"
+fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.7.0." if RUBY_PLATFORM != "java" && Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.7.0")
 
-require 'json'
+begin
+  require "sidekiq-ent/version"
+  fail <<~EOM  if Gem::Version.new(Sidekiq::Enterprise::VERSION).segments[0] != Sidekiq::MAJOR
 
-module Sidekiq
-  NAME = 'Sidekiq'
-  LICENSE = 'See LICENSE and the LGPL-3.0 for licensing details.'
-
-  DEFAULTS = {
-    queues: [],
-    labels: [],
-    concurrency: 25,
-    require: '.',
-    environment: nil,
-    timeout: 8,
-    poll_interval_average: nil,
-    average_scheduled_poll_interval: 15,
-    error_handlers: [],
-    lifecycle_events: {
-      startup: [],
-      quiet: [],
-      shutdown: [],
-      heartbeat: [],
-    },
-    dead_max_jobs: 10_000,
-    dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
-    reloader: proc { |&block| block.call },
-    executor: proc { |&block| block.call },
-  }
-
-  DEFAULT_WORKER_OPTIONS = {
-    'retry' => true,
-    'queue' => 'default'
-  }
-
-  FAKE_INFO = {
-    "redis_version" => "9.9.9",
-    "uptime_in_days" => "9999",
-    "connected_clients" => "9999",
-    "used_memory_human" => "9P",
-    "used_memory_peak_human" => "9P"
-  }.freeze
+    Sidekiq Enterprise #{Sidekiq::Enterprise::VERSION} does not work with Sidekiq #{Sidekiq::VERSION}.
+    Starting with Sidekiq 7, major versions are synchronized so Sidekiq Enterprise 7 works with Sidekiq 7.
+    Use `bundle up sidekiq-ent` to upgrade.
 
-  def self.❨╯°□°❩╯︵┻━┻
-    puts "Calm down, yo."
-  end
+  EOM
+rescue LoadError
+end
 
-  def self.options
-    @options ||= DEFAULTS.dup
-  end
-  def self.options=(opts)
-    @options = opts
-  end
+begin
+  require "sidekiq/pro/version"
+  fail <<~EOM  if Gem::Version.new(Sidekiq::Pro::VERSION).segments[0] != Sidekiq::MAJOR
 
-  ##
-  # Configuration for Sidekiq server, use like:
-  #
-  #   Sidekiq.configure_server do |config|
-  #     config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/0' }
-  #     config.server_middleware do |chain|
-  #       chain.add MyServerHook
-  #     end
-  #   end
-  def self.configure_server
-    yield self if server?
-  end
+    Sidekiq Pro #{Sidekiq::Pro::VERSION} does not work with Sidekiq #{Sidekiq::VERSION}.
+    Starting with Sidekiq 7, major versions are synchronized so Sidekiq Pro 7 works with Sidekiq 7.
+    Use `bundle up sidekiq-pro` to upgrade.
 
-  ##
-  # Configuration for Sidekiq client, use like:
-  #
-  #   Sidekiq.configure_client do |config|
-  #     config.redis = { :namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/0' }
-  #   end
-  def self.configure_client
-    yield self unless server?
+  EOM
+rescue LoadError
+end
+
+require "sidekiq/config"
+require "sidekiq/logger"
+require "sidekiq/client"
+require "sidekiq/transaction_aware_client"
+require "sidekiq/job"
+require "sidekiq/iterable_job"
+require "sidekiq/worker_compatibility_alias"
+require "sidekiq/redis_client_adapter"
+
+require "json"
+
+module Sidekiq
+  NAME = "Sidekiq"
+  LICENSE = "See LICENSE and the LGPL-3.0 for licensing details."
+
+  def self.❨╯°□°❩╯︵┻━┻
+    puts "Take a deep breath and count to ten..."
   end
 
   def self.server?
     defined?(Sidekiq::CLI)
   end
 
-  def self.redis
-    raise ArgumentError, "requires a block" unless block_given?
-    redis_pool.with do |conn|
-      retryable = true
-      begin
-        yield conn
-      rescue Redis::CommandError => ex
-        #2550 Failover can cause the server to become a slave, need
-        # to disconnect and reopen the socket to get back to the master.
-        (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
-        raise
-      end
-    end
-  end
-
-  def self.redis_info
-    redis do |conn|
-      begin
-        # admin commands can't go through redis-namespace starting
-        # in redis-namespace 2.0
-        if conn.respond_to?(:namespace)
-          conn.redis.info
-        else
-          conn.info
-        end
-      rescue Redis::CommandError => ex
-        #2850 return fake version when INFO command has (probably) been renamed
-        raise unless ex.message =~ /unknown command/
-        FAKE_INFO
-      end
-    end
+  def self.load_json(string)
+    JSON.parse(string)
   end
 
-  def self.redis_pool
-    @redis ||= Sidekiq::RedisConnection.create
+  def self.dump_json(object)
+    JSON.generate(object)
   end
 
-  def self.redis=(hash)
-    @redis = if hash.is_a?(ConnectionPool)
-      hash
-    else
-      Sidekiq::RedisConnection.create(hash)
-    end
+  def self.pro?
+    defined?(Sidekiq::Pro)
   end
 
-  def self.client_middleware
-    @client_chain ||= Middleware::Chain.new
-    yield @client_chain if block_given?
-    @client_chain
+  def self.ent?
+    defined?(Sidekiq::Enterprise)
   end
 
-  def self.server_middleware
-    @server_chain ||= default_server_middleware
-    yield @server_chain if block_given?
-    @server_chain
+  def self.redis_pool
+    (Thread.current[:sidekiq_capsule] || default_configuration).redis_pool
   end
 
-  def self.default_server_middleware
-    require 'sidekiq/middleware/server/retry_jobs'
-    require 'sidekiq/middleware/server/logging'
-
-    Middleware::Chain.new do |m|
-      m.add Middleware::Server::RetryJobs
-      m.add Middleware::Server::Logging
-    end
+  def self.redis(&block)
+    (Thread.current[:sidekiq_capsule] || default_configuration).redis(&block)
   end
 
-  def self.default_worker_options=(hash)
-    @default_worker_options = default_worker_options.merge(hash.stringify_keys)
-  end
-  def self.default_worker_options
-    defined?(@default_worker_options) ? @default_worker_options : DEFAULT_WORKER_OPTIONS
+  def self.strict_args!(mode = :raise)
+    Sidekiq::Config::DEFAULTS[:on_complex_arguments] = mode
   end
 
-  # Sidekiq.configure_server do |config|
-  #   config.default_retries_exhausted = -> (job, ex) do
-  #   end
-  # end
-  def self.default_retries_exhausted=(prok)
-    @default_retries_exhausted = prok
-  end
-  @default_retries_exhausted = ->(job, ex) { }
-  def self.default_retries_exhausted
-    @default_retries_exhausted
+  def self.default_job_options=(hash)
+    @default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
   end
 
-  def self.load_json(string)
-    JSON.parse(string)
+  def self.default_job_options
+    @default_job_options ||= {"retry" => true, "queue" => "default"}
   end
-  def self.dump_json(object)
-    JSON.generate(object)
+
+  def self.default_configuration
+    @config ||= Sidekiq::Config.new
   end
 
   def self.logger
-    Sidekiq::Logging.logger
+    default_configuration.logger
   end
-  def self.logger=(log)
-    Sidekiq::Logging.logger = log
+
+  def self.configure_server(&block)
+    (@config_blocks ||= []) << block
+    yield default_configuration if server?
   end
 
-  # How frequently Redis should be checked by a random Sidekiq process for
-  # scheduled and retriable jobs. Each individual process will take turns by
-  # waiting some multiple of this value.
-  #
-  # See sidekiq/scheduled.rb for an in-depth explanation of this value
-  def self.average_scheduled_poll_interval=(interval)
-    self.options[:average_scheduled_poll_interval] = interval
+  def self.freeze!
+    @frozen = true
+    @config_blocks = nil
   end
 
-  # Register a proc to handle any error which occurs within the Sidekiq process.
+  # Creates a Sidekiq::Config instance that is more tuned for embedding
+  # within an arbitrary Ruby process. Notably it reduces concurrency by
+  # default so there is less contention for CPU time with other threads.
   #
-  #   Sidekiq.configure_server do |config|
-  #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+  #   inst = Sidekiq.configure_embed do |config|
+  #     config.queues = %w[critical default low]
   #   end
+  #   inst.run
+  #   sleep 10
+  #   inst.stop
   #
-  # The default error handler logs errors to Sidekiq.logger.
-  def self.error_handlers
-    self.options[:error_handlers]
+  # NB: it is really easy to overload a Ruby process with threads due to the GIL.
+  # I do not recommend setting concurrency higher than 2-3.
+  #
+  # NB: Sidekiq only supports one instance in memory. You will get undefined behavior
+  # if you try to embed Sidekiq twice in the same process.
+  def self.configure_embed(&block)
+    raise "Sidekiq global configuration is frozen, you must create all embedded instances BEFORE calling `run`" if @frozen
+
+    require "sidekiq/embedded"
+    cfg = default_configuration
+    cfg.concurrency = 2
+    @config_blocks&.each { |block| block.call(cfg) }
+    yield cfg
+
+    Sidekiq::Embedded.new(cfg)
   end
 
-  # Register a block to run at a point in the Sidekiq lifecycle.
-  # :startup, :quiet or :shutdown are valid events.
-  #
-  #   Sidekiq.configure_server do |config|
-  #     config.on(:shutdown) do
-  #       puts "Goodbye cruel world!"
-  #     end
-  #   end
-  def self.on(event, &block)
-    raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
-    raise ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
-    options[:lifecycle_events][event] << block
+  def self.configure_client
+    yield default_configuration unless server?
   end
 
-  # We are shutting down Sidekiq but what about workers that
+  # We are shutting down Sidekiq but what about threads that
   # are working on some long job?  This error is
-  # raised in workers that have not finished within the hard
+  # raised in jobs that have not finished within the hard
   # timeout limit.  This is needed to rollback db transactions,
   # otherwise Ruby's Thread#kill will commit.  See #377.
-  # DO NOT RESCUE THIS ERROR IN YOUR WORKERS
+  # DO NOT RESCUE THIS ERROR IN YOUR JOBS
   class Shutdown < Interrupt; end
-
 end
 
-require 'sidekiq/extensions/class_methods'
-require 'sidekiq/extensions/action_mailer'
-require 'sidekiq/extensions/active_record'
-require 'sidekiq/rails' if defined?(::Rails::Engine)
+require "sidekiq/rails" if defined?(::Rails::Engine)