sidekiq 0.10.0 → 7.2.0

data/lib/sidekiq/web.rb

@@ -1,105 +1,174 @@
-require 'sinatra/base'
-require 'slim'
-require 'sprockets'
+# frozen_string_literal: true
+
+require "erb"
+
+require "sidekiq"
+require "sidekiq/api"
+require "sidekiq/paginator"
+require "sidekiq/web/helpers"
+
+require "sidekiq/web/router"
+require "sidekiq/web/action"
+require "sidekiq/web/application"
+require "sidekiq/web/csrf_protection"
+
+require "rack/content_length"
+require "rack/builder"
+require "rack/static"
 
 module Sidekiq
-  class SprocketsMiddleware
-    def initialize(app, options={})
-      @app = app
-      @root = options[:root]
-      path   =  options[:path] || 'assets'
-      @matcher = /^\/#{path}\/*/
-      @environment = ::Sprockets::Environment.new(@root)
-      @environment.append_path 'assets/javascripts'
-      @environment.append_path 'assets/javascripts/vendor'
-      @environment.append_path 'assets/stylesheets'
-      @environment.append_path 'assets/stylesheets/vendor'
-      @environment.append_path 'assets/images'
+  class Web
+    ROOT = File.expand_path("#{File.dirname(__FILE__)}/../../web")
+    VIEWS = "#{ROOT}/views"
+    LOCALES = ["#{ROOT}/locales"]
+    LAYOUT = "#{VIEWS}/layout.erb"
+    ASSETS = "#{ROOT}/assets"
+
+    DEFAULT_TABS = {
+      "Dashboard" => "",
+      "Busy" => "busy",
+      "Queues" => "queues",
+      "Retries" => "retries",
+      "Scheduled" => "scheduled",
+      "Dead" => "morgue",
+      "Metrics" => "metrics"
+    }
+
+    if Gem::Version.new(Rack::RELEASE) < Gem::Version.new("3")
+      CONTENT_LANGUAGE = "Content-Language"
+      CONTENT_SECURITY_POLICY = "Content-Security-Policy"
+      LOCATION = "Location"
+      X_CASCADE = "X-Cascade"
+    else
+      CONTENT_LANGUAGE = "content-language"
+      CONTENT_SECURITY_POLICY = "content-security-policy"
+      LOCATION = "location"
+      X_CASCADE = "x-cascade"
     end
 
-    def call(env)
-      # Solve the problem of people requesting /sidekiq when they need to request /sidekiq/ so
-      # that relative links in templates resolve correctly.
-      return [301, { 'Location' => "#{env['SCRIPT_NAME']}/" }, []] if env['SCRIPT_NAME'] == env['REQUEST_PATH']
+    class << self
+      def settings
+        self
+      end
 
-      return @app.call(env) unless @matcher =~ env["PATH_INFO"]
-      env['PATH_INFO'].sub!(@matcher,'')
-      @environment.call(env)
-    end
-  end
+      def default_tabs
+        DEFAULT_TABS
+      end
 
-  class Web < Sinatra::Base
-    dir = File.expand_path(File.dirname(__FILE__) + "/../../web")
-    set :views,  "#{dir}/views"
-    set :root, "#{dir}/public"
-    set :slim, :pretty => true
-    use SprocketsMiddleware, :root => dir
-
-    helpers do
-      def workers
-        @workers ||= begin
-          Sidekiq.redis do |conn|
-            conn.smembers('workers').map do |w|
-              msg = conn.get("worker:#{w}")
-              msg = MultiJson.decode(msg) if msg
-              [w, msg]
-            end.sort { |x| x[1] ? -1 : 1 }
-          end
-        end
+      def custom_tabs
+        @custom_tabs ||= {}
       end
+      alias_method :tabs, :custom_tabs
 
-      def processed
-        Sidekiq.redis { |conn| conn.get('stat:processed') } || 0
+      def custom_job_info_rows
+        @custom_job_info_rows ||= []
       end
 
-      def failed
-        Sidekiq.redis { |conn| conn.get('stat:failed') } || 0
+      def locales
+        @locales ||= LOCALES
       end
 
-      def retry_count
-        Sidekiq.redis { |conn| conn.zcard('retry') }
+      def views
+        @views ||= VIEWS
       end
 
-      def retries
-        Sidekiq.redis do |conn|
-          results = conn.zrange('retry', 0, 25, :withscores => true)
-          results.each_slice(2).map { |msg, score| [MultiJson.decode(msg), Float(score)] }
-        end
+      def enable(*opts)
+        opts.each { |key| set(key, true) }
       end
 
-      def queues
-        Sidekiq.redis do |conn|
-          conn.smembers('queues').map do |q|
-            [q, conn.llen("queue:#{q}") || 0]
-          end.sort { |x,y| x[1] <=> y[1] }
-        end
+      def disable(*opts)
+        opts.each { |key| set(key, false) }
       end
 
-      def location
-        Sidekiq.redis { |conn| conn.client.location }
+      def middlewares
+        @middlewares ||= []
       end
 
-      def root_path
-        "#{env['SCRIPT_NAME']}/"
+      def use(*args, &block)
+        middlewares << [args, block]
       end
 
-      def current_status
-        return 'down' if workers.size == 0
-        return 'idle' if workers.size > 0 && workers.map { |x| x[1] }.compact.size == 0
-        return 'active'
+      def set(attribute, value)
+        send(:"#{attribute}=", value)
       end
+
+      attr_accessor :app_url, :redis_pool
+      attr_writer :locales, :views
     end
 
-    get "/" do
-      slim :index
+    def self.inherited(child)
+      child.app_url = app_url
+      child.redis_pool = redis_pool
+    end
+
+    def settings
+      self.class.settings
+    end
+
+    def middlewares
+      @middlewares ||= self.class.middlewares
+    end
+
+    def use(*args, &block)
+      middlewares << [args, block]
+    end
+
+    def call(env)
+      app.call(env)
     end
 
-    get "/queues/:name" do
-      halt 404 unless params[:name]
-      @name = params[:name]
-      @messages = Sidekiq.redis {|conn| conn.lrange("queue:#{@name}", 0, 10) }.map { |str| MultiJson.decode(str) }
-      slim :queue
+    def self.call(env)
+      @app ||= new
+      @app.call(env)
+    end
+
+    def app
+      @app ||= build
+    end
+
+    def enable(*opts)
+      opts.each { |key| set(key, true) }
+    end
+
+    def disable(*opts)
+      opts.each { |key| set(key, false) }
+    end
+
+    def set(attribute, value)
+      send(:"#{attribute}=", value)
+    end
+
+    def self.register(extension)
+      extension.registered(WebApplication)
+    end
+
+    private
+
+    def build
+      klass = self.class
+      m = middlewares
+
+      rules = []
+      rules = [[:all, {Rack::CACHE_CONTROL => "private, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
+
+      ::Rack::Builder.new do
+        use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
+          root: ASSETS,
+          cascade: true,
+          header_rules: rules
+        m.each { |middleware, block| use(*middleware, &block) }
+        use Sidekiq::Web::CsrfProtection unless $TESTING
+        run WebApplication.new(klass)
+      end
     end
   end
 
+  Sidekiq::WebApplication.helpers WebHelpers
+  Sidekiq::WebApplication.helpers Sidekiq::Paginator
+
+  Sidekiq::WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+    def _render
+      #{ERB.new(File.read(Web::LAYOUT)).src}
+    end
+  RUBY
 end

data/lib/sidekiq/worker_compatibility_alias.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
+  Worker = Job
+end

data/lib/sidekiq.rb

@@ -1,100 +1,147 @@
-require 'sidekiq/version'
-require 'sidekiq/client'
-require 'sidekiq/worker'
-require 'sidekiq/rails'
-require 'sidekiq/redis_connection'
-require 'sidekiq/util'
+# frozen_string_literal: true
 
-require 'sidekiq/extensions/action_mailer'
-require 'sidekiq/extensions/active_record'
+require "sidekiq/version"
+fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.7.0." if RUBY_PLATFORM != "java" && Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.7.0")
+
+begin
+  require "sidekiq-ent/version"
+  fail <<~EOM  if Gem::Version.new(Sidekiq::Enterprise::VERSION).segments[0] != Sidekiq::MAJOR
+
+    Sidekiq Enterprise #{Sidekiq::Enterprise::VERSION} does not work with Sidekiq #{Sidekiq::VERSION}.
+    Starting with Sidekiq 7, major versions are synchronized so Sidekiq Enterprise 7 works with Sidekiq 7.
+    Use `bundle up sidekiq-ent` to upgrade.
+
+  EOM
+rescue LoadError
+end
+
+begin
+  require "sidekiq/pro/version"
+  fail <<~EOM  if Gem::Version.new(Sidekiq::Pro::VERSION).segments[0] != Sidekiq::MAJOR
+
+    Sidekiq Pro #{Sidekiq::Pro::VERSION} does not work with Sidekiq #{Sidekiq::VERSION}.
+    Starting with Sidekiq 7, major versions are synchronized so Sidekiq Pro 7 works with Sidekiq 7.
+    Use `bundle up sidekiq-pro` to upgrade.
+
+  EOM
+rescue LoadError
+end
+
+require "sidekiq/config"
+require "sidekiq/logger"
+require "sidekiq/client"
+require "sidekiq/transaction_aware_client"
+require "sidekiq/job"
+require "sidekiq/worker_compatibility_alias"
+require "sidekiq/redis_client_adapter"
+
+require "json"
 
 module Sidekiq
+  NAME = "Sidekiq"
+  LICENSE = "See LICENSE and the LGPL-3.0 for licensing details."
 
-  DEFAULTS = {
-    :queues => [],
-    :concurrency => 25,
-    :require => '.',
-    :environment => nil,
-    :timeout => 5,
-    :enable_rails_extensions => true,
-  }
+  def self.❨╯°□°❩╯︵┻━┻
+    puts "Take a deep breath and count to ten..."
+  end
 
-  def self.options
-    @options ||= DEFAULTS.dup
+  def self.server?
+    defined?(Sidekiq::CLI)
   end
 
-  def self.options=(opts)
-    @options = opts
+  def self.load_json(string)
+    JSON.parse(string)
   end
 
-  ##
-  # Configuration for Sidekiq server, use like:
-  #
-  #   Sidekiq.configure_server do |config|
-  #     config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/mydb' }
-  #     config.server_middleware do |chain|
-  #       chain.add MyServerHook
-  #     end
-  #   end
-  def self.configure_server
-    yield self if server?
+  def self.dump_json(object)
+    JSON.generate(object)
   end
 
-  ##
-  # Configuration for Sidekiq client, use like:
-  #
-  #   Sidekiq.configure_client do |config|
-  #     config.redis = { :namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/mydb' }
-  #   end
-  def self.configure_client
-    yield self unless server?
+  def self.pro?
+    defined?(Sidekiq::Pro)
   end
 
-  def self.server?
-    defined?(Sidekiq::CLI)
+  def self.ent?
+    defined?(Sidekiq::Enterprise)
+  end
+
+  def self.redis_pool
+    (Thread.current[:sidekiq_capsule] || default_configuration).redis_pool
   end
 
   def self.redis(&block)
-    @redis ||= Sidekiq::RedisConnection.create
-    if block_given?
-      @redis.with(&block)
-    else
-      Sidekiq::Util.logger.info "*****************************************************
-Sidekiq.redis now takes a block:
+    (Thread.current[:sidekiq_capsule] || default_configuration).redis(&block)
+  end
+
+  def self.strict_args!(mode = :raise)
+    Sidekiq::Config::DEFAULTS[:on_complex_arguments] = mode
+  end
+
+  def self.default_job_options=(hash)
+    @default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
+  end
+
+  def self.default_job_options
+    @default_job_options ||= {"retry" => true, "queue" => "default"}
+  end
+
+  def self.default_configuration
+    @config ||= Sidekiq::Config.new
+  end
 
-  Sidekiq.redis { |connection| connection.smembers('myset') }
+  def self.logger
+    default_configuration.logger
+  end
 
-Please update your code accordingly.
-Called from #{caller[0]}
-*****************************************************"
-      @redis
-    end
+  def self.configure_server(&block)
+    (@config_blocks ||= []) << block
+    yield default_configuration if server?
   end
 
-  def self.redis=(hash)
-    @redis = if hash.is_a?(Hash)
-      RedisConnection.create(hash)
-    else
-      Sidekiq::Util.logger.info "*****************************************************
-Sidekiq.redis now takes a Hash:
-old: Sidekiq.redis = Sidekiq::RedisConnection.create(:url => 'redis://foo.com', :namespace => 'abc', :size => 12)
-new: Sidekiq.redis = { :url => 'redis://foo.com', :namespace => 'xyz', :size => 12 }
-Called from #{caller[0]}
-*****************************************************"
-      hash
-    end
+  def self.freeze!
+    @frozen = true
+    @config_blocks = nil
   end
 
-  def self.client_middleware
-    @client_chain ||= Client.default_middleware
-    yield @client_chain if block_given?
-    @client_chain
+  # Creates a Sidekiq::Config instance that is more tuned for embedding
+  # within an arbitrary Ruby process. Notably it reduces concurrency by
+  # default so there is less contention for CPU time with other threads.
+  #
+  #   inst = Sidekiq.configure_embed do |config|
+  #     config.queues = %w[critical default low]
+  #   end
+  #   inst.run
+  #   sleep 10
+  #   inst.terminate
+  #
+  # NB: it is really easy to overload a Ruby process with threads due to the GIL.
+  # I do not recommend setting concurrency higher than 2-3.
+  #
+  # NB: Sidekiq only supports one instance in memory. You will get undefined behavior
+  # if you try to embed Sidekiq twice in the same process.
+  def self.configure_embed(&block)
+    raise "Sidekiq global configuration is frozen, you must create all embedded instances BEFORE calling `run`" if @frozen
+
+    require "sidekiq/embedded"
+    cfg = default_configuration
+    cfg.concurrency = 2
+    @config_blocks&.each { |block| block.call(cfg) }
+    yield cfg
+
+    Sidekiq::Embedded.new(cfg)
   end
 
-  def self.server_middleware
-    @server_chain ||= Processor.default_middleware
-    yield @server_chain if block_given?
-    @server_chain
+  def self.configure_client
+    yield default_configuration unless server?
   end
 
+  # We are shutting down Sidekiq but what about threads that
+  # are working on some long job?  This error is
+  # raised in jobs that have not finished within the hard
+  # timeout limit.  This is needed to rollback db transactions,
+  # otherwise Ruby's Thread#kill will commit.  See #377.
+  # DO NOT RESCUE THIS ERROR IN YOUR JOBS
+  class Shutdown < Interrupt; end
 end
+
+require "sidekiq/rails" if defined?(::Rails::Engine)

data/sidekiq.gemspec

@@ -1,27 +1,30 @@
-# -*- encoding: utf-8 -*-
-require File.expand_path('../lib/sidekiq/version', __FILE__)
+require_relative "lib/sidekiq/version"
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Mike Perham"]
-  gem.email         = ["mperham@gmail.com"]
-  gem.description   = gem.summary = "Simple, efficient message processing for Ruby"
-  gem.homepage      = "http://mperham.github.com/sidekiq"
+  gem.authors = ["Mike Perham"]
+  gem.email = ["info@contribsys.com"]
+  gem.summary = "Simple, efficient background processing for Ruby"
+  gem.description = "Simple, efficient background processing for Ruby."
+  gem.homepage = "https://sidekiq.org"
+  gem.license = "LGPL-3.0"
 
-  gem.executables   = ['sidekiq', 'sidekiqctl']
-  gem.files         = `git ls-files`.split("\n")
-  gem.test_files    = `git ls-files -- test/*`.split("\n")
-  gem.name          = "sidekiq"
-  gem.require_paths = ["lib"]
-  gem.version       = Sidekiq::VERSION
-  gem.add_dependency                  'redis'
-  gem.add_dependency                  'redis-namespace'
-  gem.add_dependency                  'connection_pool', '~> 0.9.0'
-  gem.add_dependency                  'celluloid'
-  gem.add_dependency                  'multi_json'
-  gem.add_development_dependency      'minitest'
-  gem.add_development_dependency      'sinatra'
-  gem.add_development_dependency      'slim'
-  gem.add_development_dependency      'rake'
-  gem.add_development_dependency      'actionmailer', '~> 3'
-  gem.add_development_dependency      'activerecord', '~> 3'
+  gem.executables = ["sidekiq", "sidekiqmon"]
+  gem.files = %w[sidekiq.gemspec README.md Changes.md LICENSE.txt] + `git ls-files | grep -E '^(bin|lib|web)'`.split("\n")
+  gem.name = "sidekiq"
+  gem.version = Sidekiq::VERSION
+  gem.required_ruby_version = ">= 2.7.0"
+
+  gem.metadata = {
+    "homepage_uri" => "https://sidekiq.org",
+    "bug_tracker_uri" => "https://github.com/sidekiq/sidekiq/issues",
+    "documentation_uri" => "https://github.com/sidekiq/sidekiq/wiki",
+    "changelog_uri" => "https://github.com/sidekiq/sidekiq/blob/main/Changes.md",
+    "source_code_uri" => "https://github.com/sidekiq/sidekiq",
+    "rubygems_mfa_required" => "true"
+  }
+
+  gem.add_dependency "redis-client", ">= 0.14.0"
+  gem.add_dependency "connection_pool", ">= 2.3.0"
+  gem.add_dependency "rack", ">= 2.2.4"
+  gem.add_dependency "concurrent-ruby", "< 2"
 end