sidekiq 4.1.1 → 4.2.0

data/lib/sidekiq/core_ext.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 begin
   require 'active_support/core_ext/class/attribute'
 rescue LoadError

data/lib/sidekiq/exception_handler.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq
@@ -5,7 +6,7 @@ module Sidekiq
 
     class Logger
       def call(ex, ctxHash)
-        Sidekiq.logger.warn(ctxHash) if !ctxHash.empty?
+        Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
         Sidekiq.logger.warn "#{ex.class.name}: #{ex.message}"
         Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
       end

data/lib/sidekiq/extensions/action_mailer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq

data/lib/sidekiq/extensions/active_record.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq

data/lib/sidekiq/extensions/class_methods.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'yaml'
 
 module Sidekiq

data/lib/sidekiq/fetch.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq

data/lib/sidekiq/launcher.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # encoding: utf-8
 require 'sidekiq/manager'
 require 'sidekiq/fetch'
@@ -79,7 +80,7 @@ module Sidekiq
         workers_key = "#{key}:workers".freeze
         nowdate = Time.now.utc.strftime("%Y-%m-%d".freeze)
         Sidekiq.redis do |conn|
-          conn.pipelined do
+          conn.multi do
             conn.incrby("stat:processed".freeze, procd)
             conn.incrby("stat:processed:#{nowdate}", procd)
             conn.incrby("stat:failed".freeze, fails)
@@ -88,19 +89,24 @@ module Sidekiq
             Processor::WORKER_STATE.each_pair do |tid, hash|
               conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
             end
+            conn.expire(workers_key, 60)
           end
         end
         fails = procd = 0
 
-        _, _, _, msg = Sidekiq.redis do |conn|
-          conn.pipelined do
+        _, exists, _, _, msg = Sidekiq.redis do |conn|
+          conn.multi do
             conn.sadd('processes', key)
+            conn.exists(key)
             conn.hmset(key, 'info', json, 'busy', Processor::WORKER_STATE.size, 'beat', Time.now.to_f, 'quiet', @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
           end
         end
 
+        # first heartbeat or recovering from an outage and need to reestablish our heartbeat
+        fire_event(:heartbeat) if !exists
+
         return unless msg
 
         if JVM_RESERVED_SIGNALS.include?(msg)

data/lib/sidekiq/logging.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'time'
 require 'logger'
 require 'fcntl'
@@ -47,7 +48,7 @@ module Sidekiq
     end
 
     def self.logger=(log)
-      @logger = (log ? log : Logger.new('/dev/null'))
+      @logger = (log ? log : Logger.new(File::NULL))
     end
 
     # This reopens ALL logfiles in the process that have been rotated

data/lib/sidekiq/manager.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
 # encoding: utf-8
 require 'sidekiq/util'
 require 'sidekiq/processor'
 require 'sidekiq/fetch'
 require 'thread'
+require 'set'
 
 module Sidekiq
 

data/lib/sidekiq/middleware/chain.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   # Middleware is code configured to run before/after
   # a message is processed.  It is patterned after Rack

data/lib/sidekiq/middleware/i18n.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # Simple middleware to save the current locale and restore it when the job executes.
 # Use it by requiring it in your initializer:

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -8,14 +8,14 @@ module Sidekiq
       # Automatically retry jobs that fail in Sidekiq.
       # Sidekiq's retry support assumes a typical development lifecycle:
       #
-      #   0. push some code changes with a bug in it
-      #   1. bug causes job processing to fail, sidekiq's middleware captures
-      #      the job and pushes it onto a retry queue
-      #   2. sidekiq retries jobs in the retry queue multiple times with
-      #      an exponential delay, the job continues to fail
-      #   3. after a few days, a developer deploys a fix.  the job is
+      #   0. Push some code changes with a bug in it.
+      #   1. Bug causes job processing to fail, Sidekiq's middleware captures
+      #      the job and pushes it onto a retry queue.
+      #   2. Sidekiq retries jobs in the retry queue multiple times with
+      #      an exponential delay, the job continues to fail.
+      #   3. After a few days, a developer deploys a fix. The job is
       #      reprocessed successfully.
-      #   4. once retries are exhausted, sidekiq will give up and move the
+      #   4. Once retries are exhausted, Sidekiq will give up and move the
       #      job to the Dead Job Queue (aka morgue) where it must be dealt with
       #      manually in the Web UI.
       #   5. After 6 months on the DJQ, Sidekiq will discard the job.

data/lib/sidekiq/paginator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   module Paginator
 

data/lib/sidekiq/processor.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/util'
 require 'sidekiq/fetch'
 require 'thread'
@@ -35,6 +36,7 @@ module Sidekiq
       @job = nil
       @thread = nil
       @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
+      @reloader = Sidekiq.options[:reloader]
     end
 
     def terminate(wait=false)
@@ -117,33 +119,35 @@ module Sidekiq
       jobstr = work.job
       queue = work.queue_name
 
-      ack = false
-      begin
-        job = Sidekiq.load_json(jobstr)
-        klass  = job['class'.freeze].constantize
-        worker = klass.new
-        worker.jid = job['jid'.freeze]
-
-        stats(worker, job, queue) do
-          Sidekiq.server_middleware.invoke(worker, job, queue) do
-            # Only ack if we either attempted to start this job or
-            # successfully completed it. This prevents us from
-            # losing jobs if a middleware raises an exception before yielding
-            ack = true
-            execute_job(worker, cloned(job['args'.freeze]))
+      @reloader.call do
+        ack = false
+        begin
+          job = Sidekiq.load_json(jobstr)
+          klass  = job['class'.freeze].constantize
+          worker = klass.new
+          worker.jid = job['jid'.freeze]
+
+          stats(worker, job, queue) do
+            Sidekiq.server_middleware.invoke(worker, job, queue) do
+              # Only ack if we either attempted to start this job or
+              # successfully completed it. This prevents us from
+              # losing jobs if a middleware raises an exception before yielding
+              ack = true
+              execute_job(worker, cloned(job['args'.freeze]))
+            end
           end
+          ack = true
+        rescue Sidekiq::Shutdown
+          # Had to force kill this job because it didn't finish
+          # within the timeout.  Don't acknowledge the work since
+          # we didn't properly finish it.
+          ack = false
+        rescue Exception => ex
+          handle_exception(ex, job || { :job => jobstr })
+          raise
+        ensure
+          work.acknowledge if ack
         end
-        ack = true
-      rescue Sidekiq::Shutdown
-        # Had to force kill this job because it didn't finish
-        # within the timeout.  Don't acknowledge the work since
-        # we didn't properly finish it.
-        ack = false
-      rescue Exception => ex
-        handle_exception(ex, job || { :job => jobstr })
-        raise
-      ensure
-        work.acknowledge if ack
       end
     end
 

data/lib/sidekiq/rails.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   def self.hook_rails!
     return if defined?(@delay_removed)
@@ -34,5 +35,22 @@ module Sidekiq
     initializer 'sidekiq' do
       Sidekiq.hook_rails!
     end
+
+    class Reloader
+      def initialize(app = ::Rails.application)
+        Sidekiq.logger.debug "Enabling Rails 5+ live code reloading, so hot!" unless app.config.cache_classes
+        @app = app
+      end
+
+      def call
+        @app.reloader.wrap do
+          yield
+        end
+      end
+
+      def inspect
+        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+      end
+    end
   end if defined?(::Rails)
 end

data/lib/sidekiq/redis_connection.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'connection_pool'
 require 'redis'
 require 'uri'
@@ -7,6 +8,8 @@ module Sidekiq
     class << self
 
       def create(options={})
+        options = options.symbolize_keys
+
         options[:url] ||= determine_redis_provider
 
         size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 5) : 5)
@@ -32,7 +35,7 @@ module Sidekiq
       #   - enterprise's leader election
       #   - enterprise's cron support
       def verify_sizing(size, concurrency)
-        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work, your pool has #{size} connections but really needs to have at least #{concurrency + 2}" if size <= concurrency
+        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but really needs to have at least #{concurrency + 2}" if size <= concurrency
       end
 
       def build_client(options)
@@ -44,8 +47,8 @@ module Sidekiq
             require 'redis/namespace'
             Redis::Namespace.new(namespace, :redis => client)
           rescue LoadError
-            Sidekiq.logger.error("Your Redis configuration use the namespace '#{namespace}' but the redis-namespace gem not included in Gemfile." \
-                                 "Add the gem to your Gemfile in case you would like to keep using a namespace, otherwise remove the namespace parameter.")
+            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
+                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
             exit(-127)
           end
         else

data/lib/sidekiq/scheduled.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq'
 require 'sidekiq/util'
 require 'sidekiq/api'
@@ -45,6 +46,7 @@ module Sidekiq
         @enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
+        @thread = nil
       end
 
       # Shut down this instance, will pause until the thread is dead.

data/lib/sidekiq/testing.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'securerandom'
 require 'sidekiq'
 

data/lib/sidekiq/testing/inline.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/testing'
 
 ##

data/lib/sidekiq/util.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'socket'
 require 'securerandom'
 require 'sidekiq/exception_handler'

data/lib/sidekiq/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
-  VERSION = "4.1.1"
+  VERSION = "4.2.0"
 end

data/lib/sidekiq/web.rb

@@ -1,25 +1,28 @@
+# frozen_string_literal: true
 require 'erb'
-require 'yaml'
-require 'sinatra/base'
 
 require 'sidekiq'
 require 'sidekiq/api'
 require 'sidekiq/paginator'
-require 'sidekiq/web_helpers'
+require 'sidekiq/web/helpers'
 
-module Sidekiq
-  class Web < Sinatra::Base
-    include Sidekiq::Paginator
+require 'sidekiq/web/router'
+require 'sidekiq/web/action'
+require 'sidekiq/web/application'
 
-    enable :sessions
-    use ::Rack::Protection, :use => :authenticity_token unless ENV['RACK_ENV'] == 'test'
+require 'rack/protection'
 
-    set :root, File.expand_path(File.dirname(__FILE__) + "/../../web")
-    set :public_folder, proc { "#{root}/assets" }
-    set :views, proc { "#{root}/views" }
-    set :locales, ["#{root}/locales"]
+require 'rack/builder'
+require 'rack/file'
+require 'rack/session/cookie'
 
-    helpers WebHelpers
+module Sidekiq
+  class Web
+    ROOT = File.expand_path("#{File.dirname(__FILE__)}/../../web")
+    VIEWS = "#{ROOT}/views".freeze
+    LOCALES = ["#{ROOT}/locales".freeze]
+    LAYOUT = "#{VIEWS}/layout.erb".freeze
+    ASSETS = "#{ROOT}/assets".freeze
 
     DEFAULT_TABS = {
       "Dashboard" => '',
@@ -31,6 +34,18 @@ module Sidekiq
     }
 
     class << self
+      def settings
+        self
+      end
+
+      def middlewares
+        @middlewares ||= []
+      end
+
+      def use(*middleware_args, &block)
+        middlewares << [middleware_args, block]
+      end
+
       def default_tabs
         DEFAULT_TABS
       end
@@ -40,234 +55,97 @@ module Sidekiq
       end
       alias_method :tabs, :custom_tabs
 
-      attr_accessor :app_url
-    end
-
-    get "/busy" do
-      erb :busy
-    end
-
-    post "/busy" do
-      if params['identity']
-        p = Sidekiq::Process.new('identity' => params['identity'])
-        p.quiet! if params[:quiet]
-        p.stop! if params[:stop]
-      else
-        processes.each do |pro|
-          pro.quiet! if params[:quiet]
-          pro.stop! if params[:stop]
-        end
+      def locales
+        @locales ||= LOCALES
       end
-      redirect "#{root_path}busy"
-    end
 
-    get "/queues" do
-      @queues = Sidekiq::Queue.all
-      erb :queues
-    end
-
-    get "/queues/:name" do
-      halt 404 unless params[:name]
-      @count = (params[:count] || 25).to_i
-      @name = params[:name]
-      @queue = Sidekiq::Queue.new(@name)
-      (@current_page, @total_size, @messages) = page("queue:#{@name}", params[:page], @count)
-      @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
-      erb :queue
-    end
-
-    post "/queues/:name" do
-      Sidekiq::Queue.new(params[:name]).clear
-      redirect "#{root_path}queues"
-    end
-
-    post "/queues/:name/delete" do
-      Sidekiq::Job.new(params[:key_val], params[:name]).delete
-      redirect_with_query("#{root_path}queues/#{params[:name]}")
-    end
-
-    get '/morgue' do
-      @count = (params[:count] || 25).to_i
-      (@current_page, @total_size, @dead) = page("dead", params[:page], @count, reverse: true)
-      @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
-      erb :morgue
-    end
-
-    get "/morgue/:key" do
-      halt 404 unless params['key']
-      @dead = Sidekiq::DeadSet.new.fetch(*parse_params(params['key'])).first
-      redirect "#{root_path}morgue" if @dead.nil?
-      erb :dead
-    end
-
-    post '/morgue' do
-      redirect request.path unless params['key']
-
-      params['key'].each do |key|
-        job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
-        retry_or_delete_or_kill job, params if job
+      def views
+        @views ||= VIEWS
       end
-      redirect_with_query("#{root_path}morgue")
-    end
 
-    post "/morgue/all/delete" do
-      Sidekiq::DeadSet.new.clear
-      redirect "#{root_path}morgue"
+      attr_accessor :app_url, :session_secret, :redis_pool
+      attr_writer :locales, :views
     end
 
-    post "/morgue/all/retry" do
-      Sidekiq::DeadSet.new.retry_all
-      redirect "#{root_path}morgue"
+    def settings
+      self.class.settings
     end
 
-    post "/morgue/:key" do
-      halt 404 unless params['key']
-      job = Sidekiq::DeadSet.new.fetch(*parse_params(params['key'])).first
-      retry_or_delete_or_kill job, params if job
-      redirect_with_query("#{root_path}morgue")
+    def use(*middleware_args, &block)
+      middlewares << [middleware_args, block]
     end
 
-
-    get '/retries' do
-      @count = (params[:count] || 25).to_i
-      (@current_page, @total_size, @retries) = page("retry", params[:page], @count)
-      @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
-      erb :retries
-    end
-
-    get "/retries/:key" do
-      @retry = Sidekiq::RetrySet.new.fetch(*parse_params(params['key'])).first
-      redirect "#{root_path}retries" if @retry.nil?
-      erb :retry
-    end
-
-    post '/retries' do
-      redirect request.path unless params['key']
-
-      params['key'].each do |key|
-        job = Sidekiq::RetrySet.new.fetch(*parse_params(key)).first
-        retry_or_delete_or_kill job, params if job
-      end
-      redirect_with_query("#{root_path}retries")
+    def middlewares
+      @middlewares ||= Web.middlewares.dup
     end
 
-    post "/retries/all/delete" do
-      Sidekiq::RetrySet.new.clear
-      redirect "#{root_path}retries"
+    def call(env)
+      app.call(env)
     end
 
-    post "/retries/all/retry" do
-      Sidekiq::RetrySet.new.retry_all
-      redirect "#{root_path}retries"
+    def self.call(env)
+      @app ||= new
+      @app.call(env)
     end
 
-    post "/retries/:key" do
-      job = Sidekiq::RetrySet.new.fetch(*parse_params(params['key'])).first
-      retry_or_delete_or_kill job, params if job
-      redirect_with_query("#{root_path}retries")
+    def app
+      @app ||= build
     end
 
-    get '/scheduled' do
-      @count = (params[:count] || 25).to_i
-      (@current_page, @total_size, @scheduled) = page("schedule", params[:page], @count)
-      @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
-      erb :scheduled
+    def self.register(extension)
+      extension.registered(WebApplication)
     end
 
-    get "/scheduled/:key" do
-      @job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
-      redirect "#{root_path}scheduled" if @job.nil?
-      erb :scheduled_job_info
-    end
-
-    post '/scheduled' do
-      redirect request.path unless params['key']
+    private
 
-      params['key'].each do |key|
-        job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
-        delete_or_add_queue job, params if job
+    def using?(middleware)
+      middlewares.any? do |(m,_)|
+        m.kind_of?(Array) && (m[0] == middleware || m[0].kind_of?(middleware))
       end
-      redirect_with_query("#{root_path}scheduled")
-    end
-
-    post "/scheduled/:key" do
-      halt 404 unless params['key']
-      job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
-      delete_or_add_queue job, params if job
-      redirect_with_query("#{root_path}scheduled")
-    end
-
-    get '/' do
-      @redis_info = redis_info.select{ |k, v| REDIS_KEYS.include? k }
-      stats_history = Sidekiq::Stats::History.new((params[:days] || 30).to_i)
-      @processed_history = stats_history.processed
-      @failed_history = stats_history.failed
-      erb :dashboard
     end
 
-    REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
+    def build
+      middlewares = self.middlewares
+      klass = self.class
 
-    get '/dashboard/stats' do
-      redirect "#{root_path}stats"
-    end
-
-    get '/stats' do
-      sidekiq_stats = Sidekiq::Stats.new
-      redis_stats   = redis_info.select { |k, v| REDIS_KEYS.include? k }
-
-      content_type :json
-      Sidekiq.dump_json(
-        sidekiq: {
-          processed:       sidekiq_stats.processed,
-          failed:          sidekiq_stats.failed,
-          busy:            sidekiq_stats.workers_size,
-          processes:       sidekiq_stats.processes_size,
-          enqueued:        sidekiq_stats.enqueued,
-          scheduled:       sidekiq_stats.scheduled_size,
-          retries:         sidekiq_stats.retry_size,
-          dead:            sidekiq_stats.dead_size,
-          default_latency: sidekiq_stats.default_queue_latency
-        },
-        redis: redis_stats
-      )
-    end
+      unless using?(::Rack::Protection) || ENV['RACK_ENV'] == 'test'
+        middlewares.unshift [[::Rack::Protection, { use: :authenticity_token }], nil]
+      end
 
-    get '/stats/queues' do
-      queue_stats = Sidekiq::Stats::Queues.new
+      unless using? ::Rack::Session::Cookie
+        unless secret = Web.session_secret
+          require 'securerandom'
+          secret = SecureRandom.hex(64)
+        end
 
-      content_type :json
-      Sidekiq.dump_json(
-        queue_stats.lengths
-      )
-    end
+        middlewares.unshift [[::Rack::Session::Cookie, { secret: secret }], nil]
+      end
 
-    private
+      ::Rack::Builder.new do
+        %w(stylesheets javascripts images).each do |asset_dir|
+          map "/#{asset_dir}" do
+            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", { 'Cache-Control' => 'public, max-age=86400' })
+          end
+        end
 
-    def retry_or_delete_or_kill job, params
-      if params['retry']
-        job.retry
-      elsif params['delete']
-        job.delete
-      elsif params['kill']
-        job.kill
-      end
-    end
+        middlewares.each {|middleware, block| use(*middleware, &block) }
 
-    def delete_or_add_queue job, params
-      if params['delete']
-        job.delete
-      elsif params['add_to_queue']
-        job.add_to_queue
+        run WebApplication.new(klass)
       end
     end
   end
+
+  Sidekiq::WebApplication.helpers WebHelpers
+  Sidekiq::WebApplication.helpers Sidekiq::Paginator
+
+  Sidekiq::WebAction.class_eval "def _render\n#{ERB.new(File.read(Web::LAYOUT)).src}\nend"
 end
 
 if defined?(::ActionDispatch::Request::Session) &&
     !::ActionDispatch::Request::Session.respond_to?(:each)
   # mperham/sidekiq#2460
   # Rack apps can't reuse the Rails session store without
-  # this monkeypatch
+  # this monkeypatch, fixed in Rails 5.
   class ActionDispatch::Request::Session
     def each(&block)
       hash = self.to_hash