sidekiq 2.15.1 → 4.2.10

data/lib/sidekiq/testing.rb

@@ -1,15 +1,19 @@
+# frozen_string_literal: true
+require 'securerandom'
+require 'sidekiq'
+
 module Sidekiq
 
   class Testing
     class << self
       attr_accessor :__test_mode
 
-      def __set_test_mode(mode, &block)
-        if block
+      def __set_test_mode(mode)
+        if block_given?
           current_mode = self.__test_mode
           begin
             self.__test_mode = mode
-            block.call
+            yield
           ensure
             self.__test_mode = current_mode
           end
@@ -45,6 +49,12 @@ module Sidekiq
       def inline?
         self.__test_mode == :inline
       end
+
+      def server_middleware
+        @server_chain ||= Middleware::Chain.new
+        yield @server_chain if block_given?
+        @server_chain
+      end
     end
   end
 
@@ -54,24 +64,127 @@ module Sidekiq
   class EmptyQueueError < RuntimeError; end
 
   class Client
-    class << self
-      alias_method :raw_push_real, :raw_push
+    alias_method :raw_push_real, :raw_push
 
-      def raw_push(payloads)
-        if Sidekiq::Testing.fake?
-          payloads.each do |job|
-            job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
-          end
-          true
-        elsif Sidekiq::Testing.inline?
-          payloads.each do |item|
-            marshalled = Sidekiq.load_json(Sidekiq.dump_json(item))
-            marshalled['class'].constantize.new.perform(*marshalled['args'])
-          end
-          true
-        else
-          raw_push_real(payloads)
+    def raw_push(payloads)
+      if Sidekiq::Testing.fake?
+        payloads.each do |job|
+          job = Sidekiq.load_json(Sidekiq.dump_json(job))
+          job.merge!('enqueued_at' => Time.now.to_f) unless job['at']
+          Queues.push(job['queue'], job['class'], job)
+        end
+        true
+      elsif Sidekiq::Testing.inline?
+        payloads.each do |job|
+          klass = job['class'].constantize
+          job['id'] ||= SecureRandom.hex(12)
+          job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.process_job(job_hash)
         end
+        true
+      else
+        raw_push_real(payloads)
+      end
+    end
+  end
+
+  module Queues
+    ##
+    # The Queues class is only for testing the fake queue implementation.
+    # There are 2 data structures involved in tandem. This is due to the
+    # Rspec syntax of change(QueueWorker.jobs, :size). It keeps a reference
+    # to the array. Because the array was dervied from a filter of the total
+    # jobs enqueued, it appeared as though the array didn't change.
+    #
+    # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
+    # on the queue, and another with keys of the worker names, so the array for
+    # QueueWorker.jobs is a straight reference to a real array.
+    #
+    # Queue-based hash:
+    #
+    # {
+    #   "default"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Worker-based hash:
+    #
+    # {
+    #   "TestTesting::QueueWorker"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, Sidekiq::Queues["default"].size
+    #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
+    #
+    # You can also clear all workers' jobs:
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   Sidekiq::Queues.clear_all
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Queues.clear_all
+    #     end
+    #   end
+    #
+    class << self
+      def [](queue)
+        jobs_by_queue[queue]
+      end
+
+      def push(queue, klass, job)
+        jobs_by_queue[queue] << job
+        jobs_by_worker[klass] << job
+      end
+
+      def jobs_by_queue
+        @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def jobs_by_worker
+        @jobs_by_worker ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def delete_for(jid, queue, klass)
+        jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
+        jobs_by_worker[klass].delete_if { |job| job["jid"] == jid }
+      end
+
+      def clear_for(queue, klass)
+        jobs_by_queue[queue].clear
+        jobs_by_worker[klass].clear
+      end
+
+      def clear_all
+        jobs_by_queue.clear
+        jobs_by_worker.clear
       end
     end
   end
@@ -134,51 +247,78 @@ module Sidekiq
     #
     module ClassMethods
 
+      # Queue for this worker
+      def queue
+        self.sidekiq_options["queue"]
+      end
+
       # Jobs queued for this worker
       def jobs
-        Worker.jobs[self]
+        Queues.jobs_by_worker[self.to_s]
       end
 
       # Clear all jobs for this worker
       def clear
-        jobs.clear
+        Queues.clear_for(queue, self.to_s)
       end
 
       # Drain and run all jobs for this worker
       def drain
-        while job = jobs.shift do
-          worker = new
-          worker.jid = job['jid']
-          worker.perform(*job['args'])
+        while jobs.any?
+          next_job = jobs.first
+          Queues.delete_for(next_job["jid"], next_job["queue"], self.to_s)
+          process_job(next_job)
         end
       end
 
       # Pop out a single job and perform it
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
-        job = jobs.shift
+        next_job = jobs.first
+        Queues.delete_for(next_job["jid"], queue, self.to_s)
+        process_job(next_job)
+      end
+
+      def process_job(job)
         worker = new
         worker.jid = job['jid']
-        worker.perform(*job['args'])
+        worker.bid = job['bid'] if worker.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(worker, job, job['queue']) do
+          execute_job(worker, job['args'])
+        end
+      end
+
+      def execute_job(worker, args)
+        worker.perform(*args)
       end
     end
 
     class << self
       def jobs # :nodoc:
-        @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+        Queues.jobs_by_queue.values.flatten
       end
 
       # Clear all queued jobs across all workers
       def clear_all
-        jobs.clear
+        Queues.clear_all
       end
 
       # Drain all queued jobs across all workers
       def drain_all
-        until jobs.values.all?(&:empty?) do
-          jobs.keys.each(&:drain)
+        while jobs.any?
+          worker_classes = jobs.map { |job| job["class"] }.uniq
+
+          worker_classes.each do |worker_class|
+            worker_class.constantize.drain
+          end
         end
       end
     end
   end
 end
+
+if defined?(::Rails) && !Rails.env.test?
+  puts("**************************************************")
+  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
+  puts("**************************************************")
+end

data/lib/sidekiq/testing/inline.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/testing'
 
 ##

data/lib/sidekiq/util.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
 require 'socket'
+require 'securerandom'
 require 'sidekiq/exception_handler'
 require 'sidekiq/core_ext'
 
@@ -14,10 +16,17 @@ module Sidekiq
     def watchdog(last_words)
       yield
     rescue Exception => ex
-      handle_exception(ex, { :context => last_words })
+      handle_exception(ex, { context: last_words })
       raise ex
     end
 
+    def safe_thread(name, &block)
+      Thread.new do
+        Thread.current['sidekiq_label'] = name
+        watchdog(name, &block)
+      end
+    end
+
     def logger
       Sidekiq.logger
     end
@@ -26,12 +35,29 @@ module Sidekiq
       Sidekiq.redis(&block)
     end
 
-    def process_id
-      @@process_id ||= SecureRandom.hex
+    def hostname
+      ENV['DYNO'] || Socket.gethostname
     end
 
-    def hostname
-      Socket.gethostname
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def identity
+      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
+    end
+
+    def fire_event(event, reverse=false)
+      arr = Sidekiq.options[:lifecycle_events][event]
+      arr.reverse! if reverse
+      arr.each do |block|
+        begin
+          block.call
+        rescue => ex
+          handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
+        end
+      end
+      arr.clear
     end
   end
 end

data/lib/sidekiq/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
-  VERSION = "2.15.1"
+  VERSION = "4.2.10"
 end

data/lib/sidekiq/web.rb

@@ -1,340 +1,213 @@
+# frozen_string_literal: true
 require 'erb'
-require 'yaml'
-require 'sinatra/base'
 
 require 'sidekiq'
 require 'sidekiq/api'
 require 'sidekiq/paginator'
+require 'sidekiq/web/helpers'
 
-module Sidekiq
-  class Web < Sinatra::Base
-    include Sidekiq::Paginator
-
-    set :root, File.expand_path(File.dirname(__FILE__) + "/../../web")
-    set :public_folder, Proc.new { "#{root}/assets" }
-    set :views, Proc.new { "#{root}/views" }
-    set :locales, Proc.new { "#{root}/locales" }
-
-    helpers do
-      def strings
-        @strings ||= begin
-          Dir["#{settings.locales}/*.yml"].inject({}) do |memo, file|
-            memo.merge(YAML.load(File.open(file)))
-          end
-        end
-      end
-
-      def locale
-        lang = (request.env["HTTP_ACCEPT_LANGUAGE"] || 'en')[0,2]
-        strings[lang] ? lang : 'en'
-      end
-
-      def get_locale
-        strings[locale]
-      end
-
-      def t(msg, options={})
-        string = get_locale[msg] || msg
-        string % options
-      end
-
-      def reset_worker_list
-        Sidekiq.redis do |conn|
-          workers = conn.smembers('workers')
-          conn.srem('workers', workers) if !workers.empty?
-        end
-      end
-
-      def workers_size
-        @workers_size ||= Sidekiq.redis do |conn|
-          conn.scard('workers')
-        end
-      end
+require 'sidekiq/web/router'
+require 'sidekiq/web/action'
+require 'sidekiq/web/application'
 
-      def workers
-        @workers ||= begin
-          to_rem = []
-          workers = Sidekiq.redis do |conn|
-            conn.smembers('workers').map do |w|
-              msg = conn.get("worker:#{w}")
-              msg ? [w, Sidekiq.load_json(msg)] : (to_rem << w; nil)
-            end.compact.sort { |x| x[1] ? -1 : 1 }
-          end
-
-          # Detect and clear out any orphaned worker records.
-          # These can be left in Redis if Sidekiq crashes hard
-          # while processing jobs.
-          if to_rem.size > 0
-            Sidekiq.redis { |conn| conn.srem('workers', to_rem) }
-          end
-          workers
-        end
-      end
-
-      def stats
-        @stats ||= Sidekiq::Stats.new
-      end
-
-      def retries_with_score(score)
-        Sidekiq.redis do |conn|
-          results = conn.zrangebyscore('retry', score, score)
-          results.map { |msg| Sidekiq.load_json(msg) }
-        end
-      end
+require 'rack/protection'
 
-      def location
-        Sidekiq.redis { |conn| conn.client.location }
-      end
+require 'rack/builder'
+require 'rack/file'
+require 'rack/session/cookie'
 
-      def namespace
-        @@ns ||= Sidekiq.redis {|conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
-      end
+module Sidekiq
+  class Web
+    ROOT = File.expand_path("#{File.dirname(__FILE__)}/../../web")
+    VIEWS = "#{ROOT}/views".freeze
+    LOCALES = ["#{ROOT}/locales".freeze]
+    LAYOUT = "#{VIEWS}/layout.erb".freeze
+    ASSETS = "#{ROOT}/assets".freeze
 
-      def root_path
-        "#{env['SCRIPT_NAME']}/"
-      end
+    DEFAULT_TABS = {
+      "Dashboard" => '',
+      "Busy"      => 'busy',
+      "Queues"    => 'queues',
+      "Retries"   => 'retries',
+      "Scheduled" => 'scheduled',
+      "Dead"      => 'morgue',
+    }
 
-      def current_path
-        @current_path ||= request.path_info.gsub(/^\//,'')
+    class << self
+      def settings
+        self
       end
 
-      def current_status
-        return 'idle' if workers_size == 0
-        return 'active'
+      def middlewares
+        @middlewares ||= []
       end
 
-      def relative_time(time)
-        %{<time datetime="#{time.getutc.iso8601}">#{time}</time>}
+      def use(*middleware_args, &block)
+        middlewares << [middleware_args, block]
       end
 
-      def job_params(job, score)
-        "#{score}-#{job['jid']}"
+      def default_tabs
+        DEFAULT_TABS
       end
 
-      def parse_params(params)
-        score, jid = params.split("-")
-        [score.to_f, jid]
+      def custom_tabs
+        @custom_tabs ||= {}
       end
+      alias_method :tabs, :custom_tabs
 
-      def truncate(text, truncate_after_chars = 2000)
-        truncate_after_chars && text.size > truncate_after_chars ? "#{text[0..truncate_after_chars]}..." : text
+      def locales
+        @locales ||= LOCALES
       end
 
-      def display_args(args, truncate_after_chars = 2000)
-        args.map do |arg|
-          a = arg.inspect
-          truncate(a)
-        end.join(", ")
+      def views
+        @views ||= VIEWS
       end
 
-      RETRY_JOB_KEYS = Set.new(%w(
-        queue class args retry_count retried_at failed_at
-        jid error_message error_class backtrace
-        error_backtrace enqueued_at retry
-      ))
-
-      def retry_extra_items(retry_job)
-        @retry_extra_items ||= {}.tap do |extra|
-          retry_job.item.each do |key, value|
-            extra[key] = value unless RETRY_JOB_KEYS.include?(key)
-          end
-        end
+      def enable(*opts)
+        opts.each {|key| set(key, true) }
       end
 
-      def tabs
-        @tabs ||= {
-          "Dashboard" => '',
-          "Workers"   => 'workers',
-          "Queues"    => 'queues',
-          "Retries"   => 'retries',
-          "Scheduled" => 'scheduled',
-        }
+      def disable(*opts)
+        opts.each {|key| set(key, false) }
       end
 
-      def custom_tabs
-        self.class.tabs
+      # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
+      def set(attribute, value)
+        send(:"#{attribute}=", value)
       end
 
-      def number_with_delimiter(number)
-        begin
-          Float(number)
-        rescue ArgumentError, TypeError
-          return number
-        end
-
-        options = {:delimiter => ',', :separator => '.'}
-        parts = number.to_s.to_str.split('.')
-        parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
-        parts.join(options[:separator])
-      end
+      attr_accessor :app_url, :session_secret, :redis_pool, :sessions
+      attr_writer :locales, :views
+    end
 
-      def redis_keys
-        ["redis_stats", "uptime_in_days", "connected_clients", "used_memory_human", "used_memory_peak_human"]
-      end
+    def self.inherited(child)
+      child.app_url = self.app_url
+      child.session_secret = self.session_secret
+      child.redis_pool = self.redis_pool
+      child.sessions = self.sessions
+    end
 
-      def h(text)
-        ERB::Util.h(text)
-      end
+    def settings
+      self.class.settings
     end
 
-    get "/workers" do
-      erb :index
+    def use(*middleware_args, &block)
+      middlewares << [middleware_args, block]
     end
 
-    get "/queues" do
-      @queues = Sidekiq::Queue.all
-      erb :queues
+    def middlewares
+      @middlewares ||= Web.middlewares.dup
     end
 
-    get "/queues/:name" do
-      halt 404 unless params[:name]
-      @count = (params[:count] || 25).to_i
-      @name = params[:name]
-      (@current_page, @total_size, @messages) = page("queue:#{@name}", params[:page], @count)
-      @messages = @messages.map {|msg| Sidekiq.load_json(msg) }
-      erb :queue
+    def call(env)
+      app.call(env)
     end
 
-    post "/reset" do
-      reset_worker_list
-      redirect root_path
+    def self.call(env)
+      @app ||= new
+      @app.call(env)
     end
 
-    post "/queues/:name" do
-      Sidekiq::Queue.new(params[:name]).clear
-      redirect "#{root_path}queues"
+    def app
+      @app ||= build
     end
 
-    post "/queues/:name/delete" do
-      Sidekiq::Job.new(params[:key_val], params[:name]).delete
-      redirect "#{root_path}queues/#{params[:name]}"
+    def enable(*opts)
+      opts.each {|key| set(key, true) }
     end
 
-    get '/retries' do
-      @count = (params[:count] || 25).to_i
-      (@current_page, @total_size, @retries) = page("retry", params[:page], @count)
-      @retries = @retries.map {|msg, score| [Sidekiq.load_json(msg), score] }
-      erb :retries
+    def disable(*opts)
+      opts.each {|key| set(key, false) }
     end
 
-    get "/retries/:key" do
-      halt 404 unless params['key']
-      @retry = Sidekiq::RetrySet.new.fetch(*parse_params(params['key'])).first
-      redirect "#{root_path}retries" if @retry.nil?
-      erb :retry
+    def set(attribute, value)
+      send(:"#{attribute}=", value)
     end
 
-    post '/retries' do
-      halt 404 unless params['key']
+    # Default values
+    set :sessions, true
 
-      params['key'].each do |key|
-        job = Sidekiq::RetrySet.new.fetch(*parse_params(key)).first
-        next unless job
-        if params['retry']
-          job.retry
-        elsif params['delete']
-          job.delete
-        end
+    attr_writer :sessions
+
+    def sessions
+      unless instance_variable_defined?("@sessions")
+        @sessions = self.class.sessions
+        @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
       end
-      redirect "#{root_path}retries"
-    end
 
-    post "/retries/all/delete" do
-      Sidekiq::RetrySet.new.clear
-      redirect "#{root_path}retries"
+      @sessions
     end
 
-    post "/retries/all/retry" do
-      Sidekiq::RetrySet.new.retry_all
-      redirect "#{root_path}retries"
+    def self.register(extension)
+      extension.registered(WebApplication)
     end
 
-    post "/retries/:key" do
-      halt 404 unless params['key']
-      job = Sidekiq::RetrySet.new.fetch(*parse_params(params['key'])).first
-      if job
-        if params['retry']
-          job.retry
-        elsif params['delete']
-          job.delete
-        end
+    private
+
+    def using?(middleware)
+      middlewares.any? do |(m,_)|
+        m.kind_of?(Array) && (m[0] == middleware || m[0].kind_of?(middleware))
       end
-      redirect "#{root_path}retries"
     end
 
-    get '/scheduled' do
-      @count = (params[:count] || 25).to_i
-      (@current_page, @total_size, @scheduled) = page("schedule", params[:page], @count)
-      @scheduled = @scheduled.map {|msg, score| [Sidekiq.load_json(msg), score] }
-      erb :scheduled
-    end
+    def build_sessions
+      middlewares = self.middlewares
 
-    get "/scheduled/:key" do
-      halt 404 unless params['key']
-      @job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
-      redirect "#{root_path}scheduled" if @job.nil?
-      erb :scheduled_job_info
-    end
+      unless using?(::Rack::Protection) || ENV['RACK_ENV'] == 'test'
+        middlewares.unshift [[::Rack::Protection, { use: :authenticity_token }], nil]
+      end
 
-    post '/scheduled' do
-      halt 404 unless params['key']
+      s = sessions
+      return unless s
 
-      params['key'].each do |key|
-        job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
-        if job
-          if params['delete']
-            job.delete
-          elsif params['add_to_queue']
-            job.add_to_queue
-          end
+      unless using? ::Rack::Session::Cookie
+        unless secret = Web.session_secret
+          require 'securerandom'
+          secret = SecureRandom.hex(64)
         end
+
+        options = { secret: secret }
+        options = options.merge(s.to_hash) if s.respond_to? :to_hash
+
+        middlewares.unshift [[::Rack::Session::Cookie, options], nil]
       end
-      redirect "#{root_path}scheduled"
     end
 
-    post "/scheduled/:key" do
-      halt 404 unless params['key']
-      job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
-      if job
-        if params['add_to_queue']
-          job.add_to_queue
-        elsif params['delete']
-          job.delete
+    def build
+      build_sessions
+
+      middlewares = self.middlewares
+      klass = self.class
+
+      ::Rack::Builder.new do
+        %w(stylesheets javascripts images).each do |asset_dir|
+          map "/#{asset_dir}" do
+            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", { 'Cache-Control' => 'public, max-age=86400' })
+          end
         end
+
+        middlewares.each {|middleware, block| use(*middleware, &block) }
+
+        run WebApplication.new(klass)
       end
-      redirect "#{root_path}scheduled"
     end
+  end
 
-    get '/' do
-      @redis_info = Sidekiq.redis { |conn| conn.info }.select{ |k, v| redis_keys.include? k }
-      stats_history = Sidekiq::Stats::History.new((params[:days] || 30).to_i)
-      @processed_history = stats_history.processed
-      @failed_history = stats_history.failed
-      erb :dashboard
-    end
+  Sidekiq::WebApplication.helpers WebHelpers
+  Sidekiq::WebApplication.helpers Sidekiq::Paginator
 
-    get '/dashboard/stats' do
-      sidekiq_stats = Sidekiq::Stats.new
-      redis_stats   = Sidekiq.redis { |conn| conn.info }.select{ |k, v| redis_keys.include? k }
-
-      content_type :json
-      Sidekiq.dump_json({
-        sidekiq: {
-          processed:  sidekiq_stats.processed,
-          failed:     sidekiq_stats.failed,
-          busy:       workers_size,
-          enqueued:   sidekiq_stats.enqueued,
-          scheduled:  sidekiq_stats.scheduled_size,
-          retries:    sidekiq_stats.retry_size,
-        },
-        redis: redis_stats
-      })
-    end
+  Sidekiq::WebAction.class_eval "def _render\n#{ERB.new(File.read(Web::LAYOUT)).src}\nend"
+end
 
-    def self.tabs
-      @custom_tabs ||= {}
+if defined?(::ActionDispatch::Request::Session) &&
+    !::ActionDispatch::Request::Session.method_defined?(:each)
+  # mperham/sidekiq#2460
+  # Rack apps can't reuse the Rails session store without
+  # this monkeypatch, fixed in Rails 5.
+  class ActionDispatch::Request::Session
+    def each(&block)
+      hash = self.to_hash
+      hash.each(&block)
     end
-
   end
-
 end