sidekiq 5.0.0

data/lib/sidekiq/testing/inline.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+require 'sidekiq/testing'
+
+##
+# The Sidekiq inline infrastructure overrides perform_async so that it
+# actually calls perform instead. This allows workers to be run inline in a
+# testing environment.
+#
+# This is similar to `Resque.inline = true` functionality.
+#
+# Example:
+#
+#   require 'sidekiq/testing/inline'
+#
+#   $external_variable = 0
+#
+#   class ExternalWorker
+#     include Sidekiq::Worker
+#
+#     def perform
+#       $external_variable = 1
+#     end
+#   end
+#
+#   assert_equal 0, $external_variable
+#   ExternalWorker.perform_async
+#   assert_equal 1, $external_variable
+#
+Sidekiq::Testing.inline!

data/lib/sidekiq/util.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+require 'socket'
+require 'securerandom'
+require 'sidekiq/exception_handler'
+require 'sidekiq/core_ext'
+
+module Sidekiq
+  ##
+  # This module is part of Sidekiq core and not intended for extensions.
+  #
+  module Util
+    include ExceptionHandler
+
+    EXPIRY = 60 * 60 * 24
+
+    def watchdog(last_words)
+      yield
+    rescue Exception => ex
+      handle_exception(ex, { context: last_words })
+      raise ex
+    end
+
+    def safe_thread(name, &block)
+      Thread.new do
+        Thread.current['sidekiq_label'] = name
+        watchdog(name, &block)
+      end
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+
+    def redis(&block)
+      Sidekiq.redis(&block)
+    end
+
+    def hostname
+      ENV['DYNO'] || Socket.gethostname
+    end
+
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def identity
+      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
+    end
+
+    def fire_event(event, reverse=false)
+      arr = Sidekiq.options[:lifecycle_events][event]
+      arr.reverse! if reverse
+      arr.each do |block|
+        begin
+          block.call
+        rescue => ex
+          handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
+        end
+      end
+      arr.clear
+    end
+  end
+end

data/lib/sidekiq/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module Sidekiq
+  VERSION = "5.0.0"
+end

data/lib/sidekiq/web.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+require 'erb'
+
+require 'sidekiq'
+require 'sidekiq/api'
+require 'sidekiq/paginator'
+require 'sidekiq/web/helpers'
+
+require 'sidekiq/web/router'
+require 'sidekiq/web/action'
+require 'sidekiq/web/application'
+
+require 'rack/protection'
+
+require 'rack/builder'
+require 'rack/file'
+require 'rack/session/cookie'
+
+module Sidekiq
+  class Web
+    ROOT = File.expand_path("#{File.dirname(__FILE__)}/../../web")
+    VIEWS = "#{ROOT}/views".freeze
+    LOCALES = ["#{ROOT}/locales".freeze]
+    LAYOUT = "#{VIEWS}/layout.erb".freeze
+    ASSETS = "#{ROOT}/assets".freeze
+
+    DEFAULT_TABS = {
+      "Dashboard" => '',
+      "Busy"      => 'busy',
+      "Queues"    => 'queues',
+      "Retries"   => 'retries',
+      "Scheduled" => 'scheduled',
+      "Dead"      => 'morgue',
+    }
+
+    class << self
+      def settings
+        self
+      end
+
+      def middlewares
+        @middlewares ||= []
+      end
+
+      def use(*middleware_args, &block)
+        middlewares << [middleware_args, block]
+      end
+
+      def default_tabs
+        DEFAULT_TABS
+      end
+
+      def custom_tabs
+        @custom_tabs ||= {}
+      end
+      alias_method :tabs, :custom_tabs
+
+      def locales
+        @locales ||= LOCALES
+      end
+
+      def views
+        @views ||= VIEWS
+      end
+
+      def enable(*opts)
+        opts.each {|key| set(key, true) }
+      end
+
+      def disable(*opts)
+        opts.each {|key| set(key, false) }
+      end
+
+      # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
+      def set(attribute, value)
+        send(:"#{attribute}=", value)
+      end
+
+      attr_accessor :app_url, :session_secret, :redis_pool, :sessions
+      attr_writer :locales, :views
+    end
+
+    def self.inherited(child)
+      child.app_url = self.app_url
+      child.session_secret = self.session_secret
+      child.redis_pool = self.redis_pool
+      child.sessions = self.sessions
+    end
+
+    def settings
+      self.class.settings
+    end
+
+    def use(*middleware_args, &block)
+      middlewares << [middleware_args, block]
+    end
+
+    def middlewares
+      @middlewares ||= Web.middlewares.dup
+    end
+
+    def call(env)
+      app.call(env)
+    end
+
+    def self.call(env)
+      @app ||= new
+      @app.call(env)
+    end
+
+    def app
+      @app ||= build
+    end
+
+    def enable(*opts)
+      opts.each {|key| set(key, true) }
+    end
+
+    def disable(*opts)
+      opts.each {|key| set(key, false) }
+    end
+
+    def set(attribute, value)
+      send(:"#{attribute}=", value)
+    end
+
+    # Default values
+    set :sessions, true
+
+    attr_writer :sessions
+
+    def sessions
+      unless instance_variable_defined?("@sessions")
+        @sessions = self.class.sessions
+        @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
+      end
+
+      @sessions
+    end
+
+    def self.register(extension)
+      extension.registered(WebApplication)
+    end
+
+    private
+
+    def using?(middleware)
+      middlewares.any? do |(m,_)|
+        m.kind_of?(Array) && (m[0] == middleware || m[0].kind_of?(middleware))
+      end
+    end
+
+    def build_sessions
+      middlewares = self.middlewares
+
+      unless using?(::Rack::Protection) || ENV['RACK_ENV'] == 'test'
+        middlewares.unshift [[::Rack::Protection, { use: :authenticity_token }], nil]
+      end
+
+      s = sessions
+      return unless s
+
+      unless using? ::Rack::Session::Cookie
+        unless secret = Web.session_secret
+          require 'securerandom'
+          secret = SecureRandom.hex(64)
+        end
+
+        options = { secret: secret }
+        options = options.merge(s.to_hash) if s.respond_to? :to_hash
+
+        middlewares.unshift [[::Rack::Session::Cookie, options], nil]
+      end
+    end
+
+    def build
+      build_sessions
+
+      middlewares = self.middlewares
+      klass = self.class
+
+      ::Rack::Builder.new do
+        %w(stylesheets javascripts images).each do |asset_dir|
+          map "/#{asset_dir}" do
+            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", { 'Cache-Control' => 'public, max-age=86400' })
+          end
+        end
+
+        middlewares.each {|middleware, block| use(*middleware, &block) }
+
+        run WebApplication.new(klass)
+      end
+    end
+  end
+
+  Sidekiq::WebApplication.helpers WebHelpers
+  Sidekiq::WebApplication.helpers Sidekiq::Paginator
+
+  Sidekiq::WebAction.class_eval "def _render\n#{ERB.new(File.read(Web::LAYOUT)).src}\nend"
+end
+
+if defined?(::ActionDispatch::Request::Session) &&
+    !::ActionDispatch::Request::Session.method_defined?(:each)
+  # mperham/sidekiq#2460
+  # Rack apps can't reuse the Rails session store without
+  # this monkeypatch, fixed in Rails 5.
+  class ActionDispatch::Request::Session
+    def each(&block)
+      hash = self.to_hash
+      hash.each(&block)
+    end
+  end
+end

data/lib/sidekiq/web/action.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class WebAction
+    RACK_SESSION = 'rack.session'.freeze
+
+    attr_accessor :env, :block, :type
+
+    def settings
+      Web.settings
+    end
+
+    def request
+      @request ||= ::Rack::Request.new(env)
+    end
+
+    def halt(res)
+      throw :halt, res
+    end
+
+    def redirect(location)
+      throw :halt, [302, { "Location" => "#{request.base_url}#{location}" }, []]
+    end
+
+    def params
+      indifferent_hash = Hash.new {|hash,key| hash[key.to_s] if Symbol === key }
+
+      indifferent_hash.merge! request.params
+      route_params.each {|k,v| indifferent_hash[k.to_s] = v }
+
+      indifferent_hash
+    end
+
+    def route_params
+      env[WebRouter::ROUTE_PARAMS]
+    end
+
+    def session
+      env[RACK_SESSION]
+    end
+
+    def erb(content, options = {})
+      if content.kind_of? Symbol
+        unless respond_to?(:"_erb_#{content}")
+          src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
+          WebAction.class_eval("def _erb_#{content}\n#{src}\n end")
+        end
+      end
+
+      if @_erb
+        _erb(content, options[:locals])
+      else
+        @_erb = true
+        content = _erb(content, options[:locals])
+
+        _render { content }
+      end
+    end
+
+    def render(engine, content, options = {})
+      raise "Only erb templates are supported" if engine != :erb
+
+      erb(content, options)
+    end
+
+    def json(payload)
+      [200, { "Content-Type" => "application/json", "Cache-Control" => "no-cache" }, [Sidekiq.dump_json(payload)]]
+    end
+
+    def initialize(env, block)
+      @_erb = false
+      @env = env
+      @block = block
+      @@files ||= {}
+    end
+
+    private
+
+    def _erb(file, locals)
+      locals.each {|k, v| define_singleton_method(k){ v } } if locals
+
+      if file.kind_of?(String)
+        ERB.new(file).result(binding)
+      else
+        send(:"_erb_#{file}")
+      end
+    end
+  end
+end

data/lib/sidekiq/web/application.rb

@@ -0,0 +1,331 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class WebApplication
+    extend WebRouter
+
+    CONTENT_LENGTH = "Content-Length".freeze
+    CONTENT_TYPE = "Content-Type".freeze
+    REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
+
+    def initialize(klass)
+      @klass = klass
+    end
+
+    def settings
+      @klass.settings
+    end
+
+    def self.settings
+      Sidekiq::Web.settings
+    end
+
+    def self.tabs
+      Sidekiq::Web.tabs
+    end
+
+    def self.set(key, val)
+      # nothing, backwards compatibility
+    end
+
+    get "/" do
+      @redis_info = redis_info.select{ |k, v| REDIS_KEYS.include? k }
+      stats_history = Sidekiq::Stats::History.new((params['days'] || 30).to_i)
+      @processed_history = stats_history.processed
+      @failed_history = stats_history.failed
+
+      erb(:dashboard)
+    end
+
+    get "/busy" do
+      erb(:busy)
+    end
+
+    post "/busy" do
+      if params['identity']
+        p = Sidekiq::Process.new('identity' => params['identity'])
+        p.quiet! if params['quiet']
+        p.stop! if params['stop']
+      else
+        processes.each do |pro|
+          pro.quiet! if params['quiet']
+          pro.stop! if params['stop']
+        end
+      end
+
+      redirect "#{root_path}busy"
+    end
+
+    get "/queues" do
+      @queues = Sidekiq::Queue.all
+
+      erb(:queues)
+    end
+
+    get "/queues/:name" do
+      @name = route_params[:name]
+
+      halt(404) unless @name
+
+      @count = (params['count'] || 25).to_i
+      @queue = Sidekiq::Queue.new(@name)
+      (@current_page, @total_size, @messages) = page("queue:#{@name}", params['page'], @count)
+      @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
+
+      erb(:queue)
+    end
+
+    post "/queues/:name" do
+      Sidekiq::Queue.new(route_params[:name]).clear
+
+      redirect "#{root_path}queues"
+    end
+
+    post "/queues/:name/delete" do
+      name = route_params[:name]
+      Sidekiq::Job.new(params['key_val'], name).delete
+
+      redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
+    end
+
+    get '/morgue' do
+      @count = (params['count'] || 25).to_i
+      (@current_page, @total_size, @dead) = page("dead", params['page'], @count, reverse: true)
+      @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+
+      erb(:morgue)
+    end
+
+    get "/morgue/:key" do
+      halt(404) unless key = route_params[:key]
+
+      @dead = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
+
+      if @dead.nil?
+        redirect "#{root_path}morgue"
+      else
+        erb(:dead)
+      end
+    end
+
+    post '/morgue' do
+      redirect(request.path) unless params['key']
+
+      params['key'].each do |key|
+        job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
+        retry_or_delete_or_kill job, params if job
+      end
+
+      redirect_with_query("#{root_path}morgue")
+    end
+
+    post "/morgue/all/delete" do
+      Sidekiq::DeadSet.new.clear
+
+      redirect "#{root_path}morgue"
+    end
+
+    post "/morgue/all/retry" do
+      Sidekiq::DeadSet.new.retry_all
+
+      redirect "#{root_path}morgue"
+    end
+
+    post "/morgue/:key" do
+      halt(404) unless key = route_params[:key]
+
+      job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
+      retry_or_delete_or_kill job, params if job
+
+      redirect_with_query("#{root_path}morgue")
+    end
+
+    get '/retries' do
+      @count = (params['count'] || 25).to_i
+      (@current_page, @total_size, @retries) = page("retry", params['page'], @count)
+      @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+
+      erb(:retries)
+    end
+
+    get "/retries/:key" do
+      @retry = Sidekiq::RetrySet.new.fetch(*parse_params(route_params[:key])).first
+
+      if @retry.nil?
+        redirect "#{root_path}retries"
+      else
+        erb(:retry)
+      end
+    end
+
+    post '/retries' do
+      redirect(request.path) unless params['key']
+
+      params['key'].each do |key|
+        job = Sidekiq::RetrySet.new.fetch(*parse_params(key)).first
+        retry_or_delete_or_kill job, params if job
+      end
+
+      redirect_with_query("#{root_path}retries")
+    end
+
+    post "/retries/all/delete" do
+      Sidekiq::RetrySet.new.clear
+
+      redirect "#{root_path}retries"
+    end
+
+    post "/retries/all/retry" do
+      Sidekiq::RetrySet.new.retry_all
+
+      redirect "#{root_path}retries"
+    end
+
+    post "/retries/:key" do
+      job = Sidekiq::RetrySet.new.fetch(*parse_params(route_params[:key])).first
+
+      retry_or_delete_or_kill job, params if job
+
+      redirect_with_query("#{root_path}retries")
+    end
+
+    get '/scheduled' do
+      @count = (params['count'] || 25).to_i
+      (@current_page, @total_size, @scheduled) = page("schedule", params['page'], @count)
+      @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+
+      erb(:scheduled)
+    end
+
+    get "/scheduled/:key" do
+      @job = Sidekiq::ScheduledSet.new.fetch(*parse_params(route_params[:key])).first
+
+      if @job.nil?
+        redirect "#{root_path}scheduled"
+      else
+        erb(:scheduled_job_info)
+      end
+    end
+
+    post '/scheduled' do
+      redirect(request.path) unless params['key']
+
+      params['key'].each do |key|
+        job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
+        delete_or_add_queue job, params if job
+      end
+
+      redirect_with_query("#{root_path}scheduled")
+    end
+
+    post "/scheduled/:key" do
+      halt(404) unless key = route_params[:key]
+
+      job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
+      delete_or_add_queue job, params if job
+
+      redirect_with_query("#{root_path}scheduled")
+    end
+
+    get '/dashboard/stats' do
+      redirect "#{root_path}stats"
+    end
+
+    get '/stats' do
+      sidekiq_stats = Sidekiq::Stats.new
+      redis_stats   = redis_info.select { |k, v| REDIS_KEYS.include? k }
+
+      json(
+        sidekiq: {
+          processed:       sidekiq_stats.processed,
+          failed:          sidekiq_stats.failed,
+          busy:            sidekiq_stats.workers_size,
+          processes:       sidekiq_stats.processes_size,
+          enqueued:        sidekiq_stats.enqueued,
+          scheduled:       sidekiq_stats.scheduled_size,
+          retries:         sidekiq_stats.retry_size,
+          dead:            sidekiq_stats.dead_size,
+          default_latency: sidekiq_stats.default_queue_latency
+        },
+        redis: redis_stats
+      )
+    end
+
+    get '/stats/queues' do
+      json Sidekiq::Stats::Queues.new.lengths
+    end
+
+    def call(env)
+      action = self.class.match(env)
+      return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass" }, ["Not Found"]] unless action
+
+      resp = catch(:halt) do
+        app = @klass
+        self.class.run_befores(app, action)
+        begin
+          resp = action.instance_exec env, &action.block
+        ensure
+          self.class.run_afters(app, action)
+        end
+
+        resp
+      end
+
+      resp = case resp
+      when Array
+        resp
+      else
+        headers = {
+          "Content-Type" => "text/html",
+          "Cache-Control" => "no-cache",
+          "Content-Language" => action.locale,
+        }
+
+        [200, headers, [resp]]
+      end
+
+      resp[1] = resp[1].dup
+
+      resp[1][CONTENT_LENGTH] = resp[2].inject(0) { |l, p| l + p.bytesize }.to_s
+
+      resp
+    end
+
+    def self.helpers(mod=nil, &block)
+      if block_given?
+        WebAction.class_eval(&block)
+      else
+        WebAction.send(:include, mod)
+      end
+    end
+
+    def self.before(path=nil, &block)
+      befores << [path && Regexp.new("\\A#{path.gsub("*", ".*")}\\z"), block]
+    end
+
+    def self.after(path=nil, &block)
+      afters << [path && Regexp.new("\\A#{path.gsub("*", ".*")}\\z"), block]
+    end
+
+    def self.run_befores(app, action)
+      run_hooks(befores, app, action)
+    end
+
+    def self.run_afters(app, action)
+      run_hooks(afters, app, action)
+    end
+
+    def self.run_hooks(hooks, app, action)
+      hooks.select { |p,_| !p || p =~ action.env[WebRouter::PATH_INFO] }.
+            each {|_,b| action.instance_exec(action.env, app, &b) }
+    end
+
+    def self.befores
+      @befores ||= []
+    end
+
+    def self.afters
+      @afters ||= []
+    end
+  end
+end