sidekiq 3.5.4 → 5.2.7

data/lib/sidekiq/web/action.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class WebAction
+    RACK_SESSION = 'rack.session'
+
+    attr_accessor :env, :block, :type
+
+    def settings
+      Web.settings
+    end
+
+    def request
+      @request ||= ::Rack::Request.new(env)
+    end
+
+    def halt(res)
+      throw :halt, res
+    end
+
+    def redirect(location)
+      throw :halt, [302, { "Location" => "#{request.base_url}#{location}" }, []]
+    end
+
+    def params
+      indifferent_hash = Hash.new {|hash,key| hash[key.to_s] if Symbol === key }
+
+      indifferent_hash.merge! request.params
+      route_params.each {|k,v| indifferent_hash[k.to_s] = v }
+
+      indifferent_hash
+    end
+
+    def route_params
+      env[WebRouter::ROUTE_PARAMS]
+    end
+
+    def session
+      env[RACK_SESSION]
+    end
+
+    def erb(content, options = {})
+      if content.kind_of? Symbol
+        unless respond_to?(:"_erb_#{content}")
+          src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
+          WebAction.class_eval("def _erb_#{content}\n#{src}\n end")
+        end
+      end
+
+      if @_erb
+        _erb(content, options[:locals])
+      else
+        @_erb = true
+        content = _erb(content, options[:locals])
+
+        _render { content }
+      end
+    end
+
+    def render(engine, content, options = {})
+      raise "Only erb templates are supported" if engine != :erb
+
+      erb(content, options)
+    end
+
+    def json(payload)
+      [200, { "Content-Type" => "application/json", "Cache-Control" => "no-cache" }, [Sidekiq.dump_json(payload)]]
+    end
+
+    def initialize(env, block)
+      @_erb = false
+      @env = env
+      @block = block
+      @@files ||= {}
+    end
+
+    private
+
+    def _erb(file, locals)
+      locals.each {|k, v| define_singleton_method(k){ v } unless (singleton_methods.include? k)} if locals
+
+      if file.kind_of?(String)
+        ERB.new(file).result(binding)
+      else
+        send(:"_erb_#{file}")
+      end
+    end
+  end
+end

data/lib/sidekiq/web/application.rb

@@ -0,0 +1,353 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class WebApplication
+    extend WebRouter
+
+    CONTENT_LENGTH = "Content-Length"
+    CONTENT_TYPE = "Content-Type"
+    REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
+    CSP_HEADER = [
+      "default-src 'self' https: http:",
+      "child-src 'self'",
+      "connect-src 'self' https: http: wss: ws:",
+      "font-src 'self' https: http:",
+      "frame-src 'self'",
+      "img-src 'self' https: http: data:",
+      "manifest-src 'self'",
+      "media-src 'self'",
+      "object-src 'none'",
+      "script-src 'self' https: http: 'unsafe-inline'",
+      "style-src 'self' https: http: 'unsafe-inline'",
+      "worker-src 'self'",
+      "base-uri 'self'"
+    ].join('; ').freeze
+
+    def initialize(klass)
+      @klass = klass
+    end
+
+    def settings
+      @klass.settings
+    end
+
+    def self.settings
+      Sidekiq::Web.settings
+    end
+
+    def self.tabs
+      Sidekiq::Web.tabs
+    end
+
+    def self.set(key, val)
+      # nothing, backwards compatibility
+    end
+
+    get "/" do
+      @redis_info = redis_info.select{ |k, v| REDIS_KEYS.include? k }
+      stats_history = Sidekiq::Stats::History.new((params['days'] || 30).to_i)
+      @processed_history = stats_history.processed
+      @failed_history = stats_history.failed
+
+      erb(:dashboard)
+    end
+
+    get "/busy" do
+      erb(:busy)
+    end
+
+    post "/busy" do
+      if params['identity']
+        p = Sidekiq::Process.new('identity' => params['identity'])
+        p.quiet! if params['quiet']
+        p.stop! if params['stop']
+      else
+        processes.each do |pro|
+          pro.quiet! if params['quiet']
+          pro.stop! if params['stop']
+        end
+      end
+
+      redirect "#{root_path}busy"
+    end
+
+    get "/queues" do
+      @queues = Sidekiq::Queue.all
+
+      erb(:queues)
+    end
+
+    get "/queues/:name" do
+      @name = route_params[:name]
+
+      halt(404) unless @name
+
+      @count = (params['count'] || 25).to_i
+      @queue = Sidekiq::Queue.new(@name)
+      (@current_page, @total_size, @messages) = page("queue:#{@name}", params['page'], @count)
+      @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
+
+      erb(:queue)
+    end
+
+    post "/queues/:name" do
+      Sidekiq::Queue.new(route_params[:name]).clear
+
+      redirect "#{root_path}queues"
+    end
+
+    post "/queues/:name/delete" do
+      name = route_params[:name]
+      Sidekiq::Job.new(params['key_val'], name).delete
+
+      redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
+    end
+
+    get '/morgue' do
+      @count = (params['count'] || 25).to_i
+      (@current_page, @total_size, @dead) = page("dead", params['page'], @count, reverse: true)
+      @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+
+      erb(:morgue)
+    end
+
+    get "/morgue/:key" do
+      halt(404) unless key = route_params[:key]
+
+      @dead = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
+
+      if @dead.nil?
+        redirect "#{root_path}morgue"
+      else
+        erb(:dead)
+      end
+    end
+
+    post '/morgue' do
+      redirect(request.path) unless params['key']
+
+      params['key'].each do |key|
+        job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
+        retry_or_delete_or_kill job, params if job
+      end
+
+      redirect_with_query("#{root_path}morgue")
+    end
+
+    post "/morgue/all/delete" do
+      Sidekiq::DeadSet.new.clear
+
+      redirect "#{root_path}morgue"
+    end
+
+    post "/morgue/all/retry" do
+      Sidekiq::DeadSet.new.retry_all
+
+      redirect "#{root_path}morgue"
+    end
+
+    post "/morgue/:key" do
+      halt(404) unless key = route_params[:key]
+
+      job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
+      retry_or_delete_or_kill job, params if job
+
+      redirect_with_query("#{root_path}morgue")
+    end
+
+    get '/retries' do
+      @count = (params['count'] || 25).to_i
+      (@current_page, @total_size, @retries) = page("retry", params['page'], @count)
+      @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+
+      erb(:retries)
+    end
+
+    get "/retries/:key" do
+      @retry = Sidekiq::RetrySet.new.fetch(*parse_params(route_params[:key])).first
+
+      if @retry.nil?
+        redirect "#{root_path}retries"
+      else
+        erb(:retry)
+      end
+    end
+
+    post '/retries' do
+      redirect(request.path) unless params['key']
+
+      params['key'].each do |key|
+        job = Sidekiq::RetrySet.new.fetch(*parse_params(key)).first
+        retry_or_delete_or_kill job, params if job
+      end
+
+      redirect_with_query("#{root_path}retries")
+    end
+
+    post "/retries/all/delete" do
+      Sidekiq::RetrySet.new.clear
+
+      redirect "#{root_path}retries"
+    end
+
+    post "/retries/all/retry" do
+      Sidekiq::RetrySet.new.retry_all
+
+      redirect "#{root_path}retries"
+    end
+
+    post "/retries/all/kill" do
+      Sidekiq::RetrySet.new.kill_all
+
+      redirect "#{root_path}retries"
+    end
+
+    post "/retries/:key" do
+      job = Sidekiq::RetrySet.new.fetch(*parse_params(route_params[:key])).first
+
+      retry_or_delete_or_kill job, params if job
+
+      redirect_with_query("#{root_path}retries")
+    end
+
+    get '/scheduled' do
+      @count = (params['count'] || 25).to_i
+      (@current_page, @total_size, @scheduled) = page("schedule", params['page'], @count)
+      @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+
+      erb(:scheduled)
+    end
+
+    get "/scheduled/:key" do
+      @job = Sidekiq::ScheduledSet.new.fetch(*parse_params(route_params[:key])).first
+
+      if @job.nil?
+        redirect "#{root_path}scheduled"
+      else
+        erb(:scheduled_job_info)
+      end
+    end
+
+    post '/scheduled' do
+      redirect(request.path) unless params['key']
+
+      params['key'].each do |key|
+        job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
+        delete_or_add_queue job, params if job
+      end
+
+      redirect_with_query("#{root_path}scheduled")
+    end
+
+    post "/scheduled/:key" do
+      halt(404) unless key = route_params[:key]
+
+      job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
+      delete_or_add_queue job, params if job
+
+      redirect_with_query("#{root_path}scheduled")
+    end
+
+    get '/dashboard/stats' do
+      redirect "#{root_path}stats"
+    end
+
+    get '/stats' do
+      sidekiq_stats = Sidekiq::Stats.new
+      redis_stats   = redis_info.select { |k, v| REDIS_KEYS.include? k }
+      json(
+        sidekiq: {
+          processed:       sidekiq_stats.processed,
+          failed:          sidekiq_stats.failed,
+          busy:            sidekiq_stats.workers_size,
+          processes:       sidekiq_stats.processes_size,
+          enqueued:        sidekiq_stats.enqueued,
+          scheduled:       sidekiq_stats.scheduled_size,
+          retries:         sidekiq_stats.retry_size,
+          dead:            sidekiq_stats.dead_size,
+          default_latency: sidekiq_stats.default_queue_latency
+        },
+        redis: redis_stats,
+        server_utc_time: server_utc_time
+      )
+    end
+
+    get '/stats/queues' do
+      json Sidekiq::Stats::Queues.new.lengths
+    end
+
+    def call(env)
+      action = self.class.match(env)
+      return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass" }, ["Not Found"]] unless action
+
+      resp = catch(:halt) do
+        app = @klass
+        self.class.run_befores(app, action)
+        begin
+          resp = action.instance_exec env, &action.block
+        ensure
+          self.class.run_afters(app, action)
+        end
+
+        resp
+      end
+
+      resp = case resp
+      when Array
+        resp
+      else
+        headers = {
+          "Content-Type" => "text/html",
+          "Cache-Control" => "no-cache",
+          "Content-Language" => action.locale,
+          "Content-Security-Policy" => CSP_HEADER
+        }
+
+        [200, headers, [resp]]
+      end
+
+      resp[1] = resp[1].dup
+
+      resp[1][CONTENT_LENGTH] = resp[2].inject(0) { |l, p| l + p.bytesize }.to_s
+
+      resp
+    end
+
+    def self.helpers(mod=nil, &block)
+      if block_given?
+        WebAction.class_eval(&block)
+      else
+        WebAction.send(:include, mod)
+      end
+    end
+
+    def self.before(path=nil, &block)
+      befores << [path && Regexp.new("\\A#{path.gsub("*", ".*")}\\z"), block]
+    end
+
+    def self.after(path=nil, &block)
+      afters << [path && Regexp.new("\\A#{path.gsub("*", ".*")}\\z"), block]
+    end
+
+    def self.run_befores(app, action)
+      run_hooks(befores, app, action)
+    end
+
+    def self.run_afters(app, action)
+      run_hooks(afters, app, action)
+    end
+
+    def self.run_hooks(hooks, app, action)
+      hooks.select { |p,_| !p || p =~ action.env[WebRouter::PATH_INFO] }.
+            each {|_,b| action.instance_exec(action.env, app, &b) }
+    end
+
+    def self.befores
+      @befores ||= []
+    end
+
+    def self.afters
+      @afters ||= []
+    end
+  end
+end

data/lib/sidekiq/{web_helpers.rb → web/helpers.rb}

@@ -1,4 +1,8 @@
+# frozen_string_literal: true
 require 'uri'
+require 'set'
+require 'yaml'
+require 'cgi'
 
 module Sidekiq
   # This is not a public API
@@ -11,18 +15,28 @@ module Sidekiq
         settings.locales.each_with_object({}) do |path, global|
           find_locale_files(lang).each do |file|
             strs = YAML.load(File.open(file))
-            global.deep_merge!(strs[lang])
+            global.merge!(strs[lang])
           end
         end
       end
     end
 
+    def clear_caches
+      @@strings = nil
+      @@locale_files = nil
+      @@available_locales = nil
+    end
+
     def locale_files
-      @@locale_files = settings.locales.flat_map do |path|
+      @@locale_files ||= settings.locales.flat_map do |path|
         Dir["#{path}/*.yml"]
       end
     end
 
+    def available_locales
+      @@available_locales ||= locale_files.map { |path| File.basename(path, '.yml') }.uniq
+    end
+
     def find_locale_files(lang)
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
@@ -39,46 +53,75 @@ module Sidekiq
     #     <meta .../>
     #   <% end %>
     #
-    def add_to_head(&block)
+    def add_to_head
       @head_html ||= []
-      @head_html << block if block_given?
+      @head_html << yield.dup if block_given?
     end
 
     def display_custom_head
-      return unless defined?(@head_html)
-      @head_html.map { |block| capture(&block) }.join
+      @head_html.join if defined?(@head_html)
+    end
+
+    def poll_path
+      if current_path != '' && params['poll']
+        root_path + current_path
+      else
+        ""
+      end
+    end
+
+    def text_direction
+      get_locale['TextDirection'] || 'ltr'
+    end
+
+    def rtl?
+      text_direction == 'rtl'
     end
 
-    # Simple capture method for erb templates. The origin was
-    # capture method from sinatra-contrib library.
-    def capture(&block)
-      block.call
-      eval('', block.binding)
+    # See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
+    def user_preferred_languages
+      languages = env['HTTP_ACCEPT_LANGUAGE']
+      languages.to_s.downcase.gsub(/\s+/, '').split(',').map do |language|
+        locale, quality = language.split(';q=', 2)
+        locale  = nil if locale == '*' # Ignore wildcards
+        quality = quality ? quality.to_f : 1.0
+        [locale, quality]
+      end.sort do |(_, left), (_, right)|
+        right <=> left
+      end.map(&:first).compact
     end
 
-    # Given a browser request Accept-Language header like
-    # "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2", this function
-    # will return "fr" since that's the first code with a matching
-    # locale in web/locales
+    # Given an Accept-Language header like "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2"
+    # this method will try to best match the available locales to the user's preferred languages.
+    #
+    # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
       @locale ||= begin
-        locale = 'en'.freeze
-        languages = request.env['HTTP_ACCEPT_LANGUAGE'.freeze] || 'en'.freeze
-        languages.downcase.split(','.freeze).each do |lang|
-          next if lang == '*'.freeze
-          lang = lang.split(';'.freeze)[0]
-          break locale = lang if find_locale_files(lang).any?
-        end
-        locale
+        matched_locale = user_preferred_languages.map do |preferred|
+          preferred_language = preferred.split('-', 2).first
+
+          lang_group = available_locales.select do |available|
+            preferred_language == available.split('-', 2).first
+          end
+
+          lang_group.find { |lang| lang == preferred } || lang_group.min_by(&:length)
+        end.compact.first
+
+        matched_locale || 'en'
       end
     end
 
+    # mperham/sidekiq#3243
+    def unfiltered?
+      yield unless env['PATH_INFO'].start_with?("/filter/")
+    end
+
     def get_locale
       strings(locale)
     end
 
     def t(msg, options={})
-      string = get_locale[msg] || msg
+      string = get_locale[msg] || strings('en')[msg] || msg
       if options.empty?
         string
       else
@@ -104,28 +147,19 @@ module Sidekiq
       end.map { |msg| Sidekiq.load_json(msg) }
     end
 
-    def location
-      Sidekiq.redis { |conn| conn.client.location }
-    end
-
     def redis_connection
-      Sidekiq.redis { |conn| conn.client.id }
+      Sidekiq.redis do |conn|
+        c = conn.connection
+        "redis://#{c[:location]}/#{c[:db]}"
+      end
     end
 
     def namespace
-      @@ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
+      @ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
     end
 
     def redis_info
-      Sidekiq.redis do |conn|
-        # admin commands can't go through redis-namespace starting
-        # in redis-namespace 2.0
-        if conn.respond_to?(:namespace)
-          conn.redis.info
-        else
-          conn.info
-        end
-      end
+      Sidekiq.redis_info
     end
 
     def root_path
@@ -141,7 +175,8 @@ module Sidekiq
     end
 
     def relative_time(time)
-      %{<time datetime="#{time.getutc.iso8601}">#{time}</time>}
+      stamp = time.getutc.iso8601
+      %{<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>}
     end
 
     def job_params(job, score)
@@ -149,7 +184,7 @@ module Sidekiq
     end
 
     def parse_params(params)
-      score, jid = params.split("-")
+      score, jid = params.split("-", 2)
       [score.to_f, jid]
     end
 
@@ -157,10 +192,14 @@ module Sidekiq
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)
-      options = options.stringify_keys
+      # stringify
+      options.keys.each do |key|
+        options[key.to_s] = options.delete(key)
+      end
+
       params.merge(options).map do |key, value|
-        SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
-      end.join("&")
+        SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
+      end.compact.join("&")
     end
 
     def truncate(text, truncate_after_chars = 2000)
@@ -168,9 +207,16 @@ module Sidekiq
     end
 
     def display_args(args, truncate_after_chars = 2000)
-      args.map do |arg|
-        h(truncate(to_display(arg)))
-      end.join(", ")
+      return "Invalid job payload, args is nil" if args == nil
+      return "Invalid job payload, args must be an Array, not #{args.class.name}" if !args.is_a?(Array)
+
+      begin
+        args.map do |arg|
+          h(truncate(to_display(arg), truncate_after_chars))
+        end.join(", ")
+      rescue
+        "Illegal job arguments: #{h args.inspect}"
+      end
     end
 
     def csrf_tag
@@ -193,6 +239,7 @@ module Sidekiq
       queue class args retry_count retried_at failed_at
       jid error_message error_class backtrace
       error_backtrace enqueued_at retry wrapped
+      created_at
     ))
 
     def retry_extra_items(retry_job)
@@ -245,5 +292,34 @@ module Sidekiq
     def product_version
       "Sidekiq v#{Sidekiq::VERSION}"
     end
+
+    def server_utc_time
+      Time.now.utc.strftime('%H:%M:%S UTC')
+    end
+
+    def redis_connection_and_namespace
+      @redis_connection_and_namespace ||= begin
+        namespace_suffix = namespace == nil ? '' : "##{namespace}"
+        "#{redis_connection}#{namespace_suffix}"
+      end
+    end
+
+    def retry_or_delete_or_kill(job, params)
+      if params['retry']
+        job.retry
+      elsif params['delete']
+        job.delete
+      elsif params['kill']
+        job.kill
+      end
+    end
+
+    def delete_or_add_queue(job, params)
+      if params['delete']
+        job.delete
+      elsif params['add_to_queue']
+        job.add_to_queue
+      end
+    end
   end
 end