sidekiq 6.5.10 → 8.1.7

data/lib/sidekiq/web/config.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class Web
+    ##
+    # Configure the Sidekiq::Web instance in this process:
+    #
+    #   require "sidekiq/web"
+    #   Sidekiq::Web.configure do |config|
+    #     config.register(MyExtension, name: "myext", tab: "TabName", index: "tabpage/")
+    #   end
+    #
+    # This should go in your `config/routes.rb` or similar. It
+    # does not belong in your initializer since Web should not be
+    # loaded in some processes (like an actual Sidekiq process).
+    # See `examples/webui-ext` for a sample web extension.
+    class Config
+      extend Forwardable
+
+      OPTIONS = {
+        # By default we support direct uploads to p.f.c since the UI is a JS SPA
+        # and very difficult for us to vendor or provide ourselves. If you are worried
+        # about data security and wish to self-host, you can change these URLs.
+        profile_view_url: "https://profiler.firefox.com/public/%s",
+        profile_store_url: "https://api.profiler.firefox.com/compressed-store"
+      }
+
+      ##
+      # Allows users to add custom rows to all of the Job
+      # tables, e.g. Retries, Dead, Scheduled, with custom
+      # links to other systems, see _job_info.erb and test
+      # in web_test.rb
+      #
+      #   Sidekiq::Web.configure do |cfg|
+      #     cfg.custom_job_info_rows << JobLogLink.new
+      #   end
+      #
+      #   class JobLogLink
+      #     def add_pair(job)
+      #       yield "External Logs", "<a href='https://example.com/logs/#{job.jid}'>Logs for #{job.jid}</a>"
+      #     end
+      #   end
+      attr_accessor :custom_job_info_rows
+
+      attr_reader :tabs
+      attr_reader :locales
+      attr_reader :views
+      attr_reader :middlewares
+
+      # Adds the "Back to App" link in the header
+      attr_accessor :app_url
+      attr_accessor :assets_path
+
+      def initialize
+        @options = OPTIONS.dup
+        @locales = LOCALES
+        @views = VIEWS
+        @assets_path = ASSETS
+        @tabs = DEFAULT_TABS.dup
+        @middlewares = []
+        @custom_job_info_rows = []
+      end
+
+      def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
+
+      def use(*args, &block)
+        middlewares << [args, block]
+      end
+
+      # Register a class as a Sidekiq Web UI extension. The class should
+      # provide one or more tabs which map to an index route. Options:
+      #
+      # @param extclass [Class] Class which contains the HTTP actions, required
+      # @param name [String] the name of the extension, used to namespace assets
+      # @param tab [String | Array] labels(s) of the UI tabs
+      # @param index [String | Array] index route(s) for each tab
+      # @param root_dir [String] directory location to find assets, locales and views, typically `web/` within the gemfile
+      # @param asset_paths [Array] one or more directories under {root}/assets/{name} to be publicly served, e.g. ["js", "css", "img"]
+      # @param cache_for [Integer] amount of time to cache assets, default one day
+      #
+      # Web extensions will have a root `web/` directory with `locales/`, `assets/`
+      # and `views/` subdirectories.
+      def register_extension(extclass, name:, tab:, index:, root_dir: nil, cache_for: 86400, asset_paths: nil)
+        tab = Array(tab)
+        index = Array(index)
+        tab.zip(index).each do |tab, index|
+          tabs[tab] = index
+        end
+        if root_dir
+          locdir = File.join(root_dir, "locales")
+          locales << locdir if File.directory?(locdir)
+
+          if asset_paths && name
+            # if you have {root}/assets/{name}/js/scripts.js
+            # and {root}/assets/{name}/css/styles.css
+            # you would pass in:
+            #   asset_paths: ["js", "css"]
+            # See script_tag and style_tag in web/helpers.rb
+            assdir = File.join(root_dir, "assets")
+            assurls = Array(asset_paths).map { |x| "/#{name}/#{x}" }
+            assetprops = {
+              urls: assurls,
+              root: assdir,
+              cascade: true
+            }
+            assetprops[:header_rules] = [[:all, {"cache-control" => "private, max-age=#{cache_for.to_i}"}]] if cache_for
+            middlewares << [[Rack::Static, assetprops], nil]
+          end
+        end
+
+        yield self if block_given?
+        extclass.registered(Web::Application)
+      end
+      alias_method :register, :register_extension
+    end
+  end
+end

data/lib/sidekiq/web/helpers.rb

@@ -1,26 +1,112 @@
 # frozen_string_literal: true
 
 require "uri"
-require "set"
-require "yaml"
-require "cgi"
+require "cgi/escape"
 
 module Sidekiq
-  # This is not a public API
+  # These methods are available to pages within the Web UI and UI extensions.
+  # They are not public APIs for applications to use.
   module WebHelpers
+    def store_name
+      hash = redis_info
+      return "Dragonfly" if hash.has_key?("dragonfly_version")
+      return "Valkey" if hash.has_key?("valkey_version")
+      "Redis"
+    end
+
+    def store_version
+      hash = redis_info
+      return hash["dragonfly_version"] if hash.has_key?("dragonfly_version")
+      return hash["valkey_version"] if hash.has_key?("valkey_version")
+      hash["redis_version"]
+    end
+
+    def style_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/css",
+        media: "screen",
+        rel: "stylesheet",
+        nonce: csp_nonce,
+        href: location
+      }
+      add_to_head do
+        html_tag(:link, attrs.merge(kwargs))
+      end
+    end
+
+    def script_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/javascript",
+        nonce: csp_nonce,
+        src: location
+      }
+      html_tag(:script, attrs.merge(kwargs)) {}
+    end
+
+    # NB: keys and values are not escaped; do not allow user input
+    # in the attributes
+    private def html_tag(tagname, attrs)
+      s = "<#{tagname}"
+      attrs.each_pair do |k, v|
+        next unless v
+        s << " #{k}=\"#{v}\""
+      end
+      if block_given?
+        s << ">"
+        yield s
+        s << "</#{tagname}>"
+      else
+        s << " />"
+      end
+      s
+    end
+
     def strings(lang)
-      @strings ||= {}
+      @@strings ||= {}
 
       # Allow sidekiq-web extensions to add locale paths
       # so extensions can be localized
-      @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
+      @@strings[lang] ||= config.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
-          strs = YAML.safe_load(File.open(file))
+          strs = parse_yaml(file)
           global.merge!(strs[lang])
         end
       end
     end
 
+    def parse_yaml(path)
+      locale = nil
+      map = {}
+      IO.readlines(path, chomp: true).each do |line|
+        case line
+        when /\A\s*\#.*/
+          # line comment
+        when !locale && /\A([a-zA-Z\-_]+):/
+          locale = $1
+          map[locale] = {}
+        when /\A\s+(\w+):\s+(.+)\z/
+          # A few values have double quotes to include special characters in YAML.
+          # Strip them off manually as our greedy match will include them.
+          key = $1
+          s = $2
+          s = s[1..] if s[0] == "\""
+          s = s[0..-2] if s[-1] == "\""
+          map[locale][key] = s
+        else
+          raise ArgumentError, "unable to parse #{path}: #{line}"
+        end
+      end
+      map
+    end
+
+    def to_json(x)
+      Sidekiq.dump_json(x)
+    end
+
     def singularize(str, count)
       if count == 1 && str.respond_to?(:singularize) # rails
         str.singularize
@@ -30,27 +116,53 @@ module Sidekiq
     end
 
     def clear_caches
-      @strings = nil
-      @locale_files = nil
-      @available_locales = nil
+      @@strings = nil
+      @@locale_files = nil
+      @@available_locales = nil
     end
 
     def locale_files
-      @locale_files ||= settings.locales.flat_map { |path|
+      @@locale_files ||= config.locales.flat_map { |path|
         Dir["#{path}/*.yml"]
       }
     end
 
     def available_locales
-      @available_locales ||= locale_files.map { |path| File.basename(path, ".yml") }.uniq
+      @@available_locales ||= Set.new(locale_files.map { |path| File.basename(path, ".yml") })
     end
 
     def find_locale_files(lang)
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
 
-    # This is a hook for a Sidekiq Pro feature.  Please don't touch.
-    def filtering(*)
+    def language_name(locale)
+      strings(locale).fetch("LanguageName", locale)
+    end
+
+    def search(jobset, substr)
+      resultset = jobset.scan(substr).to_a
+      @current_page = 1
+      @count = @total_size = resultset.size
+      resultset
+    end
+
+    def filtering(which, placeholder_key: "AnyJobContent", label_key: "Filter")
+      erb(:filtering, locals: {which:, placeholder_key:, label_key:})
+    end
+
+    def filter_link(str, within = "retries")
+      hstr = h(str)
+      if within.nil?
+        hstr
+      else
+        "<a href='#{root_path}#{within}?substr=#{hstr}'>#{hstr}</a>"
+      end
+    end
+
+    def display_tags(job, within = "retries")
+      job.tags.map { |tag|
+        "<span class='label label-info jobtag jobtag-#{h(tag)}'>#{filter_link(tag, within)}</span>"
+      }.join(" ")
     end
 
     # This view helper provide ability display you html code in
@@ -78,10 +190,13 @@ module Sidekiq
       text_direction == "rtl"
     end
 
-    # See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
+    # See https://www.rfc-editor.org/rfc/rfc9110.html#section-12.5.4
+    # Returns an array of language tags ordered by their quality value
+    #
+    # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def user_preferred_languages
       languages = env["HTTP_ACCEPT_LANGUAGE"]
-      languages.to_s.downcase.gsub(/\s+/, "").split(",").map { |language|
+      languages.to_s.gsub(/\s+/, "").split(",").map { |language|
         locale, quality = language.split(";q=", 2)
         locale = nil if locale == "*" # Ignore wildcards
         quality = quality ? quality.to_f : 1.0
@@ -93,34 +208,39 @@ module Sidekiq
 
     # Given an Accept-Language header like "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2"
     # this method will try to best match the available locales to the user's preferred languages.
-    #
-    # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
-      @locale ||= begin
-        matched_locale = user_preferred_languages.map { |preferred|
-          preferred_language = preferred.split("-", 2).first
+      # session[:locale] is set via the locale selector from the footer
+      @locale ||= if (l = session&.fetch(:locale, nil)) && available_locales.include?(l)
+        l
+      else
+        matched_locale = nil
+        # Attempt to find a case-insensitive exact match first
+        user_preferred_languages.each do |preferred|
+          # We only care about the language and primary subtag
+          # "en-GB-oxendict" becomes "en-GB"
+          language_tag = preferred.split("-")[0..1].join("-")
+          matched_locale = available_locales.find { |available_locale| available_locale.casecmp?(language_tag) }
+          break if matched_locale
+        end
 
-          lang_group = available_locales.select { |available|
-            preferred_language == available.split("-", 2).first
-          }
+        return matched_locale if matched_locale
 
-          lang_group.find { |lang| lang == preferred } || lang_group.min_by(&:length)
-        }.compact.first
+        # Find the first base language match
+        # "en-US,es-MX;q=0.9" matches "en"
+        user_preferred_languages.each do |preferred|
+          base_language = preferred.split("-", 2).first
+          matched_locale = available_locales.find { |available_locale| available_locale.casecmp?(base_language) }
+          break if matched_locale
+        end
 
         matched_locale || "en"
       end
     end
 
-    # within is used by Sidekiq Pro
-    def display_tags(job, within = nil)
-      job.tags.map { |tag|
-        "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
-      }.join(" ")
-    end
-
-    # mperham/sidekiq#3243
+    # sidekiq/sidekiq#3243
     def unfiltered?
-      yield unless env["PATH_INFO"].start_with?("/filter/")
+      s = url_params("substr")
+      yield unless s && s.size > 0
     end
 
     def get_locale
@@ -137,7 +257,7 @@ module Sidekiq
     end
 
     def sort_direction_label
-      (params[:direction] == "asc") ? "&uarr;" : "&darr;"
+      (url_params("direction") == "asc") ? "&uarr;" : "&darr;"
     end
 
     def workset
@@ -165,18 +285,14 @@ module Sidekiq
       @stats ||= Sidekiq::Stats.new
     end
 
-    def redis_connection
+    def redis_url
       Sidekiq.redis do |conn|
-        conn.connection[:id]
+        conn.config.server_url
       end
     end
 
-    def namespace
-      @ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
-    end
-
     def redis_info
-      Sidekiq.redis_info
+      @info ||= Sidekiq.default_configuration.redis_info
     end
 
     def root_path
@@ -196,26 +312,37 @@ module Sidekiq
       %(<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>)
     end
 
+    def queue_names_by_capsule(pro)
+      cap = pro.capsules
+      if cap
+        cap.map { |k, v| v["weights"].keys.join(", ") }.join("; ")
+      else
+        # DEPRECATED Backwards compatibility with older processes.
+        # 'capsules' element added in v8.0.9
+        pro.queues.join(", ")
+      end
+    end
+
     def job_params(job, score)
       "#{score}-#{job["jid"]}"
     end
 
-    def parse_params(params)
-      score, jid = params.split("-", 2)
+    def parse_key(key)
+      score, jid = key.split("-", 2)
       [score.to_f, jid]
     end
 
-    SAFE_QPARAMS = %w[page direction]
+    SAFE_QPARAMS = %w[page direction only]
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)
       stringified_options = options.transform_keys(&:to_s)
 
-      to_query_string(params.merge(stringified_options))
+      to_query_string(request.params.merge(stringified_options))
     end
 
-    def to_query_string(params)
-      params.map { |key, value|
+    def to_query_string(hash)
+      hash.map { |key, value|
         SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
       }.compact.join("&")
     end
@@ -238,7 +365,11 @@ module Sidekiq
     end
 
     def csrf_tag
-      "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
+      ""
+    end
+
+    def csp_nonce
+      env[:csp_nonce]
     end
 
     def to_display(arg)
@@ -274,27 +405,17 @@ module Sidekiq
       elsif rss_kb < 10_000_000
         "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
       else
-        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
+        "#{number_with_delimiter(rss_kb / (1024.0 * 1024.0), precision: 1)} GB"
       end
     end
 
-    def number_with_delimiter(number)
-      return "" if number.nil?
-
-      begin
-        Float(number)
-      rescue ArgumentError, TypeError
-        return number
-      end
-
-      options = {delimiter: ",", separator: "."}
-      parts = number.to_s.to_str.split(".")
-      parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
-      parts.join(options[:separator])
+    def number_with_delimiter(number, options = {})
+      precision = options[:precision] || 0
+      %(<span data-nwp="#{precision}">#{number.round(precision)}</span>)
     end
 
     def h(text)
-      ::Rack::Utils.escape_html(text)
+      ::CGI.escapeHTML(text.to_s)
     rescue ArgumentError => e
       raise unless e.message.eql?("invalid byte sequence in UTF-8")
       text.encode!("UTF-16", "UTF-8", invalid: :replace, replace: "").encode!("UTF-8", "UTF-16")
@@ -314,7 +435,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq.default_configuration[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end
@@ -327,11 +448,9 @@ module Sidekiq
       Time.now.utc.strftime("%H:%M:%S UTC")
     end
 
-    def redis_connection_and_namespace
-      @redis_connection_and_namespace ||= begin
-        namespace_suffix = namespace.nil? ? "" : "##{namespace}"
-        "#{redis_connection}#{namespace_suffix}"
-      end
+    def pollable?
+      # there's no point to refreshing the metrics pages every N seconds
+      !(current_path == "" || current_path.index("metrics"))
     end
 
     def retry_or_delete_or_kill(job, params)

data/lib/sidekiq/web/router.rb

@@ -1,103 +1,90 @@
 # frozen_string_literal: true
 
-require "rack"
+require "uri"
 
 module Sidekiq
-  module WebRouter
-    GET = "GET"
-    DELETE = "DELETE"
-    POST = "POST"
-    PUT = "PUT"
-    PATCH = "PATCH"
-    HEAD = "HEAD"
-
-    ROUTE_PARAMS = "rack.route_params"
-    REQUEST_METHOD = "REQUEST_METHOD"
-    PATH_INFO = "PATH_INFO"
-
-    def head(path, &block)
-      route(HEAD, path, &block)
-    end
+  class Web
+    # Provides an API to declare endpoints, along with a match
+    # API to dynamically route a request to an endpoint.
+    module Router
+      def head(path, &) = route(:head, path, &)
 
-    def get(path, &block)
-      route(GET, path, &block)
-    end
+      def get(path, &) = route(:get, path, &)
 
-    def post(path, &block)
-      route(POST, path, &block)
-    end
+      def post(path, &) = route(:post, path, &)
 
-    def put(path, &block)
-      route(PUT, path, &block)
-    end
+      def put(path, &) = route(:put, path, &)
 
-    def patch(path, &block)
-      route(PATCH, path, &block)
-    end
+      def patch(path, &) = route(:patch, path, &)
 
-    def delete(path, &block)
-      route(DELETE, path, &block)
-    end
+      def delete(path, &) = route(:delete, path, &)
 
-    def route(method, path, &block)
-      @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
-
-      @routes[method] << WebRoute.new(method, path, block)
-    end
-
-    def match(env)
-      request_method = env[REQUEST_METHOD]
-      path_info = ::Rack::Utils.unescape env[PATH_INFO]
+      def route(*methods, path, &block)
+        methods.each do |method|
+          raise ArgumentError, "Invalid method #{method}. Must be one of #{@routes.keys.join(",")}" unless route_cache.has_key?(method)
+          route_cache[method] << Route.new(method, path, block)
+        end
+      end
 
-      # There are servers which send an empty string when requesting the root.
-      # These servers should be ashamed of themselves.
-      path_info = "/" if path_info == ""
+      def match(env)
+        request_method = env["REQUEST_METHOD"].downcase.to_sym
+        path_info = ::URI::RFC2396_PARSER.unescape env["PATH_INFO"]
 
-      @routes[request_method].each do |route|
-        params = route.match(request_method, path_info)
-        if params
-          env[ROUTE_PARAMS] = params
+        # There are servers which send an empty string when requesting the root.
+        # These servers should be ashamed of themselves.
+        path_info = "/" if path_info == ""
 
-          return WebAction.new(env, route.block)
+        route_cache[request_method].each do |route|
+          params = route.match(request_method, path_info)
+          if params
+            env["rack.route_params"] = params
+            return Action.new(env, route.block)
+          end
         end
+
+        nil
       end
 
-      nil
+      def route_cache
+        @@routes ||= {get: [], post: [], put: [], patch: [], delete: [], head: []}
+      end
     end
-  end
 
-  class WebRoute
-    attr_accessor :request_method, :pattern, :block, :name
+    class Route
+      attr_accessor :request_method, :pattern, :block, :name
 
-    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^.:$\/]+)/
+      NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^.:$\/]+)/
 
-    def initialize(request_method, pattern, block)
-      @request_method = request_method
-      @pattern = pattern
-      @block = block
-    end
+      def initialize(request_method, pattern, block)
+        @request_method = request_method
+        @pattern = pattern
+        @block = block
+      end
 
-    def matcher
-      @matcher ||= compile
-    end
+      def matcher
+        @matcher ||= compile
+      end
 
-    def compile
-      if pattern.match?(NAMED_SEGMENTS_PATTERN)
-        p = pattern.gsub(NAMED_SEGMENTS_PATTERN, '/\1(?<\2>[^$/]+)')
+      def compile
+        if pattern.match?(NAMED_SEGMENTS_PATTERN)
+          p = pattern.gsub(NAMED_SEGMENTS_PATTERN, '/\1(?<\2>[^$/]+)')
 
-        Regexp.new("\\A#{p}\\Z")
-      else
-        pattern
+          Regexp.new("\\A#{p}\\Z")
+        else
+          pattern
+        end
       end
-    end
 
-    def match(request_method, path)
-      case matcher
-      when String
-        {} if path == matcher
-      else
-        path_match = path.match(matcher)
-        path_match&.named_captures&.transform_keys(&:to_sym)
+      EMPTY = {}.freeze
+
+      def match(request_method, path)
+        case matcher
+        when String
+          EMPTY if path == matcher
+        else
+          path_match = path.match(matcher)
+          path_match&.named_captures&.transform_keys(&:to_sym)
+        end
       end
     end
   end