sidekiq 8.0.10 → 8.1.3

data/lib/sidekiq/tui.rb

@@ -0,0 +1,380 @@
+# https://sr.ht/~kerrick/ratatui_ruby/
+# https://git.sr.ht/~kerrick/ratatui_ruby/tree/stable/item/examples/
+gem "ratatui_ruby", ">=1.4.0"
+require "ratatui_ruby"
+
+RatatuiRuby.debug_mode! if !!ENV["DEBUG"]
+
+require "sidekiq/api"
+require "sidekiq/paginator"
+
+require_relative "tui/filtering"
+require_relative "tui/controls"
+require_relative "tui/tabs"
+
+module Sidekiq
+  class TUI
+    include Sidekiq::Component
+
+    PageOptions = Data.define(:page, :size)
+
+    REFRESH_INTERVAL_SECONDS = 2
+    LOCALE_DIRECTORIES = [File.expand_path("#{File.dirname(__FILE__)}/../../web/locales")]
+
+    # language is meant to be a locale code, e.g.
+    # LANG=en_US.utf-8
+    def initialize(cfg, language: ENV["LANG"] || "en")
+      @lang = language
+      @config = cfg
+      @base_style = nil
+      @last_refresh = Time.at(0)
+      @fps = Array.new(2) { 0 }
+      @previous_fps = 0
+      @showing = :main
+    end
+
+    def prepare(tui)
+      load_locale
+
+      @tui = tui
+      @highlight_style = @tui.style(fg: :light_red, modifiers: [:underlined])
+      @hotkey_style = @tui.style(modifiers: [:bold, :underlined])
+      # eager load tabs
+      all
+    end
+
+    def run_loop
+      # Must log to a file, terminal is now controlled by Ratatui
+      config.logger = Logger.new("tui.log")
+
+      loop do
+        refresh_data if should_refresh?
+        render
+        break if handle_input == :quit
+      end
+    end
+
+    def render
+      track_fps do
+        if @showing == :main
+          @tui.draw do |frame|
+            main_area, controls_area = @tui.layout_split(
+              frame.area,
+              direction: :vertical,
+              constraints: [
+                @tui.constraint_fill(1),
+                @tui.constraint_length(5)
+              ]
+            )
+
+            # Split main area into tabs and content
+            tabs_area, content_area = @tui.layout_split(
+              main_area,
+              direction: :vertical,
+              constraints: [
+                @tui.constraint_length(3),
+                @tui.constraint_fill(1)
+              ]
+            )
+
+            all_tabs = all
+            tabs = @tui.tabs(
+              titles: all_tabs.map { |tab| t(tab.name) },
+              selected_index: all_tabs.index(current_tab),
+              block: @tui.block(title: " #{Sidekiq::NAME}", borders: [:all], title_style: @tui.style(fg: :light_red, modifiers: [:bold])),
+              divider: " | ",
+              highlight_style: @highlight_style,
+              style: @base_style
+            )
+            frame.render_widget(tabs, tabs_area)
+
+            render_content_area(frame, content_area)
+            render_controls(frame, controls_area)
+          end
+        end
+
+        if @showing == :help
+          @tui.draw do |frame|
+            main_area, controls_area = @tui.layout_split(
+              frame.area,
+              direction: :vertical,
+              constraints: [
+                @tui.constraint_fill(1),
+                @tui.constraint_length(4)
+              ]
+            )
+            content = @tui.block(
+              title: " #{Sidekiq::NAME} ",
+              borders: [:all],
+              title_style: @tui.style(fg: :light_red, modifiers: [:bold]),
+              children: [
+                # TODO convert to table
+                @tui.paragraph(
+                  text: [
+                    @tui.text_line(spans: ["Welcome to the Sidekiq Terminal UI"], alignment: :center),
+                    @tui.text_line(spans: [
+                      @tui.text_span(content: "Global hotkeys")
+                    ]),
+                    @tui.text_line(spans: []),
+                    @tui.text_line(spans: [
+                      @tui.text_span(content: "Esc", style: @hotkey_style),
+                      @tui.text_span(content: ": Close this window")
+                    ]),
+                    @tui.text_line(spans: [
+                      @tui.text_span(content: "←/→", style: @hotkey_style),
+                      @tui.text_span(content: ": Move between tabs")
+                    ]),
+                    @tui.text_line(spans: [
+                      @tui.text_span(content: "h/l", style: @hotkey_style),
+                      @tui.text_span(content: ": Move to prev/next page of data")
+                    ]),
+                    @tui.text_line(spans: [
+                      @tui.text_span(content: "j/k", style: @hotkey_style),
+                      @tui.text_span(content: ": Move to prev/next row in current page")
+                    ]),
+                    @tui.text_line(spans: [
+                      @tui.text_span(content: "x", style: @hotkey_style),
+                      @tui.text_span(content: ": Select/deselect current row")
+                    ]),
+                    @tui.text_line(spans: [
+                      @tui.text_span(content: "A", style: @hotkey_style),
+                      @tui.text_span(content: ": Select/deselect All rows in current page")
+                    ]),
+                    @tui.text_line(spans: [
+                      @tui.text_span(content: "q", style: @hotkey_style),
+                      @tui.text_span(content: ": Quit")
+                    ])
+                  ]
+                )
+              ]
+            )
+            frame.render_widget(content, main_area)
+            controls = @tui.block(
+              title: t("Controls"),
+              borders: [:all],
+              children: [
+                @tui.paragraph(
+                  text: [
+                    @tui.text_line(spans: [
+                      @tui.text_span(content: "Esc", style: @hotkey_style),
+                      @tui.text_span(content: ": Close  ")
+                    ])
+                  ]
+                )
+              ]
+            )
+            frame.render_widget(controls, controls_area)
+          end
+        end
+      end
+    end
+
+    def render_content_area(frame, content_area)
+      return render_error(frame, content_area, current_tab.error) if current_tab.error
+
+      current_tab.render(@tui, frame, content_area)
+    end
+
+    def render_controls(frame, area)
+      active_keys = current_tab.controls.filter { |hash| hash[:description] }
+
+      # Split controls into two lines, 8 is arbitrary
+      # TODO Dynamically split based on term width?
+      first = active_keys[...8]
+      lines = []
+      lines << @tui.text_line(spans: first.map { |hash|
+        [
+          @tui.text_span(content: hash[:display] || hash[:code], style: @hotkey_style),
+          @tui.text_span(content: ": #{t(hash[:description])}  ")
+        ]
+      }.flatten)
+
+      last = active_keys[8...]
+      lines << if last && last.size > 0
+        @tui.text_line(spans: last.map { |hash|
+          [
+            @tui.text_span(content: hash[:display] || hash[:code], style: @hotkey_style),
+            @tui.text_span(content: ": #{t(hash[:description])}  ")
+          ]
+        }.flatten)
+      else
+        @tui.text_line(spans: [])
+      end
+
+      footer = [
+        @tui.text_span(content: "Redis: #{redis_url}    "),
+        @tui.text_span(content: "#{t("Now")}: #{Time.now.utc}    "),
+        @tui.text_span(content: "#{t("Locale")}: #{@lang}")
+      ]
+
+      if current_tab.data[:filter]
+        @filter_style = @tui.style(fg: :white, bg: :dark_gray)
+        footer += [
+          @tui.text_span(content: "   #{t("Filter")}: ", style: @filter_style),
+          @tui.text_span(content: current_tab.data[:filter], style: @filter_style),
+          @tui.text_span(content: "_", style: @tui.style(fg: :white, bg: :dark_gray, modifiers: [:slow_blink]))
+        ]
+      end
+      footer << @tui.text_span(content: "  FPS: #{previous_fps}") if debugging?
+      lines << @tui.text_line(spans: footer)
+
+      controls = @tui.block(title: t("Controls"), borders: [:all],
+        children: [@tui.paragraph(text: lines)])
+      frame.render_widget(controls, area)
+    end
+
+    def handle_input
+      # We shouldn't need more than 10 FPS for a data-oriented app.
+      # This throttles down our CPU usage. Default is 60 FPS.
+      case @tui.poll_event(timeout: 0.1)
+      in {type: :key, code: "esc"} if @showing == :help
+        @showing = :main
+      in {type: :key, code: code} if current_tab.filtering? && code.length == 1
+        current_tab.append_to_filter(code)
+        current_tab.refresh_data
+      in {type: :key, code:, modifiers:}
+        control = current_tab.controls.find { |ctrl|
+          ctrl[:code] == code &&
+            (ctrl[:modifiers] || []) == (modifiers || [])
+        }
+        return unless control
+        control[:action].call(self, current_tab).tap {
+          refresh_data if control[:refresh]
+        }
+      else
+        # Ignore other events
+      end
+    rescue => ex
+      logger.error { [ex.message, ex.backtrace] }
+    end
+
+    def redis_url
+      Sidekiq.redis do |conn|
+        conn.config.server_url
+      end
+    rescue
+      "N/A"
+    end
+
+    def should_refresh?
+      Time.now - @last_refresh >= REFRESH_INTERVAL_SECONDS
+    end
+
+    def refresh_data
+      # logger.info GC.stat
+      current_tab.refresh_data
+      @last_refresh = Time.now
+    rescue => e
+      handle_exception(e)
+      current_tab.error = e
+    end
+
+    def render_error(frame, area, err)
+      header = [@tui.text_line(
+        spans: [@tui.text_span(content: err.message, style: @tui.style(modifiers: [:bold]))],
+        alignment: :center
+      )]
+      lines = Array(err.backtrace).map { |line| @tui.text_line(spans: [@tui.text_span(content: line)]) }
+
+      frame.render_widget(
+        @tui.paragraph(
+          text: header + lines,
+          alignment: :left,
+          block: @tui.block(title: t("Error"), borders: [:all], border_style: @tui.style(fg: :light_red))
+        ),
+        area
+      )
+    end
+
+    def show_help
+      @showing = :help
+    end
+
+    def all
+      @all ||= Tabs::All.map { |kls| kls.new(self) }
+    end
+
+    def current_tab
+      @current ||= @all.first
+    end
+
+    # Navigate tabs to the left or right.
+    # @param direction [Symbol] :left or :right
+    def navigate(direction)
+      index_change = (direction == :right) ? 1 : -1
+      @current = @all[(@all.index(current_tab) + index_change) % @all.size]
+      @current.reset_data
+    end
+
+    public def t(msg, options = nil)
+      string = @strings[msg] || msg
+      if options.nil?
+        string
+      else
+        string % options
+      end
+    end
+
+    def load_strings(lang)
+      {}.tap do |all|
+        find_locale_files(lang).each do |file|
+          strs = YAML.safe_load_file(file)
+          all.merge! strs[lang]
+        end
+      end
+    end
+
+    def locale_files
+      @@locale_files ||= LOCALE_DIRECTORIES.flat_map { |path|
+        Dir["#{path}/*.yml"]
+      }
+    end
+
+    def available_locales
+      @@available_locales ||= Set.new(locale_files.map { |path| File.basename(path, ".yml") })
+    end
+
+    def find_locale_files(lang)
+      locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
+    end
+
+    def load_locale
+      require "yaml"
+      lang = @lang.split(".").first # "en_US"
+      while lang.size > 0
+        hash = load_strings(lang)
+        if hash.size > 0
+          # found a working language dataset
+          @lang = lang
+          @strings = hash
+          Sidekiq.logger.debug { "using the #{lang} locale" }
+          break
+        end
+        # Try "en_US", "en_U", "en_", "en"
+        # It's ugly and bruteforce but it works
+        lang = lang[..-2]
+      end
+    end
+
+    def track_fps
+      # We hold two fps buckets: one for current second, one for previous second
+      idx = Time.now.to_i % 2
+      @fps[idx] += 1
+      yield
+    end
+
+    def previous_fps
+      curidx = Time.now.to_i % 2
+      prev = (curidx == 1) ? 0 : 1
+      if (val = @fps[prev]) != 0
+        @previous_fps = val
+        @fps[prev] = 0
+      end
+      @previous_fps
+    end
+
+    def debugging?
+      !!ENV["DEBUG"]
+    end
+  end
+end

data/lib/sidekiq/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "8.0.10"
+  VERSION = "8.1.3"
   MAJOR = 8
 
   def self.gem_version

data/lib/sidekiq/web/action.rb

@@ -117,7 +117,7 @@ module Sidekiq
         if content.is_a? Symbol
           unless respond_to?(:"_erb_#{content}")
             views = options[:views] || Web.views
-            filename = "#{views}/#{content}.erb"
+            filename = "#{views}/#{content}.html.erb"
             src = ERB.new(File.read(filename)).src
 
             # Need to use lineno less by 1 because erb generates a

data/lib/sidekiq/web/application.rb

@@ -99,8 +99,8 @@ module Sidekiq
         if url_params("identity")
           pro = Sidekiq::ProcessSet[url_params("identity")]
 
-          pro.quiet! if url_params("quiet")
-          pro.stop! if url_params("stop")
+          pro&.quiet! if url_params("quiet")
+          pro&.stop! if url_params("stop")
         else
           processes.each do |pro|
             next if pro.embedded?

data/lib/sidekiq/web/config.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "sidekiq/web/csrf_protection"
-
 module Sidekiq
   class Web
     ##
@@ -24,10 +22,7 @@ module Sidekiq
         # and very difficult for us to vendor or provide ourselves. If you are worried
         # about data security and wish to self-host, you can change these URLs.
         profile_view_url: "https://profiler.firefox.com/public/%s",
-        profile_store_url: "https://api.profiler.firefox.com/compressed-store",
-        # TODO Will be false in Sidekiq 9.0.
-        # CSRF is unnecessary if you are using SameSite=(Strict|Lax) cookies.
-        csrf: true
+        profile_store_url: "https://api.profiler.firefox.com/compressed-store"
       }
 
       ##
@@ -54,11 +49,13 @@ module Sidekiq
 
       # Adds the "Back to App" link in the header
       attr_accessor :app_url
+      attr_accessor :assets_path
 
       def initialize
         @options = OPTIONS.dup
         @locales = LOCALES
         @views = VIEWS
+        @assets_path = ASSETS
         @tabs = DEFAULT_TABS.dup
         @middlewares = []
         @custom_job_info_rows = []

data/lib/sidekiq/web/helpers.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "uri"
-require "yaml"
 require "cgi/escape"
 
 module Sidekiq
@@ -73,12 +72,42 @@ module Sidekiq
       # so extensions can be localized
       @@strings[lang] ||= config.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
-          strs = YAML.safe_load_file(file)
+          strs = parse_yaml_new(file)
           global.merge!(strs[lang])
         end
       end
     end
 
+    def parse_yaml_old(path)
+      require "yaml"
+      YAML.safe_load_file(path)
+    end
+
+    def parse_yaml_new(path)
+      locale = nil
+      map = {}
+      IO.readlines(path, chomp: true).each do |line|
+        case line
+        when /\A\s*\#.*/
+          # line comment
+        when !locale && /\A([a-zA-Z\-_]+):/
+          locale = $1
+          map[locale] = {}
+        when /\A\s+(\w+):\s+(.+)\z/
+          # A few values have double quotes to include special characters in YAML.
+          # Strip them off manually as our greedy match will include them.
+          key = $1
+          s = $2
+          s = s[1..] if s[0] == "\""
+          s = s[0..-2] if s[-1] == "\""
+          map[locale][key] = s
+        else
+          raise ArgumentError, "unable to parse #{path}: #{line}"
+        end
+      end
+      map
+    end
+
     def to_json(x)
       Sidekiq.dump_json(x)
     end
@@ -287,6 +316,17 @@ module Sidekiq
       %(<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>)
     end
 
+    def queue_names_by_capsule(pro)
+      cap = pro.capsules
+      if cap
+        cap.map { |k, v| v["weights"].keys.join(", ") }.join("; ")
+      else
+        # DEPRECATED Backwards compatibility with older processes.
+        # 'capsules' element added in v8.0.9
+        pro.queues.join(", ")
+      end
+    end
+
     def job_params(job, score)
       "#{score}-#{job["jid"]}"
     end
@@ -329,7 +369,7 @@ module Sidekiq
     end
 
     def csrf_tag
-      "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
+      ""
     end
 
     def csp_nonce

data/lib/sidekiq/web.rb

@@ -13,7 +13,7 @@ module Sidekiq
     ROOT = File.expand_path("#{File.dirname(__FILE__)}/../../web")
     VIEWS = "#{ROOT}/views"
     LOCALES = ["#{ROOT}/locales"]
-    LAYOUT = "#{VIEWS}/layout.erb"
+    LAYOUT = "#{VIEWS}/layout.html.erb"
     ASSETS = "#{ROOT}/assets"
 
     DEFAULT_TABS = {
@@ -42,6 +42,12 @@ module Sidekiq
         @@config.app_url = url
       end
 
+      def assets_path=(path)
+        @@config.assets_path = path
+      end
+
+      def assets_path = @@config.assets_path
+
       def tabs = @@config.tabs
 
       def locales = @@config.locales
@@ -87,13 +93,27 @@ module Sidekiq
       env[:web_config] = Sidekiq::Web.configure
       env[:csp_nonce] = SecureRandom.hex(8)
       env[:redis_pool] = self.class.redis_pool
-      app.call(env)
+      safe_request?(env) ? app.call(env) : deny(env)
     end
 
     def app
       @app ||= build(@@config)
     end
 
+    def safe_methods?(env)
+      %w[GET HEAD OPTIONS TRACE].include? env["REQUEST_METHOD"]
+    end
+
+    def safe_request?(env)
+      return true if safe_methods?(env)
+      env["HTTP_SEC_FETCH_SITE"] == "same-origin"
+    end
+
+    def deny(env)
+      Sidekiq.logger.warn "attack prevented by #{self.class}"
+      [403, {Rack::CONTENT_TYPE => "text/plain"}, ["Forbidden"]]
+    end
+
     private
 
     def build(cfg)
@@ -105,11 +125,10 @@ module Sidekiq
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
-          root: ASSETS,
+          root: cfg.assets_path,
           cascade: true,
           header_rules: rules
         m.each { |middleware, block| use(*middleware, &block) }
-        use CsrfProtection if cfg[:csrf]
         run Sidekiq::Web::Application.new(self.class)
       end
     end

data/lib/sidekiq.rb

@@ -47,6 +47,13 @@ module Sidekiq
     puts "Take a deep breath and count to ten..."
   end
 
+  def self.testing!(mode = :fake, &block)
+    raise "Unknown testing mode: #{mode}" unless %i[fake disable inline].include?(mode)
+
+    require "sidekiq/test_api"
+    Sidekiq::Testing.__set_test_mode(mode, &block)
+  end
+
   def self.server?
     defined?(Sidekiq::CLI)
   end

data/sidekiq.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |gem|
   gem.homepage = "https://sidekiq.org"
   gem.license = "LGPL-3.0"
 
-  gem.executables = ["sidekiq", "sidekiqmon"]
+  gem.executables = ["sidekiq", "sidekiqmon", "kiq"]
   gem.files = %w[sidekiq.gemspec README.md Changes.md LICENSE.txt] + `git ls-files | grep -E '^(bin|lib|web)'`.split("\n")
   gem.name = "sidekiq"
   gem.version = Sidekiq::VERSION
@@ -23,9 +23,9 @@ Gem::Specification.new do |gem|
     "rubygems_mfa_required" => "true"
   }
 
-  gem.add_dependency "redis-client", ">= 0.23.2"
-  gem.add_dependency "connection_pool", ">= 2.5.0"
-  gem.add_dependency "rack", ">= 3.1.0"
-  gem.add_dependency "json", ">= 2.9.0"
-  gem.add_dependency "logger", ">= 1.6.2"
+  gem.add_dependency "redis-client", ">= 0.26.0"
+  gem.add_dependency "connection_pool", ">= 3.0.0"
+  gem.add_dependency "rack", ">= 3.2.0"
+  gem.add_dependency "json", ">= 2.16.0"
+  gem.add_dependency "logger", ">= 1.7.0"
 end

data/web/assets/javascripts/application.js

@@ -154,7 +154,7 @@ function livePollCallback() {
 
 function checkResponse(resp) {
   if (!resp.ok) {
-    throw response.error();
+    throw resp.error();
   }
   return resp
 }

data/web/assets/stylesheets/style.css

@@ -448,9 +448,7 @@ canvas + .table_container {
 }
 
 .buttons-row {
-  margin-top: var(--space-3x);
   display: flex;
-  flex-wrap: wrap;
   gap: 8px;
 }
 
@@ -536,6 +534,8 @@ input[type="checkbox"] { accent-color: var(--color-primary); }
 
 .pagination a { text-decoration: none; }
 
+.pagination .active a { color: var(--color-primary); font-weight: 700; }
+
 .pagination .disabled { opacity: 0.3; }
 
 div:has(.pagination.pull-right) { margin-left: auto; }

data/web/locales/ar.yml

@@ -74,7 +74,7 @@ ar:
   Stop: إيقاف
   StopAll: إيقاف الكل
   StopPolling: إيقاف الاستعلامات
-  TextDirection: 'rtl'
+  TextDirection: rtl
   Thread: نيسب
   Threads: نياسب
   ThreeMonths: ثلاثة أشهر

data/web/locales/fa.yml

@@ -69,7 +69,7 @@ fa:
   Stop: توقف
   StopAll: توقف همه
   StopPolling: Stop Polling
-  TextDirection: 'rtl'
+  TextDirection: rtl
   Thread: رشته
   Threads: رشته ها
   ThreeMonths: ۳ ماه

data/web/locales/gd.yml

@@ -1,4 +1,4 @@
-# elements like %{queue} are variables and should not be translated
+# elements like %{queue} are variables and should not be translated
 gd:
   LanguageName: Gaeilge
   Actions: Gnìomhan

data/web/locales/he.yml

@@ -69,7 +69,7 @@ he:
   Stop: עצור
   StopAll: עצור הכל
   StopPolling: עצור תשאול
-  TextDirection: 'rtl'
+  TextDirection: rtl
   Thread: חוט
   Threads: חוטים
   ThreeMonths: 3 חדשים