sidekiq 6.2.2 → 8.1.5

data/lib/sidekiq/transaction_aware_client.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "sidekiq/client"
+
+module Sidekiq
+  class TransactionAwareClient
+    def initialize(pool: nil, config: nil)
+      @redis_client = Client.new(pool: pool, config: config)
+      @transaction_backend =
+        if ActiveRecord.version >= Gem::Version.new("7.2")
+          ActiveRecord.method(:after_all_transactions_commit)
+        else
+          AfterCommitEverywhere.method(:after_commit)
+        end
+    end
+
+    def batching?
+      Thread.current[:sidekiq_batch]
+    end
+
+    def push(item)
+      # 6160 we can't support both Sidekiq::Batch and transactions.
+      return @redis_client.push(item) if batching?
+
+      # pre-allocate the JID so we can return it immediately and
+      # save it to the database as part of the transaction.
+      item["jid"] ||= SecureRandom.hex(12)
+      @transaction_backend.call { @redis_client.push(item) }
+      item["jid"]
+    end
+
+    ##
+    # We don't provide transactionality for push_bulk because we don't want
+    # to hold potentially hundreds of thousands of job records in memory due to
+    # a long running enqueue process.
+    def push_bulk(items)
+      @redis_client.push_bulk(items)
+    end
+  end
+end
+
+##
+# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
+module Sidekiq
+  def self.transactional_push!
+    if ActiveRecord.version < Gem::Version.new("7.2")
+      begin
+        require "after_commit_everywhere"
+      rescue LoadError
+        raise %q(You need ActiveRecord >= 7.2 or to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
+      end
+    end
+
+    Sidekiq.default_job_options["client_class"] = Sidekiq::TransactionAwareClient
+    Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
+    true
+  end
+end

data/lib/sidekiq/tui/controls.rb

@@ -0,0 +1,53 @@
+module Sidekiq
+  class TUI
+    module Controls
+      # Defines data for input handling and for displaying controls.
+      # :code is the key code for input handling.
+      # :display and :description are shown in the controls area, with different
+      #   styling between them. If :display is omitted, :code is displayed instead.
+      # :action is a lambda to execute when the control is triggered.
+      # :refresh means the action requires immediate refreshing of data
+      #
+      # Conventions: dangerous/irreversible actions should use UPPERCASE codes.
+      # The Shift button means "I'm sure".
+      GLOBAL = [
+        {code: "?", display: "?", description: "Help", action: ->(tui, tab) { tui.show_help }},
+        {code: "left", display: "←/→", description: "Select Tab", action: ->(tui, tab) { tui.navigate(:left) }, refresh: true},
+        {code: "right", action: ->(tui, tab) { tui.navigate(:right) }, refresh: true},
+        {code: "q", description: "Quit", action: ->(tui, tab) { :quit }},
+        {code: "c", modifiers: ["ctrl"], action: ->(tui, tab) { :quit }}
+      ].freeze
+
+      SHARED = {
+        pageable: [
+          {code: "h", display: "h/l", description: "Prev/Next Page",
+           action: ->(tui, tab) { tab.prev_page }, refresh: true},
+          {code: "l", action: ->(tui, tab) { tab.next_page }, refresh: true}
+        ],
+        selectable: [
+          {code: "k", display: "j/k", description: "Prev/Next Row",
+           action: ->(tui, tab) { tab.navigate_row(:up) }},
+          {code: "j", action: ->(tui, tab) { tab.navigate_row(:down) }},
+          {code: "x", description: "Select", action: ->(tui, tab) { tab.toggle_select }},
+          {code: "A", modifiers: ["shift"], display: "A", description: "Select All",
+           action: ->(tui, tab) { tab.toggle_select(:all) }}
+        ],
+        filterable: [
+          {code: "/", display: "/", description: "Filter", action: ->(tui, tab) { tab.start_filtering }},
+          {code: "backspace", action: ->(tui, tab) { tab.remove_last_char_from_filter }, refresh: true},
+          {code: "enter", action: ->(tui, tab) { tab.stop_filtering }, refresh: true},
+          {code: "esc", action: ->(tui, tab) { tab.stop_and_clear_filtering }, refresh: true}
+        ]
+      }.freeze
+
+      # Returns an array of symbols for functionality which this tab implements
+      def features
+        []
+      end
+
+      def controls
+        GLOBAL + SHARED.slice(*features).values.flatten
+      end
+    end
+  end
+end

data/lib/sidekiq/tui/filtering.rb

@@ -0,0 +1,53 @@
+module Sidekiq
+  class TUI
+    module Filtering
+      def filtering?
+        @data[:filtering]
+      end
+
+      def current_filter
+        @data[:filter]
+      end
+
+      def start_filtering
+        @data[:filtering] = true
+        @data[:filter] = ""
+      end
+
+      def stop_filtering
+        return unless @data[:filtering]
+
+        @data[:filtering] = false
+        @data[:selected] = []
+      end
+
+      def stop_and_clear_filtering
+        return unless @data[:filtering]
+
+        @data[:filtering] = false
+        @data[:filter] = nil
+        @data[:selected] = []
+        on_filter_change
+      end
+
+      def remove_last_char_from_filter
+        return unless @data[:filtering]
+
+        @data[:filter] = @data[:filter].empty? ? "" : @data[:filter][0..-2]
+        on_filter_change
+      end
+
+      def append_to_filter(string)
+        return unless @data[:filtering]
+
+        @data[:filter] += string
+        @data[:selected] = []
+        on_filter_change
+      end
+
+      def on_filter_change
+        # callback for subclasses
+      end
+    end
+  end
+end

data/lib/sidekiq/tui/tabs/base_tab.rb

@@ -0,0 +1,204 @@
+module Sidekiq
+  class TUI
+    class BaseTab
+      include Controls
+
+      attr_reader :name
+      attr_reader :data
+
+      def initialize(parent)
+        @parent = parent
+        @name = self.class.name.split("::").last
+        reset_data
+      end
+
+      def t(*)
+        @parent.t(*)
+      end
+
+      def reset_data
+        @data = {selected: [], selected_row_index: 0}
+      end
+
+      def error
+        @data[:error]
+      end
+
+      def error=(e)
+        @data[:error] = e
+      end
+
+      def selected?(entry)
+        @data[:selected].index(entry.id)
+      end
+
+      def filtering?
+        false
+      end
+
+      def each_selection(unselect: true, &)
+        sel = @data[:selected]
+        finished = []
+        if !sel.empty?
+          sel.each do |id|
+            yield id
+            # When processing multiple items in bulk, we want to unselect
+            # each row if its operation succeeds so our UI will not
+            # re-process rows 1-3 if row 4 fails.
+            finished << id
+          end
+        else
+          ids = @data.dig(:table, :row_ids)
+          return if !ids || ids.empty?
+          yield ids[@data[:selected_row_index]]
+        end
+      ensure
+        @data[:selected] = sel - finished if unselect
+      end
+
+      # Navigate the row selection up or down in the current tab's table.
+      # @param direction [Symbol] :up or :down
+      def navigate_row(direction)
+        ids = @data.dig(:table, :row_ids)
+        return if !ids || ids.empty?
+
+        index_change = (direction == :down) ? 1 : -1
+        @data[:selected_row_index] = (@data[:selected_row_index] + index_change) % ids.count
+      end
+
+      def prev_page
+        opts = @data.dig(:table, :pager)
+        return unless opts
+        return if opts.page < 2
+
+        @data[:table][:pager] = Sidekiq::TUI::PageOptions.new(opts.page - 1, opts.size)
+      end
+
+      def next_page
+        np = @data.dig(:table, :next_page)
+        return unless np
+        opts = @data.dig(:table, :pager)
+        return unless opts
+
+        @data[:table][:pager] = Sidekiq::TUI::PageOptions.new(np, opts.size)
+      end
+
+      def toggle_select(which = :current)
+        sel = @data[:selected]
+        # log(which, sel)
+        if which == :current
+          x = @data[:table][:row_ids][@data[:selected_row_index]]
+          if sel.index(x)
+            # already checked, uncheck it
+            sel.delete(x)
+          else
+            sel << x
+          end
+        elsif sel.empty?
+          @data[:selected] = @data[:table][:row_ids]
+        else
+          sel.clear
+        end
+      end
+
+      def refresh_data_for_stats
+        stats = Sidekiq::Stats.new
+        @data[:stats] = {
+          processed: stats.processed,
+          failed: stats.failed,
+          busy: stats.workers_size,
+          enqueued: stats.enqueued,
+          retries: stats.retry_size,
+          scheduled: stats.scheduled_size,
+          dead: stats.dead_size
+        }
+      end
+
+      def render_table(tui, frame, area)
+        page = @data.dig(:table, :current_page) || 1
+        rows = @data.dig(:table, :rows) || []
+        total = @data.dig(:table, :total) || 0
+        footer = ["", "Page: #{page}", "Count: #{rows.size}", "Total: #{total}"]
+        footer << "Selected: #{@data[:selected].size}" unless @data[:selected].empty?
+
+        defaults = {
+          title: "TableName",
+          footer: footer
+        }
+        if features.include?(:selectable)
+          defaults.merge!({
+            highlight_symbol: "➡️",
+            selected_row: @data[:selected_row_index],
+            row_highlight_style: tui.style(fg: :white, bg: :blue)
+          })
+        end
+        hash = defaults.merge(yield)
+        hash[:block] ||= tui.block(title: hash.delete(:title), borders: :all)
+        table = tui.table(**hash)
+        frame.render_widget(table, area)
+      end
+
+      def render_stats_section(tui, frame, area)
+        stats = @data[:stats]
+
+        keys = ["Processed", "Failed", "Busy", "Enqueued", "Retries", "Scheduled", "Dead"]
+        values = [
+          stats[:processed],
+          stats[:failed],
+          stats[:busy],
+          stats[:enqueued],
+          stats[:retries],
+          stats[:scheduled],
+          stats[:dead]
+        ]
+
+        # Format keys and values with spacing
+        keys_line = keys.map { |k| t(k).to_s.ljust(12) }.join("  ")
+        values_line = values.map { |v| number_with_delimiter(v).ljust(12) }.join("  ")
+
+        frame.render_widget(
+          tui.paragraph(
+            text: [keys_line, values_line],
+            block: tui.block(title: "Statistics", borders: [:all])
+          ),
+          area
+        )
+      end
+
+      # [thousands_separator, decimal_separator] per locale.
+      # Locales not listed here use the English default [",", "."].
+      NUMERIC_SEPARATORS = {
+        # period thousands, comma decimal
+        "da" => [".", ","], "de" => [".", ","], "el" => [".", ","],
+        "es" => [".", ","], "it" => [".", ","], "nl" => [".", ","],
+        "pt" => [".", ","], "pt-BR" => [".", ","], "tr" => [".", ","],
+        "vi" => [".", ","],
+        # space thousands, comma decimal
+        "cs" => [" ", ","], "fr" => [" ", ","], "lt" => [" ", ","],
+        "nb" => [" ", ","], "pl" => [" ", ","], "ru" => [" ", ","],
+        "sv" => [" ", ","], "uk" => [" ", ","]
+      }.freeze
+
+      def number_with_delimiter(number, options = {})
+        precision = options[:precision] || 0
+        rounded = number.round(precision)
+        thousands, decimal = NUMERIC_SEPARATORS.fetch(@parent.lang, [",", "."])
+        integer_part, decimal_part = rounded.to_s.split(".")
+        integer_with_sep = integer_part.gsub(/(\d)(?=(\d{3})+(?!\d))/, "\\1#{thousands}")
+        (precision > 0) ? "#{integer_with_sep}#{decimal}#{(decimal_part || "").ljust(precision, "0")}" : integer_with_sep
+      end
+
+      def format_memory(rss_kb)
+        return "0" if rss_kb.nil? || rss_kb == 0
+
+        if rss_kb < 100_000
+          "#{number_with_delimiter(rss_kb)} KB"
+        elsif rss_kb < 10_000_000
+          "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
+        else
+          "#{number_with_delimiter(rss_kb / (1024.0 * 1024.0), precision: 1)} GB"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/tui/tabs/busy.rb

@@ -0,0 +1,118 @@
+require_relative "base_tab"
+
+module Sidekiq
+  class TUI
+    module Tabs
+      class Busy < BaseTab
+        def features
+          %i[selectable]
+        end
+
+        def controls
+          @controls ||= super + [
+            {code: "T", modifiers: ["shift"], description: "Terminate", action: ->(tui, tab) { tab.terminate! }},
+            {code: "Q", modifiers: ["shift"], description: "Quiet", action: ->(tui, tab) { tab.quiet! }}
+          ]
+        end
+
+        def quiet!
+          each_selection do |id|
+            Sidekiq::Process.new("identity" => id).quiet!
+          end
+        end
+
+        def terminate!
+          each_selection do |id|
+            Sidekiq::Process.new("identity" => id).stop!
+          end
+        end
+
+        def refresh_data
+          refresh_data_for_stats
+
+          busy = []
+          table_row_ids = []
+
+          Sidekiq::ProcessSet.new.each do |p|
+            name = "#{p["hostname"]}:#{p["pid"]}"
+            name += " ⭐️" if p.leader?
+            name += " 🛑" if p.stopping?
+            busy << [
+              selected?(p) ? "✅" : "",
+              name,
+              Time.at(p["started_at"]).utc,
+              format_memory(p["rss"].to_i),
+              number_with_delimiter(p["concurrency"]),
+              number_with_delimiter(p["busy"])
+            ]
+            table_row_ids << p.identity
+          end
+
+          @data[:busy] = busy
+          @data[:table] = {row_ids: table_row_ids}
+        end
+
+        def render(tui, frame, area)
+          chunks = tui.layout_split(
+            area,
+            direction: :vertical,
+            constraints: [
+              tui.constraint_length(4), # Stats
+              tui.constraint_length(4), # Status
+              tui.constraint_fill(1)   # Graph
+            ]
+          )
+
+          render_stats_section(tui, frame, chunks[0])
+          render_status_section(tui, frame, chunks[1])
+          render_table(tui, frame, chunks[2]) do
+            {
+              title: t("Processes"),
+              header: ["☑️", "Name", "Started", "RSS", "Threads", "Busy"].map { |x| t(x) },
+              widths: [
+                tui.constraint_length(5),
+                tui.constraint_fill(1),
+                tui.constraint_length(24),
+                tui.constraint_length(10),
+                tui.constraint_length(6),
+                tui.constraint_length(6)
+              ],
+              rows: @data[:busy].map.with_index { |cells, idx|
+                tui.table_row(
+                  cells:,
+                  style: idx.even? ? nil : tui.style(bg: :dark_gray)
+                )
+              }
+            }
+          end
+        end
+
+        def render_status_section(tui, frame, area)
+          values = []
+          processes = Sidekiq::ProcessSet.new
+          workset = Sidekiq::WorkSet.new
+          ws = workset.size
+          values << (s = processes.size
+                     number_with_delimiter(s))
+          values << (x = processes.total_concurrency
+                     number_with_delimiter(x))
+          values << number_with_delimiter(ws)
+          values << "#{(x == 0) ? 0 : ((ws / x.to_f) * 100).round(0)}%"
+          values << format_memory(processes.total_rss)
+
+          keys = %w[Processes Threads Busy Utilization RSS]
+          keys_line = keys.map { |k| t(k).to_s.ljust(12) }.join("  ")
+          values_line = values.map { |v| v.to_s.ljust(12) }.join("  ")
+
+          frame.render_widget(
+            tui.paragraph(
+              text: [keys_line, values_line],
+              block: tui.block(title: t("Status"), borders: [:all])
+            ),
+            area
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/tui/tabs/dead.rb

@@ -0,0 +1,19 @@
+require_relative "base_tab"
+require_relative "set_tab"
+
+module Sidekiq
+  class TUI
+    module Tabs
+      class Dead < BaseTab
+        include SetTab
+
+        def set_class = Sidekiq::DeadSet
+
+        def refresh_data
+          refresh_data_for_stats
+          refresh_data_for_set
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/tui/tabs/home.rb

@@ -0,0 +1,144 @@
+require_relative "base_tab"
+
+module Sidekiq
+  class TUI
+    module Tabs
+      class Home < BaseTab
+        def refresh_data
+          refresh_data_for_stats
+
+          stats = Sidekiq::Stats.new
+          @data[:chart] ||= {
+            previous_stats: {
+              processed: stats.processed,
+              failed: stats.failed
+            },
+            deltas: {
+              processed: Array.new(50, 0),
+              failed: Array.new(50, 0)
+            }
+          }
+
+          processed_delta = stats.processed - @data[:chart][:previous_stats][:processed]
+          failed_delta = stats.failed - @data[:chart][:previous_stats][:failed]
+
+          @data[:chart][:deltas][:processed].shift
+          @data[:chart][:deltas][:processed].push(processed_delta)
+          @data[:chart][:deltas][:failed].shift
+          @data[:chart][:deltas][:failed].push(failed_delta)
+
+          @data[:chart][:previous_stats] = {
+            processed: stats.processed,
+            failed: stats.failed
+          }
+
+          redis_info = Sidekiq.default_configuration.redis_info
+
+          @data[:redis_info] = {
+            version: redis_info["redis_version"] || "N/A",
+            uptime_days: redis_info["uptime_in_days"] || "N/A",
+            connected_clients: redis_info["connected_clients"] || "N/A",
+            used_memory: redis_info["used_memory_human"] || "N/A",
+            peak_memory: redis_info["used_memory_peak_human"] || "N/A"
+          }
+        end
+
+        def render(tui, frame, area)
+          chunks = tui.layout_split(
+            area,
+            direction: :vertical,
+            constraints: [
+              tui.constraint_length(4), # Stats
+              tui.constraint_fill(1),   # Graph
+              tui.constraint_length(4) # Redis
+            ]
+          )
+
+          render_stats_section(tui, frame, chunks[0])
+          render_chart_section(tui, frame, chunks[1])
+          render_redis_info_section(tui, frame, chunks[2])
+        end
+
+        def render_chart_section(tui, frame, area)
+          max_value = [@data[:chart][:deltas][:processed].max, @data[:chart][:deltas][:failed].max, 1].max
+          y_max = [max_value, 5].max
+
+          processed_data = @data[:chart][:deltas][:processed].each_with_index.map { |value, idx| [idx.to_f, value.to_f] }
+          failed_data = @data[:chart][:deltas][:failed].each_with_index.map { |value, idx| [idx.to_f, value.to_f] }
+
+          datasets = [
+            tui.dataset(
+              name: "",
+              data: processed_data,
+              style: tui.style(fg: :green),
+              marker: :dot,
+              graph_type: :line
+            ),
+            tui.dataset(
+              name: "",
+              data: failed_data,
+              style: tui.style(fg: :red),
+              marker: :dot,
+              graph_type: :line
+            )
+          ]
+
+          num_labels = 5
+          y_labels = (0...num_labels).map do |i|
+            value = ((y_max * i) / (num_labels - 1)).round
+            value.to_s
+          end
+
+          beacon_pulse = (Time.now.to_i % 2 == 0) ? "●" : " "
+
+          chart = tui.chart(
+            datasets: datasets,
+            x_axis: tui.axis(
+              bounds: [0.0, 49.0],
+              labels: [],
+              style: tui.style(fg: :white)
+            ),
+            y_axis: tui.axis(
+              bounds: [0.0, y_max.to_f],
+              labels: y_labels,
+              style: tui.style(fg: :white)
+            ),
+            block: tui.block(
+              title: "Dashboard #{beacon_pulse}",
+              borders: [:all]
+            )
+          )
+
+          frame.render_widget(chart, area)
+        end
+
+        def render_redis_info_section(tui, frame, area)
+          redis_info = @data[:redis_info]
+
+          uptime_value = (redis_info[:uptime_days] == "N/A") ? "N/A" : "#{redis_info[:uptime_days]} days"
+
+          keys = ["Version", "Uptime", "Connected Clients", "Memory Usage", "Peak Memory"]
+          values = [
+            redis_info[:version].to_s,
+            uptime_value,
+            redis_info[:connected_clients].to_s,
+            redis_info[:used_memory].to_s,
+            redis_info[:peak_memory].to_s
+          ]
+
+          # Format keys and values with spacing
+          keys_line = keys.map { |k| t(k).ljust(18) }.join("  ")
+          values_line = values.map { |v| v.ljust(18) }.join("  ")
+
+          frame.render_widget(
+            tui.paragraph(
+              text: [keys_line, values_line],
+              block: tui.block(title: "Redis Information", borders: [:all])
+            ),
+            area
+          )
+        end
+      end
+    end
+  end
+end