sidekiq 8.1.1 → 8.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fb090d79e2cf2b320fdd47ee55a574a68a46b77a23db27b849ddb6f54acb00a
-  data.tar.gz: 4863ef28ecdd8ad2cd868caf2f47cc9302b8eb88f4ff9da4f2b5641beca50d88
+  metadata.gz: 205ebd1ee2e6fdbe27b860b06a03147e96d77a3abc32481eb3bbf51ce01a64a6
+  data.tar.gz: 355195d5aadd03117e7eee7bdf144d193a4c9a41d4534b5f05ea6fad847cb7df
 SHA512:
-  metadata.gz: 4ba71a2c43cec613337119cd29830339e213a8f876db4db84c62ac67890ee020ab841a4f7fe478f7f0fcd135571b885e54c9af12998ea6947ac51b543d52b42c
-  data.tar.gz: 9cabfc243aacec3c57c697b83374a5b934cc985c1e4ee86185a6faba7fe18f6540e3fa0b203c5dbfc0e22c03513f66e973be4152d8c8e12db29e97e21a94360f
+  metadata.gz: a592aa88c757173e70ee511388f15642e2aad1347df15252e972cea12e42ab91a0e8df702c3ff9e0069595bb68dae32a6d53318df390563a9ae7cc0df31852dd
+  data.tar.gz: d10b27d602e3c53fe79a85d21a59d0492831c32c8a67e2a344869c42e22e402ad591f82e3835e14c879a81fe8020430450fecd6a324d008bf33c4592dee23ea9

data/Changes.md

@@ -2,16 +2,41 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+8.1.3
+----------
+
+- Fix edge case leading to duplicate, concurrent execution [#6379]
+  If 2 Capsules process jobs from the same queue, long-running
+  jobs could run in parallel during process shutdown.
+- [SECURITY] Remove as much YAML usage as possible. [#6950]
+  Localization files in `web/locales` are now manually parsed.
+  Sidekiq::CLI will now only require YAML if you use a `-C` .yml file.
+
+8.1.2
+----------
+
+- Initial release for `kiq`, Sidekiq's official terminal UI:
+```
+bundle exec kiq
+```
+Use REDIS_URL or REDIS_PROVIDER to point `kiq` to Redis.
+- Mutation during iteration in `SortedSet#each` caused it to miss half of the jobs [#6936]
+- Fix edge case resulting in nil crash on /busy page [#6954]
+
 8.1.1
 ----------
 
-- Add new `Sidekiq.testing!(mode)` API [#6931]
-Requiring code should not enable process-wide changes.
+- **DEPRECATION** `require 'sidekiq/testing'` and
+  `require 'sidekiq/testing/inline'`.
+  Add new `Sidekiq.testing!(mode)` API [#6931]
+  Requiring code should not enable process-wide changes.
 ```ruby
 # Old, implicit
 require "sidekiq/testing"
+require "sidekiq/testing/inline"
 # New, more explicit
 Sidekiq.testing!(:fake)
+Sidekiq.testing!(:inline)
 ```
 - Fix race condition with Stop button in UI [#6935]
 - Fix javascript error handler [#6893]

data/README.md

@@ -90,7 +90,7 @@ Useful resources:
 * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
 
 Every Thursday morning is Sidekiq Office Hour: I video chat and answer questions.
-See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
+See the [Sidekiq support page](https://sidekiq.org/support/) for details.
 
 Contributing
 -----------------

data/bin/kiq

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+# This requires the default gemset so Sidekiq Pro
+# and Sidekiq Enterprise can load any code extensions.
+Bundler.require(:default, :tui)
+
+require_relative "../lib/sidekiq/tui"
+
+# Run any load hooks registered during Bundler.require
+Sidekiq.loader.run_load_hooks(:tui)
+
+tt = Sidekiq::TUI.new(Sidekiq.default_configuration)
+
+RatatuiRuby.run do |tui|
+  tt.prepare(tui)
+  tt.run_loop
+end

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -63,19 +63,19 @@ begin
         def enqueue(job)
           # NB: Active Job only serializes keys it recognizes. We
           # cannot set arbitrary key/values here.
-          wrapper = Sidekiq::ActiveJob::Wrapper.set(
-            wrapped: job.class,
-            queue: job.queue_name
-          )
+          options = {wrapped: job.class, queue: job.queue_name}
+          options[:profile] = job.profile if job.respond_to?(:profile) && !job.profile.nil?
+
+          wrapper = Sidekiq::ActiveJob::Wrapper.set(options)
           job.provider_job_id = wrapper.perform_async(job.serialize)
         end
 
         # @api private
         def enqueue_at(job, timestamp)
-          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
-            wrapped: job.class,
-            queue: job.queue_name
-          ).perform_at(timestamp, job.serialize)
+          options = {wrapped: job.class, queue: job.queue_name}
+          options[:profile] = job.profile if job.respond_to?(:profile) && !job.profile.nil?
+
+          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(options).perform_at(timestamp, job.serialize)
         end
 
         # @api private

data/lib/sidekiq/api.rb

@@ -657,38 +657,8 @@ module Sidekiq
 
     private
 
-    def remove_job
-      Sidekiq.redis do |conn|
-        results = conn.multi { |transaction|
-          transaction.zrange(parent.name, score, score, "BYSCORE")
-          transaction.zremrangebyscore(parent.name, score, score)
-        }.first
-
-        if results.size == 1
-          yield results.first
-        else
-          # multiple jobs with the same score
-          # find the one with the right JID and push it
-          matched, nonmatched = results.partition { |message|
-            if message.index(jid)
-              msg = Sidekiq.load_json(message)
-              msg["jid"] == jid
-            else
-              false
-            end
-          }
-
-          msg = matched.first
-          yield msg if msg
-
-          # push the rest back onto the sorted set
-          conn.multi do |transaction|
-            nonmatched.each do |message|
-              transaction.zadd(parent.name, score.to_f.to_s, message)
-            end
-          end
-        end
-      end
+    def remove_job(&)
+      parent.remove_job(self, &)
     end
   end
 
@@ -857,6 +827,46 @@ module Sidekiq
       nil
     end
 
+    def remove_job(entry)
+      score = entry.score
+      jid = entry.jid
+      Sidekiq.redis do |conn|
+        results = conn.multi { |transaction|
+          transaction.zrange(name, score, score, "BYSCORE")
+          transaction.zremrangebyscore(name, score, score)
+        }.first
+
+        if results.size == 1
+          yield results.first
+          @_size -= 1
+        else
+          # multiple jobs with the same score
+          # find the one with the right JID and push it
+          matched, nonmatched = results.partition { |message|
+            if message.index(jid)
+              msg = Sidekiq.load_json(message)
+              msg["jid"] == jid
+            else
+              false
+            end
+          }
+
+          msg = matched.first
+          if msg
+            yield msg
+            @_size -= 1
+          end
+
+          # push the rest back onto the sorted set
+          conn.multi do |transaction|
+            nonmatched.each do |message|
+              transaction.zadd(name, score.to_f.to_s, message)
+            end
+          end
+        end
+      end
+    end
+
     # :nodoc:
     # @api private
     def delete_by_value(name, value)
@@ -1030,19 +1040,20 @@ module Sidekiq
         # you'll be happier this way
         conn.pipelined do |pipeline|
           procs.each do |key|
-            pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
+            pipeline.hmget(key, "info", "concurrency", "busy", "beat", "quiet", "rss", "rtt_us")
           end
         end
       }
 
-      result.each do |info, busy, beat, quiet, rss, rtt_us|
+      result.each do |info, concurrency, busy, beat, quiet, rss, rtt_us|
         # If a process is stopped between when we query Redis for `procs` and
         # when we query for `result`, we will have an item in `result` that is
         # composed of `nil` values.
         next if info.nil?
 
         hash = Sidekiq.load_json(info)
-        yield Process.new(hash.merge("busy" => busy.to_i,
+        yield Process.new(hash.merge("concurrency" => concurrency.to_i,
+          "busy" => busy.to_i,
           "beat" => beat.to_f,
           "quiet" => quiet,
           "rss" => rss.to_i,

data/lib/sidekiq/capsule.rb

@@ -51,7 +51,6 @@ module Sidekiq
     end
 
     def stop
-      fetcher&.bulk_requeue([])
     end
 
     # Sidekiq checks queues in three modes:

data/lib/sidekiq/cli.rb

@@ -2,7 +2,6 @@
 
 $stdout.sync = true
 
-require "yaml"
 require "optparse"
 require "erb"
 require "fileutils"
@@ -409,6 +408,7 @@ module Sidekiq # :nodoc:
     def parse_config(path)
       erb = ERB.new(File.read(path), trim_mode: "-")
       erb.filename = File.expand_path(path)
+      require "yaml"
       opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
 
       if opts.respond_to? :deep_symbolize_keys!

data/lib/sidekiq/client.rb

@@ -132,12 +132,14 @@ module Sidekiq
     #   push_bulk('class' => MyJob, 'args' => (1..100_000).to_a, batch_size: 1_000)
     #
     def push_bulk(items)
-      batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
       args = items["args"]
       at = items.delete("at") || items.delete(:at)
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
+      # Use a smaller batch size by default for scheduled jobs since adding to sorted sets is more costly.
+      batch_size = items.delete(:batch_size) || items.delete("batch_size") || (at ? 100 : 1_000)
+
       jid = items.delete("jid")
       raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 

data/lib/sidekiq/component.rb

@@ -57,6 +57,9 @@ module Sidekiq
     end
 
     def tid
+      # We XOR with PID to ensure Thread IDs changes after fork.
+      # I'm unclear why we don't multiply the two values to better guarantee
+      # a unique value but it's been this way for quite a while now. #3685
       Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
     end
 

data/lib/sidekiq/config.rb

@@ -255,7 +255,7 @@ module Sidekiq
     # Register a proc to handle any error which occurs within the Sidekiq process.
     #
     #   Sidekiq.configure_server do |config|
-    #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+    #     config.error_handlers << proc {|ex,ctx_hash,config| MyErrorService.notify(ex, ctx_hash) }
     #   end
     #
     # The default error handler logs errors to @logger.

data/lib/sidekiq/launcher.rb

@@ -176,6 +176,7 @@ module Sidekiq
             transaction.sadd("processes", [key])
             transaction.exists(key)
             transaction.hset(key, "info", to_json,
+              "concurrency", @config.total_concurrency,
               "busy", curstate.size,
               "beat", Time.now.to_f,
               "rtt_us", rtt,
@@ -257,7 +258,6 @@ module Sidekiq
         "started_at" => Time.now.to_f,
         "pid" => ::Process.pid,
         "tag" => @config[:tag] || "",
-        "concurrency" => @config.total_concurrency,
         "capsules" => @config.capsules.each_with_object({}) { |(name, cap), memo|
           memo[name] = cap.to_h
         },

data/lib/sidekiq/manager.rb

@@ -69,7 +69,7 @@ module Sidekiq
     def processor_result(processor, reason = nil)
       @plock.synchronize do
         @workers.delete(processor)
-        unless @done
+        if !@done && @count > @workers.size
           p = Processor.new(@config, &method(:processor_result))
           @workers << p
           p.start

data/lib/sidekiq/paginator.rb

@@ -62,7 +62,12 @@ module Sidekiq
       pageidx = current_page - 1
       starting = pageidx * page_size
       items = items.to_a
-      [current_page, items.size, items[starting, page_size]]
+      total_size = items.size
+      if starting > total_size
+        starting = 0
+        current_page = 1
+      end
+      [current_page, total_size, items[starting, page_size]]
     end
   end
 end

data/lib/sidekiq/profiler.rb

@@ -21,7 +21,7 @@ module Sidekiq
       return yield unless job["profile"]
 
       token = job["profile"]
-      type = job["class"]
+      type = job["wrapped"] || job["class"]
       jid = job["jid"]
       started_at = Time.now
 

data/lib/sidekiq/scheduled.rb

@@ -117,9 +117,7 @@ module Sidekiq
       private
 
       def wait
-        @sleeper.pop(timeout: random_poll_interval)
-      rescue Timeout::Error
-        # TODO move to exception: false
+        @sleeper.pop(timeout: random_poll_interval, exception: false)
       rescue => ex
         # if poll_interval_average hasn't been calculated yet, we can
         # raise an error trying to reach Redis.
@@ -225,8 +223,7 @@ module Sidekiq
         total += INITIAL_WAIT unless @config[:poll_interval_average]
         total += (5 * rand)
 
-        @sleeper.pop(timeout: total)
-      rescue Timeout::Error
+        @sleeper.pop(timeout: total, exception: false)
       ensure
         # periodically clean out the `processes` set in Redis which can collect
         # references to dead processes over time. The process count affects how

data/lib/sidekiq/tui/controls.rb

@@ -0,0 +1,53 @@
+module Sidekiq
+  class TUI
+    module Controls
+      # Defines data for input handling and for displaying controls.
+      # :code is the key code for input handling.
+      # :display and :description are shown in the controls area, with different
+      #   styling between them. If :display is omitted, :code is displayed instead.
+      # :action is a lambda to execute when the control is triggered.
+      # :refresh means the action requires immediate refreshing of data
+      #
+      # Conventions: dangerous/irreversible actions should use UPPERCASE codes.
+      # The Shift button means "I'm sure".
+      GLOBAL = [
+        {code: "?", display: "?", description: "Help", action: ->(tui, tab) { tui.show_help }},
+        {code: "left", display: "←/→", description: "Select Tab", action: ->(tui, tab) { tui.navigate(:left) }, refresh: true},
+        {code: "right", action: ->(tui, tab) { tui.navigate(:right) }, refresh: true},
+        {code: "q", description: "Quit", action: ->(tui, tab) { :quit }},
+        {code: "c", modifiers: ["ctrl"], action: ->(tui, tab) { :quit }}
+      ].freeze
+
+      SHARED = {
+        pageable: [
+          {code: "h", display: "h/l", description: "Prev/Next Page",
+           action: ->(tui, tab) { tab.prev_page }, refresh: true},
+          {code: "l", action: ->(tui, tab) { tab.next_page }, refresh: true}
+        ],
+        selectable: [
+          {code: "k", display: "j/k", description: "Prev/Next Row",
+           action: ->(tui, tab) { tab.navigate_row(:up) }},
+          {code: "j", action: ->(tui, tab) { tab.navigate_row(:down) }},
+          {code: "x", description: "Select", action: ->(tui, tab) { tab.toggle_select }},
+          {code: "A", modifiers: ["shift"], display: "A", description: "Select All",
+           action: ->(tui, tab) { tab.toggle_select(:all) }}
+        ],
+        filterable: [
+          {code: "/", display: "/", description: "Filter", action: ->(tui, tab) { tab.start_filtering }},
+          {code: "backspace", action: ->(tui, tab) { tab.remove_last_char_from_filter }, refresh: true},
+          {code: "enter", action: ->(tui, tab) { tab.stop_filtering }, refresh: true},
+          {code: "esc", action: ->(tui, tab) { tab.stop_and_clear_filtering }, refresh: true}
+        ]
+      }.freeze
+
+      # Returns an array of symbols for functionality which this tab implements
+      def features
+        []
+      end
+
+      def controls
+        GLOBAL + SHARED.slice(*features).values.flatten
+      end
+    end
+  end
+end

data/lib/sidekiq/tui/filtering.rb

@@ -0,0 +1,53 @@
+module Sidekiq
+  class TUI
+    module Filtering
+      def filtering?
+        @data[:filtering]
+      end
+
+      def current_filter
+        @data[:filter]
+      end
+
+      def start_filtering
+        @data[:filtering] = true
+        @data[:filter] = ""
+      end
+
+      def stop_filtering
+        return unless @data[:filtering]
+
+        @data[:filtering] = false
+        @data[:selected] = []
+      end
+
+      def stop_and_clear_filtering
+        return unless @data[:filtering]
+
+        @data[:filtering] = false
+        @data[:filter] = nil
+        @data[:selected] = []
+        on_filter_change
+      end
+
+      def remove_last_char_from_filter
+        return unless @data[:filtering]
+
+        @data[:filter] = @data[:filter].empty? ? "" : @data[:filter][0..-2]
+        on_filter_change
+      end
+
+      def append_to_filter(string)
+        return unless @data[:filtering]
+
+        @data[:filter] += string
+        @data[:selected] = []
+        on_filter_change
+      end
+
+      def on_filter_change
+        # callback for subclasses
+      end
+    end
+  end
+end

data/lib/sidekiq/tui/tabs/base_tab.rb

@@ -0,0 +1,187 @@
+module Sidekiq
+  class TUI
+    class BaseTab
+      include Controls
+
+      attr_reader :name
+      attr_reader :data
+
+      def initialize(parent)
+        @parent = parent
+        @name = self.class.name.split("::").last
+        reset_data
+      end
+
+      def t(*)
+        @parent.t(*)
+      end
+
+      def reset_data
+        @data = {selected: [], selected_row_index: 0}
+      end
+
+      def error
+        @data[:error]
+      end
+
+      def error=(e)
+        @data[:error] = e
+      end
+
+      def selected?(entry)
+        @data[:selected].index(entry.id)
+      end
+
+      def filtering?
+        false
+      end
+
+      def each_selection(unselect: true, &)
+        sel = @data[:selected]
+        finished = []
+        if !sel.empty?
+          sel.each do |id|
+            yield id
+            # When processing multiple items in bulk, we want to unselect
+            # each row if its operation succeeds so our UI will not
+            # re-process rows 1-3 if row 4 fails.
+            finished << id
+          end
+        else
+          ids = @data.dig(:table, :row_ids)
+          return if !ids || ids.empty?
+          yield ids[@data[:selected_row_index]]
+        end
+      ensure
+        @data[:selected] = sel - finished if unselect
+      end
+
+      # Navigate the row selection up or down in the current tab's table.
+      # @param direction [Symbol] :up or :down
+      def navigate_row(direction)
+        ids = @data.dig(:table, :row_ids)
+        return if !ids || ids.empty?
+
+        index_change = (direction == :down) ? 1 : -1
+        @data[:selected_row_index] = (@data[:selected_row_index] + index_change) % ids.count
+      end
+
+      def prev_page
+        opts = @data.dig(:table, :pager)
+        return unless opts
+        return if opts.page < 2
+
+        @data[:table][:pager] = Sidekiq::TUI::PageOptions.new(opts.page - 1, opts.size)
+      end
+
+      def next_page
+        np = @data.dig(:table, :next_page)
+        return unless np
+        opts = @data.dig(:table, :pager)
+        return unless opts
+
+        @data[:table][:pager] = Sidekiq::TUI::PageOptions.new(np, opts.size)
+      end
+
+      def toggle_select(which = :current)
+        sel = @data[:selected]
+        # log(which, sel)
+        if which == :current
+          x = @data[:table][:row_ids][@data[:selected_row_index]]
+          if sel.index(x)
+            # already checked, uncheck it
+            sel.delete(x)
+          else
+            sel << x
+          end
+        elsif sel.empty?
+          @data[:selected] = @data[:table][:row_ids]
+        else
+          sel.clear
+        end
+      end
+
+      def refresh_data_for_stats
+        stats = Sidekiq::Stats.new
+        @data[:stats] = {
+          processed: stats.processed,
+          failed: stats.failed,
+          busy: stats.workers_size,
+          enqueued: stats.enqueued,
+          retries: stats.retry_size,
+          scheduled: stats.scheduled_size,
+          dead: stats.dead_size
+        }
+      end
+
+      def render_table(tui, frame, area)
+        page = @data.dig(:table, :current_page) || 1
+        rows = @data.dig(:table, :rows) || []
+        total = @data.dig(:table, :total) || 0
+        footer = ["", "Page: #{page}", "Count: #{rows.size}", "Total: #{total}"]
+        footer << "Selected: #{@data[:selected].size}" unless @data[:selected].empty?
+
+        defaults = {
+          title: "TableName",
+          footer: footer
+        }
+        if features.include?(:selectable)
+          defaults.merge!({
+            highlight_symbol: "➡️",
+            selected_row: @data[:selected_row_index],
+            row_highlight_style: tui.style(fg: :white, bg: :blue)
+          })
+        end
+        hash = defaults.merge(yield)
+        hash[:block] ||= tui.block(title: hash.delete(:title), borders: :all)
+        table = tui.table(**hash)
+        frame.render_widget(table, area)
+      end
+
+      def render_stats_section(tui, frame, area)
+        stats = @data[:stats]
+
+        keys = ["Processed", "Failed", "Busy", "Enqueued", "Retries", "Scheduled", "Dead"]
+        values = [
+          stats[:processed],
+          stats[:failed],
+          stats[:busy],
+          stats[:enqueued],
+          stats[:retries],
+          stats[:scheduled],
+          stats[:dead]
+        ]
+
+        # Format keys and values with spacing
+        keys_line = keys.map { |k| t(k).to_s.ljust(12) }.join("  ")
+        values_line = values.map { |v| v.to_s.ljust(12) }.join("  ")
+
+        frame.render_widget(
+          tui.paragraph(
+            text: [keys_line, values_line],
+            block: tui.block(title: "Statistics", borders: [:all])
+          ),
+          area
+        )
+      end
+
+      # TODO Implement I18n delimiter
+      def number_with_delimiter(number, options = {})
+        precision = options[:precision] || 0
+        number.round(precision)
+      end
+
+      def format_memory(rss_kb)
+        return "0" if rss_kb.nil? || rss_kb == 0
+
+        if rss_kb < 100_000
+          "#{number_with_delimiter(rss_kb)} KB"
+        elsif rss_kb < 10_000_000
+          "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
+        else
+          "#{number_with_delimiter(rss_kb / (1024.0 * 1024.0), precision: 1)} GB"
+        end
+      end
+    end
+  end
+end