sidekiq 8.1.2 → 8.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d21db4cf06c0b4d0b5fb770162e289403a76ed996d6c99613cf45083f0e671d
-  data.tar.gz: 07cf848cd5de112f3153ecd0014991cbf2abf9e66bc0847d2b7845ec7a726329
+  metadata.gz: 44df206459c8280090bcf339baa35fbd6ff2ed2dee6ade615e71caed68e34c77
+  data.tar.gz: c3e3238d59e0150d5ab67edf02c4a73eae7b7dfd3a19b72db7354ea4ef8b677c
 SHA512:
-  metadata.gz: 395d346f8b5227480e4d830b3c1f219a35bb0e2527283f9c00a6a4043c1ebe271998874ae53240b6e7dabca89ce8ad939d0a3969da2b33c3e4155fccbebae9ba
-  data.tar.gz: 20f44288fd8990544569af7f2a87d90d4ce898b2ce760cc95dd2f30c5a0c4ffc90bcc087cbfa22dbc188a904cc2e817756a83b5b3ffd648f75dc0413d3bec3ff
+  metadata.gz: b6d89797767ce01eb6dc86ca8dc88edb96e66c3b407ba66309228ed81b993dfeaf31bf0dd9c00f1a8a26320e983006629330d28101d9a2c735df4897062bccad
+  data.tar.gz: a027b597b9b3446555fdf37d6ec0ec35edb208815c73500febbd3f914dcda4da633aa3c05ce678fa90ed25244bcd0b189ad2adab0b733a131f7f14b4a1463ce1

data/Changes.md

@@ -2,6 +2,22 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+8.1.4
+----------
+
+- The TTIN signal is undeprecated as the INFO signal is not supported on Linux
+- Show iteration job state on Busy page [#6978]
+
+8.1.3
+----------
+
+- Fix edge case leading to duplicate, concurrent execution [#6379]
+  If 2 Capsules process jobs from the same queue, long-running
+  jobs could run in parallel during process shutdown.
+- [SECURITY] Remove as much YAML usage as possible. [#6950]
+  Localization files in `web/locales` are now manually parsed.
+  Sidekiq::CLI will now only require YAML if you use a `-C` .yml file.
+
 8.1.2
 ----------
 

data/lib/sidekiq/api.rb

@@ -478,6 +478,10 @@ module Sidekiq
       self["jid"]
     end
 
+    def iterable_state
+      @iterable_state ||= Sidekiq::IterableJobQuery.new(jid)[jid]
+    end
+
     def bid
       self["bid"]
     end
@@ -1397,6 +1401,66 @@ module Sidekiq
       Sidekiq.redis { |c| c.hget(key, "data") }
     end
   end
+
+  # Persisted iteration state from Redis for jobs using Sidekiq::IterableJob.
+  class IterableJobQuery
+    def initialize(jids)
+      @cache = bulk_fetch(jids)
+    end
+
+    def [](jid)
+      @cache[jid]
+    end
+
+    private
+
+    # Bulk-fetch iteration state for multiple JIDs in a single Redis pipeline.
+    # Returns a Hash of { jid => IterableJobState } for JIDs that have iteration state.
+    def bulk_fetch(jids)
+      raise ArgumentError unless jids
+      jids_to_fetch = Array(jids).compact.uniq
+      return {} if jids_to_fetch.empty?
+
+      results = Sidekiq.redis do |conn|
+        conn.pipelined do |pipe|
+          jids_to_fetch.each { |jid| pipe.hgetall("it-#{jid}") }
+        end
+      end
+
+      # TODO Requires Ruby 4
+      # states = ::Hash.new(capacity: jids_to_fetch.size)
+      states = {}
+      jids_to_fetch.each_with_index do |jid, i|
+        raw = results[i]
+        next if raw.nil? || raw.empty?
+
+        states[jid] = State.new(jid, raw)
+      end
+      states
+    end
+
+    State = Struct.new(:jid, :raw) do
+      def executions
+        raw["ex"].to_i
+      end
+
+      def runtime
+        raw["rt"].to_f
+      end
+
+      def cursor
+        @cursor ||= begin
+          Sidekiq.load_json(raw["c"])
+        rescue JSON::ParserError
+          @raw["c"]
+        end
+      end
+
+      def cancelled
+        raw["cancelled"]&.to_i
+      end
+    end
+  end
 end
 
 Sidekiq.loader.run_load_hooks(:api)

data/lib/sidekiq/capsule.rb

@@ -51,7 +51,6 @@ module Sidekiq
     end
 
     def stop
-      fetcher&.bulk_requeue([])
     end
 
     # Sidekiq checks queues in three modes:

data/lib/sidekiq/cli.rb

@@ -2,7 +2,6 @@
 
 $stdout.sync = true
 
-require "yaml"
 require "optparse"
 require "erb"
 require "fileutils"
@@ -201,9 +200,7 @@ module Sidekiq # :nodoc:
         cli.logger.info "Received TSTP, no longer accepting new work"
         cli.launcher.quiet
       },
-      # deprecated, use INFO
       "TTIN" => ->(cli) {
-        cli.logger.error { "DEPRECATED: Please use the INFO signal for backtraces, support for TTIN will be removed in Sidekiq 9.0." }
         Thread.list.each do |thread|
           cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
           if thread.backtrace
@@ -214,6 +211,7 @@ module Sidekiq # :nodoc:
         end
       },
       "INFO" => ->(cli) {
+        cli.logger.error { "DEPRECATED: the INFO signal does not work on Linux, use TTIN instead." }
         Thread.list.each do |thread|
           cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
           if thread.backtrace
@@ -409,6 +407,7 @@ module Sidekiq # :nodoc:
     def parse_config(path)
       erb = ERB.new(File.read(path), trim_mode: "-")
       erb.filename = File.expand_path(path)
+      require "yaml"
       opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
 
       if opts.respond_to? :deep_symbolize_keys!

data/lib/sidekiq/client.rb

@@ -117,6 +117,11 @@ module Sidekiq
     # larger than 1000 but YMMV based on network quality, size of job args, etc.
     # A large number of jobs can cause a bit of Redis command processing latency.
     #
+    # Accepts an `:at` option to schedule the jobs for future execution. It
+    # accepts either a single Numeric timestamp (or seconds-from-now) applied
+    # to every job, or an Array of Numeric values with the same size as `args`
+    # to schedule each job at its corresponding time.
+    #
     # Accepts an additional `:spread_interval` option (in seconds) to randomly spread
     # the jobs schedule times over the specified interval.
     #

data/lib/sidekiq/config.rb

@@ -255,7 +255,7 @@ module Sidekiq
     # Register a proc to handle any error which occurs within the Sidekiq process.
     #
     #   Sidekiq.configure_server do |config|
-    #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+    #     config.error_handlers << proc {|ex,ctx_hash,config| MyErrorService.notify(ex, ctx_hash) }
     #   end
     #
     # The default error handler logs errors to @logger.

data/lib/sidekiq/job.rb

@@ -313,6 +313,11 @@ module Sidekiq
       #
       # +items+ must be an Array of Arrays.
       #
+      # The +:at+ option schedules the jobs for future execution. It accepts
+      # either a single Numeric timestamp (or seconds-from-now) applied to every
+      # job, or an Array of Numeric values with the same size as +items+ to
+      # schedule each job at its corresponding time.
+      #
       # For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
       #
       # Example (3 Redis round trips):
@@ -325,6 +330,14 @@ module Sidekiq
       #
       #     SomeJob.perform_bulk([[1], [2], [3]])
       #
+      # Scheduling every job 60 seconds from now (single Numeric +:at+):
+      #
+      #     SomeJob.perform_bulk([[1], [2], [3]], at: 60)
+      #
+      # Scheduling each job at its own time (Array +:at+):
+      #
+      #     SomeJob.perform_bulk([[1], [2]], at: [Time.now.to_f + 30, Time.now.to_f + 60])
+      #
       def perform_bulk(*args, **kwargs)
         Setter.new(self, {}).perform_bulk(*args, **kwargs)
       end

data/lib/sidekiq/scheduled.rb

@@ -117,9 +117,7 @@ module Sidekiq
       private
 
       def wait
-        @sleeper.pop(timeout: random_poll_interval)
-      rescue Timeout::Error
-        # TODO move to exception: false
+        @sleeper.pop(timeout: random_poll_interval, exception: false)
       rescue => ex
         # if poll_interval_average hasn't been calculated yet, we can
         # raise an error trying to reach Redis.
@@ -225,8 +223,7 @@ module Sidekiq
         total += INITIAL_WAIT unless @config[:poll_interval_average]
         total += (5 * rand)
 
-        @sleeper.pop(timeout: total)
-      rescue Timeout::Error
+        @sleeper.pop(timeout: total, exception: false)
       ensure
         # periodically clean out the `processes` set in Redis which can collect
         # references to dead processes over time. The process count affects how

data/lib/sidekiq/tui/tabs/base_tab.rb

@@ -154,7 +154,7 @@ module Sidekiq
 
         # Format keys and values with spacing
         keys_line = keys.map { |k| t(k).to_s.ljust(12) }.join("  ")
-        values_line = values.map { |v| v.to_s.ljust(12) }.join("  ")
+        values_line = values.map { |v| number_with_delimiter(v).ljust(12) }.join("  ")
 
         frame.render_widget(
           tui.paragraph(
@@ -165,10 +165,27 @@ module Sidekiq
         )
       end
 
-      # TODO Implement I18n delimiter
+      # [thousands_separator, decimal_separator] per locale.
+      # Locales not listed here use the English default [",", "."].
+      NUMERIC_SEPARATORS = {
+        # period thousands, comma decimal
+        "da" => [".", ","], "de" => [".", ","], "el" => [".", ","],
+        "es" => [".", ","], "it" => [".", ","], "nl" => [".", ","],
+        "pt" => [".", ","], "pt-BR" => [".", ","], "tr" => [".", ","],
+        "vi" => [".", ","],
+        # space thousands, comma decimal
+        "cs" => [" ", ","], "fr" => [" ", ","], "lt" => [" ", ","],
+        "nb" => [" ", ","], "pl" => [" ", ","], "ru" => [" ", ","],
+        "sv" => [" ", ","], "uk" => [" ", ","]
+      }.freeze
+
       def number_with_delimiter(number, options = {})
         precision = options[:precision] || 0
-        number.round(precision)
+        rounded = number.round(precision)
+        thousands, decimal = NUMERIC_SEPARATORS.fetch(@parent.lang, [",", "."])
+        integer_part, decimal_part = rounded.to_s.split(".")
+        integer_with_sep = integer_part.gsub(/(\d)(?=(\d{3})+(?!\d))/, "\\1#{thousands}")
+        (precision > 0) ? "#{integer_with_sep}#{decimal}#{(decimal_part || "").ljust(precision, "0")}" : integer_with_sep
       end
 
       def format_memory(rss_kb)

data/lib/sidekiq/tui.rb

@@ -21,6 +21,8 @@ module Sidekiq
     REFRESH_INTERVAL_SECONDS = 2
     LOCALE_DIRECTORIES = [File.expand_path("#{File.dirname(__FILE__)}/../../web/locales")]
 
+    attr_reader :lang
+
     # language is meant to be a locale code, e.g.
     # LANG=en_US.utf-8
     def initialize(cfg, language: ENV["LANG"] || "en")

data/lib/sidekiq/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "8.1.2"
+  VERSION = "8.1.4"
   MAJOR = 8
 
   def self.gem_version

data/lib/sidekiq/web/application.rb

@@ -92,6 +92,8 @@ module Sidekiq
         @count = (url_params("count") || 100).to_i
         (@current_page, @total_size, @workset) = page_items(workset, url_params("page"), @count)
 
+        @iterable_states = Sidekiq::IterableJobQuery.new(workset.map { |_, _, work| work.job.jid })
+
         erb(:busy)
       end
 
@@ -226,6 +228,8 @@ module Sidekiq
           @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
         end
 
+        @iterable_states = Sidekiq::IterableJobQuery.new(@retries.map(&:jid))
+
         erb(:retries)
       end
 

data/lib/sidekiq/web/helpers.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "uri"
-require "yaml"
 require "cgi/escape"
 
 module Sidekiq
@@ -73,12 +72,43 @@ module Sidekiq
       # so extensions can be localized
       @@strings[lang] ||= config.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
-          strs = YAML.safe_load_file(file)
+          strs = parse_yaml_new(file)
           global.merge!(strs[lang])
         end
       end
     end
 
+    # TODO Remove
+    def parse_yaml_old(path)
+      require "yaml"
+      YAML.safe_load_file(path)
+    end
+
+    def parse_yaml_new(path)
+      locale = nil
+      map = {}
+      IO.readlines(path, chomp: true).each do |line|
+        case line
+        when /\A\s*\#.*/
+          # line comment
+        when !locale && /\A([a-zA-Z\-_]+):/
+          locale = $1
+          map[locale] = {}
+        when /\A\s+(\w+):\s+(.+)\z/
+          # A few values have double quotes to include special characters in YAML.
+          # Strip them off manually as our greedy match will include them.
+          key = $1
+          s = $2
+          s = s[1..] if s[0] == "\""
+          s = s[0..-2] if s[-1] == "\""
+          map[locale][key] = s
+        else
+          raise ArgumentError, "unable to parse #{path}: #{line}"
+        end
+      end
+      map
+    end
+
     def to_json(x)
       Sidekiq.dump_json(x)
     end

data/web/assets/stylesheets/style.css

@@ -534,6 +534,8 @@ input[type="checkbox"] { accent-color: var(--color-primary); }
 
 .pagination a { text-decoration: none; }
 
+.pagination .active a { color: var(--color-primary); font-weight: 700; }
+
 .pagination .disabled { opacity: 0.3; }
 
 div:has(.pagination.pull-right) { margin-left: auto; }

data/web/locales/ar.yml

@@ -74,7 +74,7 @@ ar:
   Stop: إيقاف
   StopAll: إيقاف الكل
   StopPolling: إيقاف الاستعلامات
-  TextDirection: 'rtl'
+  TextDirection: rtl
   Thread: نيسب
   Threads: نياسب
   ThreeMonths: ثلاثة أشهر

data/web/locales/fa.yml

@@ -69,7 +69,7 @@ fa:
   Stop: توقف
   StopAll: توقف همه
   StopPolling: Stop Polling
-  TextDirection: 'rtl'
+  TextDirection: rtl
   Thread: رشته
   Threads: رشته ها
   ThreeMonths: ۳ ماه

data/web/locales/gd.yml

@@ -1,6 +1,6 @@
-# elements like %{queue} are variables and should not be translated
+# elements like %{queue} are variables and should not be translated
 gd:
-  LanguageName: Gaeilge
+  LanguageName: Gàidhlig
   Actions: Gnìomhan
   AddToQueue: Cuir ris a’ chiutha
   AddAllToQueue: Cuir a h-uile ris a’ chiutha
@@ -36,6 +36,8 @@ gd:
   Jobs: Obraichean
   Kill: Marbh
   KillAll: Marbh na h-uile
+  Language: Cànan
+  LastDashboardUpdateTemplateLiteral: "An t-ath-nuadhachadh mu dheireadh: Air pròiseasadh: PROCESSED_COUNT. Air fàilligeadh: FAILED_COUNT."
   LastRetry: An oidhirp mu dheireadh
   Latency: Foillidheachd
   LivePoll: Ath-nuadhachadh beò
@@ -55,6 +57,7 @@ gd:
   PeakMemoryUsage: Bàrr cleachdadh a’ chuimhne
   Plugins: Plugain
   PollingInterval: Eadaramh an ath-nuadhachaidh
+  PollingIntervalMilliseconds: Milidiogan eadaramh an ath-nuadhachaidh
   Process: Pròiseas
   Processed: Air pròiseasadh
   Processes: Pròiseasan
@@ -98,3 +101,10 @@ gd:
   AvgExecutionTime: Ùine cuibheasach nan gnìomhan
   Context: Co-theacsa
   NoJobMetricsFound: Cha deach meatraigeachd o chionn goirid air obair a lorg
+  Filter: Criathraich
+  AnyJobContent: Susbaint obrach sam bith
+  Profiles: Pròifilean
+  Data: Dàta
+  View: Seall
+  Token: Tòcan
+  ElapsedTime: An ùine a chaidh seachad

data/web/locales/he.yml

@@ -69,7 +69,7 @@ he:
   Stop: עצור
   StopAll: עצור הכל
   StopPolling: עצור תשאול
-  TextDirection: 'rtl'
+  TextDirection: rtl
   Thread: חוט
   Threads: חוטים
   ThreeMonths: 3 חדשים

data/web/locales/pt-BR.yml

@@ -1,4 +1,4 @@
-"pt-BR":
+pt-BR:
   LanguageName: Português (Brasil)
   Actions: Ações
   AddToQueue: Adicionar à fila

data/web/locales/ur.yml

@@ -69,7 +69,7 @@ ur:
   Stop: بند کرو
   StopAll: ﺗﻤﺎﻡ ﺑﻨﺪ کﺭﻭ
   StopPolling: ﺑﺮاﮦ ﺭاﺳﺖ روکيے
-  TextDirection: 'rtl'
+  TextDirection: rtl
   Thread: موضوع
   Threads: موضوع
   ThreeMonths: تین ماہ

data/web/views/_job_info.html.erb

@@ -34,6 +34,14 @@
             <code><%= job.jid %></code>
           </td>
         </tr>
+        <% if (state = job.iterable_state) %>
+        <tr>
+          <th>Iteration</th>
+          <td>
+            <code>cursor=<%= h(state.cursor.inspect) %>; exec=<%= state.executions %>; rt=<%= number_with_delimiter(state.runtime, precision: 3) %>s</code>
+          </td>
+        </tr>
+        <% end %>
         <% if job.bid %>
         <tr>
           <th>BID</th>

data/web/views/_paging.html.erb

@@ -10,7 +10,7 @@
           <a href="<%= url %>?<%= qparams(page: @current_page - 1) %>"><%= @current_page - 1 %></a>
         </li>
       <% end %>
-      <li class="disabled">
+      <li class="active">
         <a href="<%= url %>?<%= qparams(page: @current_page) %>"><%= @current_page %></a>
       </li>
       <% if @total_size > @current_page * @count %>

data/web/views/busy.html.erb

@@ -138,7 +138,12 @@
           <td>
             <code><div class="args"><%= display_args(job.display_args) %></div></code>
           </td>
-          <td><%= relative_time(work.run_at) %></td>
+          <td>
+            <%= relative_time(work.run_at) %>
+            <% if (state = @iterable_states[job.jid]) %>
+              <div><small>cursor=<%= h(truncate(state.cursor.inspect, 100)) %>; exec=<%= state.executions %>; rt=<%= number_with_delimiter(state.runtime, precision: 3) %>s</small></div>
+            <% end %>
+          </td>
         </tr>
       <% end %>
     </table>

data/web/views/retries.html.erb

@@ -52,6 +52,9 @@
               </td>
               <td>
                 <div><a href="<%= root_path %>retries/<%= job_params(entry.item, entry.score) %>"><%= h truncate("#{entry['error_class']}: #{entry['error_message']}", 200) %></a></div>
+                <% if (state = @iterable_states[entry.jid]) %>
+                  <div><small>cursor=<%= h(truncate(state.cursor.inspect, 100)) %>; exec=<%= state.executions %>; rt=<%= number_with_delimiter(state.runtime, precision: 3) %>s</small></div>
+                <% end %>
               </td>
             </tr>
           <% end %>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 8.1.2
+  version: 8.1.4
 platform: ruby
 authors:
 - Mike Perham