sidekiq 6.5.5 → 6.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56d317d1dc6e27057146cee5e5951030fb907489025a0d14f1e11c7ae952befe
-  data.tar.gz: 7626890a494eb9f55374a24fa2dc0cf69193ddf31c00ef9a96c15e3a6aa88450
+  metadata.gz: a46b127af053d0d8e34960975ba328973778e74e5e11c972b7ed90ad3ff37e81
+  data.tar.gz: b879598b3c9219e52b2c61ecc33fc03670a779a9afc3c5651f99240021391551
 SHA512:
-  metadata.gz: 11fa0967bb8dcd19aa7c5893d151500ac2b8f06fb364fd957d5a01a8fe6842247e6da3b45d7103e95026ae7741265ef84881278bc55d6c1830480ee9e83cadd6
-  data.tar.gz: d625912d1f148c56b82da498cac73109532050d4e36c51ac9c3b229207a0e32ecbd34d3bc53823703951fa0ead9304113bdee654b8898d4cc2e4973621b8a268
+  metadata.gz: b30b0d0ddf63bf32cf9dadce9090112fcc0a26f3aa255fa49b784197a22ccc49280c323a7cbcccbd64fc7fb78b0273824c763bf7fc06dfcc0f310562099597de
+  data.tar.gz: e987d3347e7bd83a7abda98c72b1f9c79f7a75686e1895a4ad6e937c8eb3350a90fc03f010ea7c781b5a3609e5e255c2773cffda3e4c4570470ef8f1ed108316

data/Changes.md

@@ -2,6 +2,18 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
+6.5.7
+----------
+
+- Updates for JA and ZH locales
+- Further optimizations for scheduled polling [#5513]
+
+6.5.6
+----------
+
+- Fix deprecation warnings with redis-rb 4.8.0 [#5484]
+- Lock redis-rb to < 5.0 as we are moving to redis-client in Sidekiq 7.0
+
 6.5.5
 ----------
 

data/bin/sidekiqload

@@ -89,7 +89,7 @@ def Process.rss
   `ps -o rss= -p #{Process.pid}`.chomp.to_i
 end
 
-iter = 50
+iter = 10
 count = 10_000
 
 iter.times do
@@ -139,7 +139,7 @@ begin
   events.clear
   
   with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
-    launcher = Sidekiq::Launcher.new(Sidekiq.options)
+    launcher = Sidekiq::Launcher.new(Sidekiq)
     launcher.run
 
     while readable_io = IO.select([self_read])

data/lib/sidekiq/api.rb

@@ -11,6 +11,17 @@ if ENV["SIDEKIQ_METRICS_BETA"]
   require "sidekiq/metrics/query"
 end
 
+#
+# Sidekiq's Data API provides a Ruby object model on top
+# of Sidekiq's runtime data in Redis. This API should never
+# be used within application code for business logic.
+#
+# The Sidekiq server process never uses this API: all data
+# manipulation is done directly for performance reasons to
+# ensure we are using Redis as efficiently as possible at
+# every callsite.
+#
+
 module Sidekiq
   # Retrieve runtime statistics from Redis regarding
   # this Sidekiq cluster.
@@ -316,7 +327,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.multi do |transaction|
           transaction.unlink(@rname)
-          transaction.srem("queues", name)
+          transaction.srem("queues", [name])
         end
       end
       true
@@ -486,7 +497,7 @@ module Sidekiq
       # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
       # memory yet so the YAML can't be loaded.
       # TODO is this still necessary? Zeitwerk reloader should handle?
-      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.config[:environment] == "development"
+      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
       default
     end
 
@@ -893,10 +904,12 @@ module Sidekiq
     # :nodoc:
     # @api private
     def cleanup
+      # dont run cleanup more than once per minute
       return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+
       count = 0
       Sidekiq.redis do |conn|
-        procs = conn.sscan_each("processes").to_a.sort
+        procs = conn.sscan_each("processes").to_a
         heartbeats = conn.pipelined { |pipeline|
           procs.each do |key|
             pipeline.hget(key, "info")

data/lib/sidekiq/client.rb

@@ -231,7 +231,7 @@ module Sidekiq
           entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
         }
-        conn.sadd("queues", queue)
+        conn.sadd("queues", [queue])
         conn.lpush("queue:#{queue}", to_push)
       end
     end

data/lib/sidekiq/fetch.rb

@@ -33,7 +33,7 @@ module Sidekiq # :nodoc:
       @queues = @config[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
         @queues.uniq!
-        @queues << TIMEOUT
+        @queues << {timeout: TIMEOUT}
       end
     end
 
@@ -83,7 +83,7 @@ module Sidekiq # :nodoc:
       else
         permute = @queues.shuffle
         permute.uniq!
-        permute << TIMEOUT
+        permute << {timeout: TIMEOUT}
         permute
       end
     end

data/lib/sidekiq/launcher.rb

@@ -86,7 +86,7 @@ module Sidekiq
       # doesn't actually exit, it'll reappear in the Web UI.
       redis do |conn|
         conn.pipelined do |pipeline|
-          pipeline.srem("processes", identity)
+          pipeline.srem("processes", [identity])
           pipeline.unlink("#{identity}:work")
         end
       end
@@ -165,7 +165,7 @@ module Sidekiq
 
         _, exists, _, _, msg = redis { |conn|
           conn.multi { |transaction|
-            transaction.sadd("processes", key)
+            transaction.sadd("processes", [key])
             transaction.exists?(key)
             transaction.hmset(key, "info", to_json,
               "busy", curstate.size,

data/lib/sidekiq/redis_connection.rb

@@ -46,8 +46,6 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
-
         # Issue #3303, redis-rb will silently retry an operation.
         # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
         # is performed twice but I believe this is much, much rarer

data/lib/sidekiq/scheduled.rb

@@ -147,13 +147,16 @@ module Sidekiq
         # As we run more processes, the scheduling interval average will approach an even spread
         # between 0 and poll interval so we don't need this artifical boost.
         #
-        if process_count < 10
+        count = process_count
+        interval = poll_interval_average(count)
+
+        if count < 10
           # For small clusters, calculate a random interval that is ±50% the desired average.
-          poll_interval_average * rand + poll_interval_average.to_f / 2
+          interval * rand + interval.to_f / 2
         else
           # With 10+ processes, we should have enough randomness to get decent polling
           # across the entire timespan
-          poll_interval_average * rand
+          interval * rand
         end
       end
 
@@ -170,14 +173,14 @@ module Sidekiq
       # the same time: the thundering herd problem.
       #
       # We only do this if poll_interval_average is unset (the default).
-      def poll_interval_average
-        @config[:poll_interval_average] ||= scaled_poll_interval
+      def poll_interval_average(count)
+        @config[:poll_interval_average] || scaled_poll_interval(count)
       end
 
       # Calculates an average poll interval based on the number of known Sidekiq processes.
       # This minimizes a single point of failure by dispersing check-ins but without taxing
       # Redis if you run many Sidekiq processes.
-      def scaled_poll_interval
+      def scaled_poll_interval(process_count)
         process_count * @config[:average_scheduled_poll_interval]
       end
 
@@ -187,9 +190,35 @@ module Sidekiq
         pcount
       end
 
+      # A copy of Sidekiq::ProcessSet#cleanup because server
+      # should never depend on sidekiq/api.
+      def cleanup
+        # dont run cleanup more than once per minute
+        return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+
+        count = 0
+        Sidekiq.redis do |conn|
+          procs = conn.sscan_each("processes").to_a
+          heartbeats = conn.pipelined { |pipeline|
+            procs.each do |key|
+              pipeline.hget(key, "info")
+            end
+          }
+
+          # the hash named key has an expiry of 60 seconds.
+          # if it's not found, that means the process has not reported
+          # in to Redis and probably died.
+          to_prune = procs.select.with_index { |proc, i|
+            heartbeats[i].nil?
+          }
+          count = conn.srem("processes", to_prune) unless to_prune.empty?
+        end
+        count
+      end
+
       def initial_wait
-        # Have all processes sleep between 5-15 seconds.  10 seconds
-        # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
+        # Have all processes sleep between 5-15 seconds. 10 seconds to give time for
+        # the heartbeat to register (if the poll interval is going to be calculated by the number
         # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
         total = 0
         total += INITIAL_WAIT unless @config[:poll_interval_average]
@@ -197,6 +226,11 @@ module Sidekiq
 
         @sleeper.pop(total)
       rescue Timeout::Error
+      ensure
+        # periodically clean out the `processes` set in Redis which can collect
+        # references to dead processes over time. The process count affects how
+        # often we scan for scheduled jobs.
+        cleanup
       end
     end
   end

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.5.5"
+  VERSION = "6.5.7"
 end

data/lib/sidekiq/web/helpers.rb

@@ -155,7 +155,7 @@ module Sidekiq
       @sorted_processes ||= begin
         return processes unless processes.all? { |p| p["hostname"] }
 
-        split_characters = /[._-]/
+        split_characters = /[._-]+/
 
         padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
 

data/lib/sidekiq/worker.rb

@@ -340,7 +340,7 @@ module Sidekiq
       # Legal options:
       #
       #   queue - use a named queue for this Worker, default 'default'
-      #   retry - enable the RetryJobs middleware for this Worker, *true* to use the default
+      #   retry - enable retries via JobRetry, *true* to use the default
       #      or *Integer* count
       #   backtrace - whether to save any error backtrace in the retry payload to display in web UI,
       #      can be true, false or an integer number of lines to save, default *false*
@@ -348,6 +348,9 @@ module Sidekiq
       #
       # In practice, any option is allowed.  This is the main mechanism to configure the
       # options for a specific job.
+      #
+      # These options will be saved into the serialized job when enqueued by
+      # the client.
       def sidekiq_options(opts = {})
         super
       end

data/sidekiq.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
     "source_code_uri" => "https://github.com/mperham/sidekiq"
   }
 
-  gem.add_dependency "redis", ">= 4.5.0"
-  gem.add_dependency "connection_pool", ">= 2.2.2"
+  gem.add_dependency "redis", "<5", ">= 4.5.0"
+  gem.add_dependency "connection_pool", ">= 2.2.5"
   gem.add_dependency "rack", "~> 2.0"
 end

data/web/locales/ja.yml

@@ -84,3 +84,10 @@ ja:
   Latency: レイテンシ
   Pause: 一時停止
   Unpause: 一時停止を解除
+  Metrics: メトリクス
+  NoDataFound: データが見つかりませんでした
+  ExecutionTime: 合計実行時間
+  AvgExecutionTime: 平均実行時間
+  Context: コンテキスト
+  Bucket: バケット
+  NoJobMetricsFound: 直近のジョブメトリクスが見つかりませんでした

data/web/locales/zh-cn.yml

@@ -7,6 +7,7 @@ zh-cn: # <---- change this to your locale code
   Realtime: 实时
   History: 历史记录
   Busy: 执行中
+  Utilization: 利用率
   Processed: 已处理
   Failed: 已失败
   Scheduled: 已计划
@@ -17,15 +18,15 @@ zh-cn: # <---- change this to your locale code
   StopPolling: 停止轮询
   Queue: 队列
   Class: 类别
-  Job: 作业
+  Job: 任务
   Arguments: 参数
   Extras: 额外的
   Started: 已开始
   ShowAll: 显示全部
-  CurrentMessagesInQueue: 目前在<span class='title'>%{queue}</span>的作业
+  CurrentMessagesInQueue: 目前在<span class='title'>%{queue}</span>的任务
   Delete: 删除
   AddToQueue: 添加至队列
-  AreYouSureDeleteJob: 你确定要删除这个作业么？
+  AreYouSureDeleteJob: 你确定要删除这个任务么？
   AreYouSureDeleteQueue: 你确定要删除%{queue}这个队列？
   Queues: 队列
   Size: 容量
@@ -33,20 +34,22 @@ zh-cn: # <---- change this to your locale code
   NextRetry: 下次重试
   RetryCount: 重试次數
   RetryNow: 现在重试
+  Kill: 终止
   LastRetry: 最后一次重试
   OriginallyFailed: 原本已失败
   AreYouSure: 你确定？
-  DeleteAll: 删除全部
-  RetryAll: 重试全部
+  DeleteAll: 全部删除
+  RetryAll: 全部重试
+  KillAll: 全部终止
   NoRetriesFound: 沒有发现可重试
   Error: 错误
   ErrorClass: 错误类别
   ErrorMessage: 错误消息
-  ErrorBacktrace: 错误的回调追踪
+  ErrorBacktrace: 错误细节
   GoBack: ← 返回
-  NoScheduledFound: 沒有发现计划作业
+  NoScheduledFound: 沒有发现计划任务
   When: 当
-  ScheduledJobs: 计划作业
+  ScheduledJobs: 计划任务
   idle: 闲置
   active: 活动中
   Version: 版本
@@ -59,10 +62,32 @@ zh-cn: # <---- change this to your locale code
   ThreeMonths: 三个月
   SixMonths: 六个月
   Failures: 失败
-  DeadJobs: 已停滞作业
-  NoDeadJobsFound: 沒有发现任何已停滞的作业
+  DeadJobs: 已停滞任务
+  NoDeadJobsFound: 沒有发现任何已停滞的任务
   Dead: 已停滞
+  Process: 进程
   Processes: 处理中
+  Name: 名称
   Thread: 线程
   Threads: 线程
-  Jobs: 作业
+  Jobs: 任务
+  Paused: 已暫停
+  Stop: 強制暫停
+  Quiet: 暫停
+  StopAll: 全部強制暫停
+  QuietAll: 全部暫停
+  PollingInterval: 輪詢週期
+  Plugins: 套件
+  NotYetEnqueued: 尚未進入佇列
+  CreatedAt: 建立時間
+  BackToApp: 回首頁
+  Latency: 延時
+  Pause: 暫停
+  Unpause: 取消暂停
+  Metrics: 指标
+  NoDataFound: 无数据
+  TotalExecutionTime: 总执行时间
+  AvgExecutionTime: 平均执行时间
+  Context: 上下文
+  Bucket: 桶
+  NoJobMetricsFound: 无任务相关指标数据

data/web/locales/zh-tw.yml

@@ -7,6 +7,7 @@ zh-tw: # <---- change this to your locale code
   Realtime: 即時
   History: 歷史資料
   Busy: 忙碌
+  Utilization: 使用率
   Processed: 已處理
   Failed: 已失敗
   Scheduled: 已排程
@@ -25,26 +26,28 @@ zh-tw: # <---- change this to your locale code
   CurrentMessagesInQueue: 目前在<span class='title'>%{queue}</span>的工作
   Delete: 刪除
   AddToQueue: 增加至佇列
-  AreYouSureDeleteJob: 你確定要刪除這個工作嗎？
-  AreYouSureDeleteQueue: 你確定要刪除%{queue}這個佇列？
+  AreYouSureDeleteJob: 確定要刪除這個工作嗎？
+  AreYouSureDeleteQueue: 確定要刪除%{queue}佇列？這會刪除佇列裡的所有工作，佇列將會在有新工作時重新出現。
   Queues: 佇列
   Size: 容量
   Actions: 動作
   NextRetry: 下次重試
   RetryCount: 重試次數
   RetryNow: 馬上重試
+  Kill: 取消
   LastRetry: 最後一次重試
   OriginallyFailed: 原本已失敗
   AreYouSure: 你確定？
-  DeleteAll: 刪除全部
-  RetryAll: 重試全部
-  NoRetriesFound: 沒有發現可重試
+  DeleteAll: 全部刪除
+  RetryAll: 全部重試
+  KillAll: 全部取消
+  NoRetriesFound: 找無可重試的工作
   Error: 錯誤
   ErrorClass: 錯誤類別
   ErrorMessage: 錯誤訊息
-  ErrorBacktrace: 錯誤的回調追踨
+  ErrorBacktrace: 詳細錯誤訊息
   GoBack: ← 返回
-  NoScheduledFound: 沒有發現已排程的工作
+  NoScheduledFound: 找無已排程的工作
   When: 當
   ScheduledJobs: 已排程的工作
   idle: 閒置
@@ -62,7 +65,29 @@ zh-tw: # <---- change this to your locale code
   DeadJobs: 停滯工作
   NoDeadJobsFound: 沒有發現任何停滯的工作
   Dead: 停滯
+  Process: 程序
   Processes: 處理中
+  Name: 名稱
   Thread: 執行緒
   Threads: 執行緒
   Jobs: 工作
+  Paused: 已暫停
+  Stop: 強制暫停
+  Quiet: 暫停
+  StopAll: 全部強制暫停
+  QuietAll: 全部暫停
+  PollingInterval: 輪詢週期
+  Plugins: 套件
+  NotYetEnqueued: 尚未進入佇列
+  CreatedAt: 建立時間
+  BackToApp: 回首頁
+  Latency: 延時
+  Pause: 暫停
+  Unpause: 取消暫停
+  Metrics: 計量
+  NoDataFound: 找無資料
+  TotalExecutionTime: 總執行時間
+  AvgExecutionTime: 平均執行時間
+  Context: 上下文
+  Bucket: 桶
+  NoJobMetricsFound: 找無工作相關計量資料

metadata

@@ -1,19 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 6.5.5
+  version: 6.5.7
 platform: ruby
 authors:
 - Mike Perham
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-22 00:00:00.000000000 Z
+date: 2022-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.5.0
@@ -21,6 +24,9 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.5.0
@@ -30,14 +36,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.5
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement