sidekiq 7.0.0.beta1 → 7.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31722cec9da8b32488242bbcf40b46f3e2eaf8f9e5f5acefb4be306e6a3521e1
-  data.tar.gz: 625639dd35227dd5c9e36d04e93afbb3f8ae6372c532625453a0d681fac19a65
+  metadata.gz: 98aab60f0a9bd14122cf62b2f31c17e41b67feb8c9fb13865ef0a16002d38ed7
+  data.tar.gz: 94038f0176ccb7785c6112cd41e915e4a1c7f66298a9670457a68e6b62f2710c
 SHA512:
-  metadata.gz: 9471e077c2122b7b8fe94d075d1203d83d463608485852237ede31f9307b57233bb6cb0ba867f8556c080d61b01a5eb5efb7f1e3d68835e5bd16adc2630c65e1
-  data.tar.gz: 95b6c47592ba8641b4398859dca9c80b0d644c9e61cffec5482ff5741047daa9523f9cc5efb1974e197f3f06cab07321a5aca8b1838635372b3ea26e37cfca45
+  metadata.gz: c41c724dc99aa3fdf49db46eeac0bec822bd6a67989792f4830900da7b74882c27837535a03a2ea6ed0575736fc895b23bb3440613d1f581e877bcbfe1c06934
+  data.tar.gz: 482af09f25f72f9fa0261f02e4e224dc58a400519f167f8cb0ea0dda4960b24a41b105731a4fcc2b81ff15f158497b9bf1cc86d55272ed8fe4ca7ed9029bc047

data/Changes.md

@@ -2,11 +2,27 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
-HEAD
+7.0.1
 ----------
 
-- Adjust CurrentAttributes to work with the String class name so we aren't referencing
-the Class within a Rails initializer [#5536]
+- Allow an embedding process to reuse its own heartbeat thread
+- Update zh-cn localization
+
+7.0.0
+----------
+
+- Embedded mode!
+- Capsules!!
+- Job Execution metrics!!!
+- See `docs/7.0-Upgrade.md` for release notes
+
+6.5.8
+----------
+
+- Fail if using a bad version of scout_apm [#5616]
+- Add pagination to Busy page [#5556]
+- Speed up WorkSet#each [#5559]
+- Adjust CurrentAttributes to work with the String class name so we aren't referencing the Class within a Rails initializer [#5536]
 
 6.5.7
 ----------

data/bin/sidekiqload

@@ -19,7 +19,7 @@ x = Sidekiq.configure_embed do |config|
 end
 
 class LoadWorker
-  include Sidekiq::Worker
+  include Sidekiq::Job
   sidekiq_options retry: 1
   sidekiq_retry_in do |x|
     1

data/lib/sidekiq/api.rb

@@ -1034,24 +1034,24 @@ module Sidekiq
 
     def each(&block)
       results = []
+      procs = nil
+      all_works = nil
+
       Sidekiq.redis do |conn|
-        procs = conn.sscan("processes").to_a
-        procs.sort.each do |key|
-          valid, workers = conn.pipelined { |pipeline|
-            pipeline.exists(key)
+        procs = conn.sscan("processes").to_a.sort
+        all_works = conn.pipelined do |pipeline|
+          procs.each do |key|
             pipeline.hgetall("#{key}:work")
-          }
-          next unless valid > 0
-          workers.each_pair do |tid, json|
-            hsh = Sidekiq.load_json(json)
-            p = hsh["payload"]
-            # avoid breaking API, this is a side effect of the JSON optimization in #4316
-            hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
-            results << [key, tid, hsh]
           end
         end
       end
 
+      procs.zip(all_works).each do |key, workers|
+        workers.each_pair do |tid, json|
+          results << [key, tid, Sidekiq.load_json(json)] unless json.empty?
+        end
+      end
+
       results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
     end
 

data/lib/sidekiq/capsule.rb

@@ -4,10 +4,10 @@ module Sidekiq
   # A Sidekiq::Capsule is the set of resources necessary to
   # process one or more queues with a given concurrency.
   # One "default" Capsule is started but the user may declare additional
-  # Capsules in the initializer.
+  # Capsules in their initializer.
   #
-  # To process a "single" queue with one thread so jobs are processed
-  # serially, you can do this:
+  # This capsule will pull jobs from the "single" queue and process
+  # the jobs with one thread, meaning the jobs will be processed serially.
   #
   # Sidekiq.configure_server do |config|
   #   config.capsule("single-threaded") do |cap|
@@ -38,7 +38,7 @@ module Sidekiq
     end
 
     def stop
-      fetcher&.bulk_requeue([], nil)
+      fetcher&.bulk_requeue([])
     end
 
     def queues=(val)

data/lib/sidekiq/cli.rb

@@ -280,6 +280,13 @@ module Sidekiq # :nodoc:
         cap.queues = opts[:queues]
         cap.concurrency = opts[:concurrency] || @config[:concurrency]
       end
+
+      opts[:capsules]&.each do |name, cap_config|
+        @config.capsule(name.to_s) do |cap|
+          cap.queues = cap_config[:queues]
+          cap.concurrency = cap_config[:concurrency]
+        end
+      end
     end
 
     def boot_application
@@ -339,10 +346,6 @@ module Sidekiq # :nodoc:
           opts[:concurrency] = Integer(arg)
         end
 
-        o.on "-d", "--daemon", "Daemonize process" do |arg|
-          puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
-        end
-
         o.on "-e", "--environment ENV", "Application environment" do |arg|
           opts[:environment] = arg
         end
@@ -372,15 +375,7 @@ module Sidekiq # :nodoc:
           opts[:config_file] = arg
         end
 
-        o.on "-L", "--logfile PATH", "path to writable logfile" do |arg|
-          puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
-        end
-
-        o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
-          puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
-        end
-
-        o.on "-V", "--version", "Print version and exit" do |arg|
+        o.on "-V", "--version", "Print version and exit" do
           puts "Sidekiq #{Sidekiq::VERSION}"
           die(0)
         end

data/lib/sidekiq/config.rb

@@ -33,7 +33,8 @@ module Sidekiq
       reloader: proc { |&block| block.call }
     }
 
-    ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
+    ERROR_HANDLER = ->(ex, ctx) {
+      cfg = ctx[:_config] || Sidekiq.default_configuration
       l = cfg.logger
       l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
       l.warn("#{ex.class.name}: #{ex.message}")
@@ -49,7 +50,6 @@ module Sidekiq
     end
 
     def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
-    attr_reader :options
     attr_reader :capsules
 
     # LEGACY: edits the default capsule
@@ -257,8 +257,9 @@ module Sidekiq
       if @options[:error_handlers].size == 0
         p ["!!!!!", ex]
       end
+      ctx[:_config] = self
       @options[:error_handlers].each do |handler|
-        handler.call(ex, ctx, self)
+        handler.call(ex, ctx)
       rescue => e
         l = logger
         l.error "!!! ERROR HANDLER THREW AN ERROR !!!"

data/lib/sidekiq/embedded.rb

@@ -15,9 +15,9 @@ module Sidekiq
       fire_event(:startup, reverse: false, reraise: true)
       @launcher = Sidekiq::Launcher.new(@config, embedded: true)
       @launcher.run
-      sleep 0.1 # pause to give threads time to spin up
+      sleep 0.2 # pause to give threads time to spin up
 
-      logger.info "Embedded mode running with #{Thread.list.size} threads"
+      logger.info "Sidekiq running embedded, total process thread count: #{Thread.list.size}"
       logger.debug { Thread.list.map(&:name) }
     end
 

data/lib/sidekiq/fetch.rb

@@ -50,7 +50,7 @@ module Sidekiq # :nodoc:
       UnitOfWork.new(queue, job, config) if queue
     end
 
-    def bulk_requeue(inprogress, _)
+    def bulk_requeue(inprogress)
       return if inprogress.empty?
 
       logger.debug { "Re-queueing terminated jobs" }

data/lib/sidekiq/launcher.rb

@@ -32,15 +32,17 @@ module Sidekiq
       @done = false
     end
 
-    def run
+    # Start this Sidekiq instance. If an embedding process already
+    # has a heartbeat thread, caller can use `async_beat: false`
+    # and instead have thread call Launcher#heartbeat every N seconds.
+    def run(async_beat: true)
       Sidekiq.freeze!
-      @thread = safe_thread("heartbeat", &method(:start_heartbeat))
+      @thread = safe_thread("heartbeat", &method(:start_heartbeat)) if async_beat
       @poller.start
       @managers.each(&:start)
     end
 
     # Stops this instance from processing any more jobs,
-    #
     def quiet
       return if @done
 
@@ -71,18 +73,30 @@ module Sidekiq
       @done
     end
 
+    # If embedding Sidekiq, you can have the process heartbeat
+    # call this method to regularly heartbeat rather than creating
+    # a separate thread.
+    def heartbeat
+      ❤
+    end
+
     private unless $TESTING
 
     BEAT_PAUSE = 10
 
     def start_heartbeat
       loop do
-        heartbeat
+        beat
         sleep BEAT_PAUSE
       end
       logger.info("Heartbeat stopping...")
     end
 
+    def beat
+      $0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ") unless @embedded
+      ❤
+    end
+
     def clear_heartbeat
       flush_stats
 
@@ -99,12 +113,6 @@ module Sidekiq
       # best effort, ignore network errors
     end
 
-    def heartbeat
-      $0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ") unless @embedded
-
-      ❤
-    end
-
     def flush_stats
       fails = Processor::FAILURE.reset
       procd = Processor::PROCESSED.reset

data/lib/sidekiq/manager.rb

@@ -104,7 +104,7 @@ module Sidekiq
         # contract says that jobs are run AT LEAST once. Process termination
         # is delayed until we're certain the jobs are back in Redis because
         # it is worse to lose a job than to run it twice.
-        capsule.fetcher.bulk_requeue(jobs, nil)
+        capsule.fetcher.bulk_requeue(jobs)
       end
 
       cleanup.each do |processor|

data/lib/sidekiq/paginator.rb

@@ -43,5 +43,13 @@ module Sidekiq
         end
       end
     end
+
+    def page_items(items, pageidx = 1, page_size = 25)
+      current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
+      pageidx = current_page - 1
+      starting = pageidx * page_size
+      items = items.to_a
+      [current_page, items.size, items[starting, page_size]]
+    end
   end
 end

data/lib/sidekiq/rails.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq/job"
+require "rails"
 
 module Sidekiq
   class Rails < ::Rails::Engine
@@ -22,7 +23,7 @@ module Sidekiq
 
     # By including the Options module, we allow AJs to directly control sidekiq features
     # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
-    # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
+    # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
     # manually retried, don't automatically die, etc.
     #
     #   class SomeJob < ActiveJob::Base
@@ -48,13 +49,6 @@ module Sidekiq
       end
     end
 
-    config.before_configuration do
-      dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
-      dep.deprecate_methods(Sidekiq.singleton_class,
-        default_worker_options: :default_job_options,
-        "default_worker_options=": :default_job_options=)
-    end
-
     # This hook happens after all initializers are run, just before returning
     # from config/environment.rb back to sidekiq/cli.rb.
     #

data/lib/sidekiq/testing.rb

@@ -211,7 +211,7 @@ module Sidekiq
     #
     # You can also clear and drain all job types:
     #
-    #   Sidekiq::Worker.clear_all # or .drain_all
+    #   Sidekiq::Job.clear_all # or .drain_all
     #
     # This can be useful to make sure jobs don't linger between tests:
     #

data/lib/sidekiq/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.0.0.beta1"
+  VERSION = "7.0.1"
+  MAJOR = 7
 end

data/lib/sidekiq/web/application.rb

@@ -74,6 +74,9 @@ module Sidekiq
     end
 
     get "/busy" do
+      @count = (params["count"] || 100).to_i
+      (@current_page, @total_size, @workset) = page_items(workset, params["page"], @count)
+
       erb(:busy)
     end
 

data/lib/sidekiq/web/helpers.rb

@@ -148,25 +148,15 @@ module Sidekiq
       @processes ||= Sidekiq::ProcessSet.new
     end
 
-    # Sorts processes by hostname following the natural sort order so that
-    # 'worker.1' < 'worker.2' < 'worker.10' < 'worker.20'
-    # '2.1.1.1' < '192.168.0.2' < '192.168.0.10'
+    # Sorts processes by hostname following the natural sort order
     def sorted_processes
       @sorted_processes ||= begin
         return processes unless processes.all? { |p| p["hostname"] }
 
-        split_characters = /[._-]+/
-
-        padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
-
         processes.to_a.sort_by do |process|
-          process["hostname"].split(split_characters).map do |substring|
-            # Left-pad the substring with '0' if it starts with a number or 'a'
-            # otherwise, so that '25' < 192' < 'a' ('025' < '192' < 'aaa')
-            padding_char = substring[0].match?(/\d/) ? "0" : "a"
-
-            substring.rjust(padding, padding_char)
-          end
+          # Kudos to `shurikk` on StackOverflow
+          # https://stackoverflow.com/a/15170063/575547
+          process["hostname"].split(/(\d+)/).map { |a| /\d+/.match?(a) ? a.to_i : a }
         end
       end
     end
@@ -333,6 +323,10 @@ module Sidekiq
       Time.now.utc.strftime("%H:%M:%S UTC")
     end
 
+    def pollable?
+      !(current_path == "" || current_path.starts_with?("metrics"))
+    end
+
     def retry_or_delete_or_kill(job, params)
       if params["retry"]
         job.retry

data/lib/sidekiq/worker_compatibility_alias.rb

@@ -1,4 +1,4 @@
-require "sidekiq/job"
+# frozen_string_literal: true
 
 module Sidekiq
   # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.

data/lib/sidekiq.rb

@@ -3,6 +3,30 @@
 require "sidekiq/version"
 fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.7.0." if RUBY_PLATFORM != "java" && Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.7.0")
 
+begin
+  require "sidekiq-ent/version"
+  fail <<~EOM  if Gem::Version.new(Sidekiq::Enterprise::VERSION).segments[0] != Sidekiq::MAJOR
+
+    Sidekiq Enterprise #{Sidekiq::Enterprise::VERSION} does not work with Sidekiq #{Sidekiq::VERSION}.
+    Starting with Sidekiq 7, major versions are synchronized so Sidekiq Enterprise 7 works with Sidekiq 7.
+    Use `bundle up sidekiq-ent` to upgrade.
+
+  EOM
+rescue LoadError
+end
+
+begin
+  require "sidekiq/pro/version"
+  fail <<~EOM  if Gem::Version.new(Sidekiq::Pro::VERSION).segments[0] != Sidekiq::MAJOR
+
+    Sidekiq Pro #{Sidekiq::Pro::VERSION} does not work with Sidekiq #{Sidekiq::VERSION}.
+    Starting with Sidekiq 7, major versions are synchronized so Sidekiq Pro 7 works with Sidekiq 7.
+    Use `bundle up sidekiq-pro` to upgrade.
+
+  EOM
+rescue LoadError
+end
+
 require "sidekiq/config"
 require "sidekiq/logger"
 require "sidekiq/client"
@@ -80,7 +104,7 @@ module Sidekiq
   end
 
   # Creates a Sidekiq::Config instance that is more tuned for embedding
-  # within an arbitrary Ruby process. Noteably it reduces concurrency by
+  # within an arbitrary Ruby process. Notably it reduces concurrency by
   # default so there is less contention for CPU time with other threads.
   #
   #   inst = Sidekiq.configure_embed do |config|

data/sidekiq.gemspec

@@ -41,10 +41,8 @@ Gem::Specification.new do |gem|
      ░░░░░░░░░  ░░░░░ ░░░░░░░░░░   ░░░░░░░░░░ ░░░░░   ░░░░ ░░░░░    ░░░░░░ ░░      ░░░      ░░    ░░░░░░
     
     
-    WARNING: This is a beta release, expect breakage!
-    
-    1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want to be a beta tester.
-    2. Read the release notes at https://github.com/mperham/sidekiq/blob/7-0/docs/7.0-Upgrade.md
+    1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want this new version.
+    2. Read the release notes at https://github.com/mperham/sidekiq/blob/main/docs/7.0-Upgrade.md
     3. Search for open/closed issues at https://github.com/mperham/sidekiq/issues/
     
     ####################################################

data/web/locales/zh-cn.yml

@@ -32,23 +32,22 @@ zh-cn: # <---- change this to your locale code
   Size: 容量
   Actions: 动作
   NextRetry: 下次重试
-  RetryCount: 重试次數
+  RetryCount: 重试次数
   RetryNow: 现在重试
   Kill: 终止
-  LastRetry: 最后一次重试
-  OriginallyFailed: 原本已失败
+  LastRetry: 上次重试
+  OriginallyFailed: 首次失败
   AreYouSure: 你确定？
   DeleteAll: 全部删除
   RetryAll: 全部重试
   KillAll: 全部终止
-  NoRetriesFound: 沒有发现可重试
+  NoRetriesFound: 没有发现可重试
   Error: 错误
-  ErrorBacktrace: 错误的回调追踪
   ErrorClass: 错误类别
   ErrorMessage: 错误消息
   ErrorBacktrace: 错误细节
   GoBack: ← 返回
-  NoScheduledFound: 沒有发现计划任务
+  NoScheduledFound: 没有发现计划任务
   When: 当
   ScheduledJobs: 计划任务
   idle: 闲置
@@ -64,26 +63,26 @@ zh-cn: # <---- change this to your locale code
   SixMonths: 六个月
   Failures: 失败
   DeadJobs: 已停滞任务
-  NoDeadJobsFound: 沒有发现任何已停滞的任务
+  NoDeadJobsFound: 没有发现任何已停滞的任务
   Dead: 已停滞
   Process: 进程
-  Processes: 处理中
+  Processes: 进程
   Name: 名称
   Thread: 线程
   Threads: 线程
   Jobs: 任务
-  Paused: 已暫停
-  Stop: 強制暫停
-  Quiet: 暫停
-  StopAll: 全部強制暫停
-  QuietAll: 全部暫停
-  PollingInterval: 輪詢週期
-  Plugins: 套件
-  NotYetEnqueued: 尚未進入佇列
-  CreatedAt: 建立時間
+  Paused: 已暂停
+  Stop: 强制暂停
+  Quiet: 暂停
+  StopAll: 全部强制暂停
+  QuietAll: 全部暂停
+  PollingInterval: 轮询周期
+  Plugins: 插件
+  NotYetEnqueued: 尚未进入队列
+  CreatedAt: 建立时间
   BackToApp: 回首頁
-  Latency: 延時
-  Pause: 暫停
+  Latency: 延迟
+  Pause: 暂停
   Unpause: 取消暂停
   Metrics: 指标
   NoDataFound: 无数据
@@ -92,3 +91,5 @@ zh-cn: # <---- change this to your locale code
   Context: 上下文
   Bucket: 桶
   NoJobMetricsFound: 无任务相关指标数据
+  Success: 成功
+  Failure: 失败

data/web/views/_poll_link.erb

@@ -1,4 +1,4 @@
-<% if current_path != '' %>
+<% if pollable? %>
     <a class="live-poll-start live-poll btn btn-primary"><%= t('LivePoll') %></a>
     <a class="live-poll-stop live-poll btn btn-primary active"><%= t('StopPolling') %></a>
 <% end %>

data/web/views/busy.erb

@@ -96,6 +96,11 @@
   <div class="col-sm-7">
     <h3><%= t('Jobs') %></h3>
   </div>
+  <% if @workset.size > 0 && @total_size > @count %>
+    <div class="col-sm-4">
+      <%= erb :_paging, locals: { url: "#{root_path}busy" } %>
+    </div>
+  <% end %>
 </div>
 
 <div class="table_container">
@@ -109,7 +114,7 @@
       <th><%= t('Arguments') %></th>
       <th><%= t('Started') %></th>
     </thead>
-    <% workset.each do |process, thread, msg| %>
+    <% @workset.each do |process, thread, msg| %>
       <% job = Sidekiq::JobRecord.new(msg['payload']) %>
       <tr>
         <td><%= process %></td>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 7.0.0.beta1
+  version: 7.0.1
 platform: ruby
 authors:
 - Mike Perham
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-26 00:00:00.000000000 Z
+date: 2022-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client
@@ -111,7 +111,6 @@ files:
 - lib/sidekiq/middleware/modules.rb
 - lib/sidekiq/monitor.rb
 - lib/sidekiq/paginator.rb
-- lib/sidekiq/pool.rb
 - lib/sidekiq/processor.rb
 - lib/sidekiq/rails.rb
 - lib/sidekiq/redis_client_adapter.rb
@@ -220,10 +219,8 @@ post_install_message: |2
    ░░░░░░░░░  ░░░░░ ░░░░░░░░░░   ░░░░░░░░░░ ░░░░░   ░░░░ ░░░░░    ░░░░░░ ░░      ░░░      ░░    ░░░░░░
 
 
-  WARNING: This is a beta release, expect breakage!
-
-  1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want to be a beta tester.
-  2. Read the release notes at https://github.com/mperham/sidekiq/blob/7-0/docs/7.0-Upgrade.md
+  1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want this new version.
+  2. Read the release notes at https://github.com/mperham/sidekiq/blob/main/docs/7.0-Upgrade.md
   3. Search for open/closed issues at https://github.com/mperham/sidekiq/issues/
 
   ####################################################
@@ -237,9 +234,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.2.32
 signing_key:

data/lib/sidekiq/pool.rb

@@ -1,7 +0,0 @@
-module Sidekiq
-  module PoolAccess
-    def redis_pool
-      Thread.current[:sidekiq_redis_pool] || (@redis ||= Sidekiq::RedisConnection.create)
-    end
-  end
-end