sidekiq 7.0.0.beta1 → 7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31722cec9da8b32488242bbcf40b46f3e2eaf8f9e5f5acefb4be306e6a3521e1
-  data.tar.gz: 625639dd35227dd5c9e36d04e93afbb3f8ae6372c532625453a0d681fac19a65
+  metadata.gz: 9c6762c97172b3f8e4b7cc1fd940756ee8796faf70ece8d5e9ede9e2dc7797fe
+  data.tar.gz: 905e0e1f381e1c40832ab2137589161fcccfb63c847ad36c9cca15765d952659
 SHA512:
-  metadata.gz: 9471e077c2122b7b8fe94d075d1203d83d463608485852237ede31f9307b57233bb6cb0ba867f8556c080d61b01a5eb5efb7f1e3d68835e5bd16adc2630c65e1
-  data.tar.gz: 95b6c47592ba8641b4398859dca9c80b0d644c9e61cffec5482ff5741047daa9523f9cc5efb1974e197f3f06cab07321a5aca8b1838635372b3ea26e37cfca45
+  metadata.gz: acfc9bb74585cc65ad1af27e263668ec154ac791baa99de2a1542cfe462a7b13544a758a7bd574d623088c9e683f705437cc4e4495f05e57770037d030214a44
+  data.tar.gz: 31320e6cd2fab0d253f3a32616f51360c4ad6a9ef30acef5f88a6ccc6eaf75e04b76ea12dd053dd29442d96fa49234f65e14801e7600cf0f300c023d75aecf43

data/Changes.md

@@ -2,9 +2,19 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
-HEAD
+main
 ----------
 
+- Embedded mode!
+- Capsules!!
+- Job Execution metrics!!!
+- See `docs/7.0-Upgrade.md` for release notes
+
+6-x
+----------
+
+- Add pagination to Busy page [#5556]
+- Speed up WorkSet#each [#5559]
 - Adjust CurrentAttributes to work with the String class name so we aren't referencing
 the Class within a Rails initializer [#5536]
 

data/bin/sidekiqload

@@ -19,7 +19,7 @@ x = Sidekiq.configure_embed do |config|
 end
 
 class LoadWorker
-  include Sidekiq::Worker
+  include Sidekiq::Job
   sidekiq_options retry: 1
   sidekiq_retry_in do |x|
     1

data/lib/sidekiq/api.rb

@@ -1034,24 +1034,24 @@ module Sidekiq
 
     def each(&block)
       results = []
+      procs = nil
+      all_works = nil
+
       Sidekiq.redis do |conn|
-        procs = conn.sscan("processes").to_a
-        procs.sort.each do |key|
-          valid, workers = conn.pipelined { |pipeline|
-            pipeline.exists(key)
+        procs = conn.sscan("processes").to_a.sort
+        all_works = conn.pipelined do |pipeline|
+          procs.each do |key|
             pipeline.hgetall("#{key}:work")
-          }
-          next unless valid > 0
-          workers.each_pair do |tid, json|
-            hsh = Sidekiq.load_json(json)
-            p = hsh["payload"]
-            # avoid breaking API, this is a side effect of the JSON optimization in #4316
-            hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
-            results << [key, tid, hsh]
           end
         end
       end
 
+      procs.zip(all_works).each do |key, workers|
+        workers.each_pair do |tid, json|
+          results << [key, tid, Sidekiq.load_json(json)] unless json.empty?
+        end
+      end
+
       results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
     end
 

data/lib/sidekiq/capsule.rb

@@ -4,10 +4,10 @@ module Sidekiq
   # A Sidekiq::Capsule is the set of resources necessary to
   # process one or more queues with a given concurrency.
   # One "default" Capsule is started but the user may declare additional
-  # Capsules in the initializer.
+  # Capsules in their initializer.
   #
-  # To process a "single" queue with one thread so jobs are processed
-  # serially, you can do this:
+  # This capsule will pull jobs from the "single" queue and process
+  # the jobs with one thread, meaning the jobs will be processed serially.
   #
   # Sidekiq.configure_server do |config|
   #   config.capsule("single-threaded") do |cap|
@@ -38,7 +38,7 @@ module Sidekiq
     end
 
     def stop
-      fetcher&.bulk_requeue([], nil)
+      fetcher&.bulk_requeue([])
     end
 
     def queues=(val)

data/lib/sidekiq/cli.rb

@@ -280,6 +280,13 @@ module Sidekiq # :nodoc:
         cap.queues = opts[:queues]
         cap.concurrency = opts[:concurrency] || @config[:concurrency]
       end
+
+      opts[:capsules]&.each do |name, cap_config|
+        @config.capsule(name.to_s) do |cap|
+          cap.queues = cap_config[:queues]
+          cap.concurrency = cap_config[:concurrency]
+        end
+      end
     end
 
     def boot_application
@@ -339,10 +346,6 @@ module Sidekiq # :nodoc:
           opts[:concurrency] = Integer(arg)
         end
 
-        o.on "-d", "--daemon", "Daemonize process" do |arg|
-          puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
-        end
-
         o.on "-e", "--environment ENV", "Application environment" do |arg|
           opts[:environment] = arg
         end
@@ -372,15 +375,7 @@ module Sidekiq # :nodoc:
           opts[:config_file] = arg
         end
 
-        o.on "-L", "--logfile PATH", "path to writable logfile" do |arg|
-          puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
-        end
-
-        o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
-          puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
-        end
-
-        o.on "-V", "--version", "Print version and exit" do |arg|
+        o.on "-V", "--version", "Print version and exit" do
           puts "Sidekiq #{Sidekiq::VERSION}"
           die(0)
         end

data/lib/sidekiq/config.rb

@@ -33,7 +33,8 @@ module Sidekiq
       reloader: proc { |&block| block.call }
     }
 
-    ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
+    ERROR_HANDLER = ->(ex, ctx) {
+      cfg = ctx[:_config] || Sidekiq.default_configuration
       l = cfg.logger
       l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
       l.warn("#{ex.class.name}: #{ex.message}")
@@ -49,7 +50,6 @@ module Sidekiq
     end
 
     def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
-    attr_reader :options
     attr_reader :capsules
 
     # LEGACY: edits the default capsule
@@ -257,8 +257,9 @@ module Sidekiq
       if @options[:error_handlers].size == 0
         p ["!!!!!", ex]
       end
+      ctx[:_config] = self
       @options[:error_handlers].each do |handler|
-        handler.call(ex, ctx, self)
+        handler.call(ex, ctx)
       rescue => e
         l = logger
         l.error "!!! ERROR HANDLER THREW AN ERROR !!!"

data/lib/sidekiq/fetch.rb

@@ -50,7 +50,7 @@ module Sidekiq # :nodoc:
       UnitOfWork.new(queue, job, config) if queue
     end
 
-    def bulk_requeue(inprogress, _)
+    def bulk_requeue(inprogress)
       return if inprogress.empty?
 
       logger.debug { "Re-queueing terminated jobs" }

data/lib/sidekiq/manager.rb

@@ -104,7 +104,7 @@ module Sidekiq
         # contract says that jobs are run AT LEAST once. Process termination
         # is delayed until we're certain the jobs are back in Redis because
         # it is worse to lose a job than to run it twice.
-        capsule.fetcher.bulk_requeue(jobs, nil)
+        capsule.fetcher.bulk_requeue(jobs)
       end
 
       cleanup.each do |processor|

data/lib/sidekiq/paginator.rb

@@ -43,5 +43,13 @@ module Sidekiq
         end
       end
     end
+
+    def page_items(items, pageidx = 1, page_size = 25)
+      current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
+      pageidx = current_page - 1
+      starting = pageidx * page_size
+      items = items.to_a
+      [current_page, items.size, items[starting, page_size]]
+    end
   end
 end

data/lib/sidekiq/rails.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq/job"
+require "rails"
 
 module Sidekiq
   class Rails < ::Rails::Engine
@@ -22,7 +23,7 @@ module Sidekiq
 
     # By including the Options module, we allow AJs to directly control sidekiq features
     # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
-    # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
+    # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
     # manually retried, don't automatically die, etc.
     #
     #   class SomeJob < ActiveJob::Base
@@ -48,13 +49,6 @@ module Sidekiq
       end
     end
 
-    config.before_configuration do
-      dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
-      dep.deprecate_methods(Sidekiq.singleton_class,
-        default_worker_options: :default_job_options,
-        "default_worker_options=": :default_job_options=)
-    end
-
     # This hook happens after all initializers are run, just before returning
     # from config/environment.rb back to sidekiq/cli.rb.
     #

data/lib/sidekiq/testing.rb

@@ -211,7 +211,7 @@ module Sidekiq
     #
     # You can also clear and drain all job types:
     #
-    #   Sidekiq::Worker.clear_all # or .drain_all
+    #   Sidekiq::Job.clear_all # or .drain_all
     #
     # This can be useful to make sure jobs don't linger between tests:
     #

data/lib/sidekiq/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.0.0.beta1"
+  VERSION = "7.0.0"
+  MAJOR = 7
 end

data/lib/sidekiq/web/application.rb

@@ -74,6 +74,9 @@ module Sidekiq
     end
 
     get "/busy" do
+      @count = (params["count"] || 100).to_i
+      (@current_page, @total_size, @workset) = page_items(workset, params["page"], @count)
+
       erb(:busy)
     end
 

data/lib/sidekiq/web/helpers.rb

@@ -148,25 +148,15 @@ module Sidekiq
       @processes ||= Sidekiq::ProcessSet.new
     end
 
-    # Sorts processes by hostname following the natural sort order so that
-    # 'worker.1' < 'worker.2' < 'worker.10' < 'worker.20'
-    # '2.1.1.1' < '192.168.0.2' < '192.168.0.10'
+    # Sorts processes by hostname following the natural sort order
     def sorted_processes
       @sorted_processes ||= begin
         return processes unless processes.all? { |p| p["hostname"] }
 
-        split_characters = /[._-]+/
-
-        padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
-
         processes.to_a.sort_by do |process|
-          process["hostname"].split(split_characters).map do |substring|
-            # Left-pad the substring with '0' if it starts with a number or 'a'
-            # otherwise, so that '25' < 192' < 'a' ('025' < '192' < 'aaa')
-            padding_char = substring[0].match?(/\d/) ? "0" : "a"
-
-            substring.rjust(padding, padding_char)
-          end
+          # Kudos to `shurikk` on StackOverflow
+          # https://stackoverflow.com/a/15170063/575547
+          process["hostname"].split(/(\d+)/).map { |a| /\d+/.match?(a) ? a.to_i : a }
         end
       end
     end

data/lib/sidekiq/worker_compatibility_alias.rb

@@ -1,4 +1,4 @@
-require "sidekiq/job"
+# frozen_string_literal: true
 
 module Sidekiq
   # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.

data/lib/sidekiq.rb

@@ -3,6 +3,30 @@
 require "sidekiq/version"
 fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.7.0." if RUBY_PLATFORM != "java" && Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.7.0")
 
+begin
+  require "sidekiq-ent/version"
+  fail <<~EOM  if Gem::Version.new(Sidekiq::Enterprise::VERSION).segments[0] != Sidekiq::MAJOR
+
+    Sidekiq Enterprise #{Sidekiq::Enterprise::VERSION} does not work with Sidekiq #{Sidekiq::VERSION}.
+    Starting with Sidekiq 7, major versions are synchronized so Sidekiq Enterprise 7 works with Sidekiq 7.
+    Use `bundle up sidekiq-ent` to upgrade.
+
+  EOM
+rescue LoadError
+end
+
+begin
+  require "sidekiq/pro/version"
+  fail <<~EOM  if Gem::Version.new(Sidekiq::Pro::VERSION).segments[0] != Sidekiq::MAJOR
+
+    Sidekiq Pro #{Sidekiq::Pro::VERSION} does not work with Sidekiq #{Sidekiq::VERSION}.
+    Starting with Sidekiq 7, major versions are synchronized so Sidekiq Pro 7 works with Sidekiq 7.
+    Use `bundle up sidekiq-pro` to upgrade.
+
+  EOM
+rescue LoadError
+end
+
 require "sidekiq/config"
 require "sidekiq/logger"
 require "sidekiq/client"
@@ -80,7 +104,7 @@ module Sidekiq
   end
 
   # Creates a Sidekiq::Config instance that is more tuned for embedding
-  # within an arbitrary Ruby process. Noteably it reduces concurrency by
+  # within an arbitrary Ruby process. Notably it reduces concurrency by
   # default so there is less contention for CPU time with other threads.
   #
   #   inst = Sidekiq.configure_embed do |config|

data/sidekiq.gemspec

@@ -44,7 +44,7 @@ Gem::Specification.new do |gem|
     WARNING: This is a beta release, expect breakage!
     
     1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want to be a beta tester.
-    2. Read the release notes at https://github.com/mperham/sidekiq/blob/7-0/docs/7.0-Upgrade.md
+    2. Read the release notes at https://github.com/mperham/sidekiq/blob/main/docs/7.0-Upgrade.md
     3. Search for open/closed issues at https://github.com/mperham/sidekiq/issues/
     
     ####################################################

data/web/views/busy.erb

@@ -96,6 +96,11 @@
   <div class="col-sm-7">
     <h3><%= t('Jobs') %></h3>
   </div>
+  <% if @workset.size > 0 && @total_size > @count %>
+    <div class="col-sm-4">
+      <%= erb :_paging, locals: { url: "#{root_path}busy" } %>
+    </div>
+  <% end %>
 </div>
 
 <div class="table_container">
@@ -109,7 +114,7 @@
       <th><%= t('Arguments') %></th>
       <th><%= t('Started') %></th>
     </thead>
-    <% workset.each do |process, thread, msg| %>
+    <% @workset.each do |process, thread, msg| %>
       <% job = Sidekiq::JobRecord.new(msg['payload']) %>
       <tr>
         <td><%= process %></td>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 7.0.0.beta1
+  version: 7.0.0
 platform: ruby
 authors:
 - Mike Perham
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-26 00:00:00.000000000 Z
+date: 2022-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client
@@ -111,7 +111,6 @@ files:
 - lib/sidekiq/middleware/modules.rb
 - lib/sidekiq/monitor.rb
 - lib/sidekiq/paginator.rb
-- lib/sidekiq/pool.rb
 - lib/sidekiq/processor.rb
 - lib/sidekiq/rails.rb
 - lib/sidekiq/redis_client_adapter.rb
@@ -223,7 +222,7 @@ post_install_message: |2
   WARNING: This is a beta release, expect breakage!
 
   1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want to be a beta tester.
-  2. Read the release notes at https://github.com/mperham/sidekiq/blob/7-0/docs/7.0-Upgrade.md
+  2. Read the release notes at https://github.com/mperham/sidekiq/blob/main/docs/7.0-Upgrade.md
   3. Search for open/closed issues at https://github.com/mperham/sidekiq/issues/
 
   ####################################################
@@ -237,9 +236,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.2.32
 signing_key:

data/lib/sidekiq/pool.rb

@@ -1,7 +0,0 @@
-module Sidekiq
-  module PoolAccess
-    def redis_pool
-      Thread.current[:sidekiq_redis_pool] || (@redis ||= Sidekiq::RedisConnection.create)
-    end
-  end
-end