sidekiq 6.0.0 → 6.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fda3ed37e1d981e1b32eea70669c678d400c071a9146a919848bcc598612de25
-  data.tar.gz: 6580ab6c188b2514efe807479ca25c06ea584d1def175d84d94639bc20f80b1e
+  metadata.gz: 792692164ce00b070d352bda8869518e3d20d92cfdb70cda88456b53d5a4a627
+  data.tar.gz: b489c2ecc7b3708dd578a0184a9da58ac32d36e737a3b704d094e1ef375e93e1
 SHA512:
-  metadata.gz: 25682d3dbf99d29fa3f56b7c74419863d3549b6e0c182e69c83fbc02a2955d053bca85eec53b474d4d22c7526e44e865f378aa4111152a91377b373be27c4d47
-  data.tar.gz: e55d07434b1e2ab26df6748101bf10640fc08d4d93f00b75ed04d82ed309ee8529d652982430cd00671c98b5e53cc733bef4b686df58a6b0e116ba04c12139aa
+  metadata.gz: 80d7bd6a6fc9ef0a6026c9c5bb8aaf727db7323e1f1d9e057550681329991ddf82de030df70dc80e8f3a3608fc1f999ecb5d251765a5bcf09f23f764d86365a7
+  data.tar.gz: 3148011168f88fbc4fa67f244a56be0b0e7da36d5009f4d465e0df2a7df93ddf8a99fc1c57c43fa0365a0cdb1de9fb770306daf070bcea5f330c1f3522210bf9

data/.circleci/config.yml

@@ -32,6 +32,9 @@ jobs:
       - <<: *save
       - <<: *unit
   "ruby-2.6":
+    environment:
+      COVERAGE: true
+      CC_TEST_REPORTER_ID: 003c3033501d70a2653bd887ff9a8b2884a263e6a4e27f2ba68748e15530918d
     docker:
       - image: circleci/ruby:2.6
       - image: circleci/redis:4.0
@@ -40,7 +43,25 @@ jobs:
       - <<: *restore
       - <<: *bundle
       - <<: *save
+
+      - run:
+          name: Setup Code Climate test-reporter
+          command: |
+            # download test reporter as a static binary
+            curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+            chmod +x ./cc-test-reporter
+
+      - run:
+          name: Code Climate before-build
+          command: |
+            ./cc-test-reporter before-build
+
       - <<: *unit
+
+      - run:
+          name: Report code coverage to Code Climate
+          command: |
+            ./cc-test-reporter after-build -t simplecov --exit-code $?
   "jruby":
     docker:
       - image: circleci/jruby:latest

data/6.0-Upgrade.md

@@ -10,6 +10,7 @@ This release has major breaking changes.  Read and test carefully in production.
 - ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
   features/internals like the retry subsystem. Prefer the native
   Sidekiq::Worker APIs as some Sidekiq features (e.g. unique jobs) do not work well with AJ.
+  (requires Rails 6.0.1)
 ```ruby
 class MyJob < ActiveJob::Base
   queue_as :myqueue
@@ -31,9 +32,10 @@ you can override it by configuring the log formatter explicitly. See
 ```ruby
 Sidekiq.configure_server do |config|
   config.log_formatter = AcmeCorp::PlainLogFormatter.new
-  # config.log_formatter = Sidekiq::Logger::Format::JSON.new
+  # config.log_formatter = Sidekiq::Logger::Formatters::JSON.new
 end
 ```
+Please see the [Logging](https://github.com/mperham/sidekiq/wiki/Logging) wiki page for the latest documentation and notes.
 - **Remove the daemonization, logfile and pidfile command line arguments and `sidekiqctl` binary**.
 I've [noted for years](https://www.mikeperham.com/2014/09/22/dont-daemonize-your-daemons/)
 how modern services should be managed with a proper init system.

data/Changes.md

@@ -2,12 +2,80 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
+HEAD
+---------
+
+- **Performance tuning**, Sidekiq should be 10-15% faster now [#4303, 4299,
+  4269, fatkodima]
+- **Dark Mode support in Web UI** (further design polish welcome!) [#4227, mperham,
+  fatkodima, silent-e]
+- **Job-specific log levels**, allowing you to turn on debugging for
+  problematic workers. [fatkodima, #4287]
+```ruby
+MyWorker.set(log_level: :debug).perform_async(...)
+```
+- **Ad-hoc job tags**. You can tag your jobs with, e.g, subdomain, tenant, country,
+  locale, application, version, user/client, "alpha/beta/pro/ent", types of jobs,
+  teams/people responsible for jobs, additional metadata, etc.
+  Tags are shown on different pages with job listings. Sidekiq Pro users
+  can filter based on them [fatkodima, #4280]
+```ruby
+class MyWorker
+  include Sidekiq::Worker
+  sidekiq_options tags: ['bank-ops', 'alpha']
+  ...
+end
+```
+- Fetch scheduled jobs in batches before pushing into specific queues.
+  This will decrease enqueueing time of scheduled jobs by a third. [fatkodima, #4273]
+```
+ScheduledSet with 10,000 jobs
+Before: 56.6 seconds
+After:  39.2 seconds
+```
+- Compress error backtraces before pushing into Redis, if you are
+  storing error backtraces, this will halve the size of your RetrySet
+  in Redis [fatkodima, #4272]
+```
+RetrySet with 100,000 jobs
+Before: 261 MB
+After:  129 MB
+```
+- Support display of ActiveJob 6.0 payloads in the Web UI [#4263]
+- Add `SortedSet#scan` for pattern based scanning. For large sets this API will be **MUCH** faster
+  than standard iteration using each. [fatkodima, #4262]
+```ruby
+  Sidekiq::DeadSet.new.scan("UnreliableApi") do |job|
+    job.retry
+  end
+```
+- Dramatically speed up SortedSet#find\_job(jid) by using Redis's ZSCAN
+  support, approx 10x faster. [fatkodima, #4259]
+```
+zscan   0.179366   0.047727   0.227093 (  1.161376)
+enum    8.522311   0.419826   8.942137 (  9.785079)
+```
+- Respect rails' generators `test_framework` option and gracefully handle extra `worker` suffix on generator [fatkodima, #4256]
+- Add ability to sort 'Enqueued' page on Web UI by position in the queue [fatkodima, #4248]
+- Support `Client.push_bulk` with different delays [fatkodima, #4243]
+```ruby
+Sidekiq::Client.push_bulk("class" => FooJob, "args" => [[1], [2]], "at" => [1.minute.from_now.to_f, 5.minutes.from_now.to_f])
+```
+- Easier way to test enqueuing specific ActionMailer and ActiveRecord delayed jobs. Instead of manually
+  parsing embedded class, you can now test by fetching jobs for specific classes. [fatkodima, #4292]
+```ruby
+assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs_for(FooMailer).size
+```
+- Add `sidekiqmon` to gemspec executables [#4242]
+- Gracefully handle `Sidekiq.logger = nil` [#4240]
+- Inject Sidekiq::LogContext module if user-supplied logger does not include it [#4239]
+
 6.0
 ---------
 
 This release has major breaking changes.  Read and test carefully in production.
 
-- ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
+- With Rails 6.0.1+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
   features/internals like the retry subsystem. [#4213, pirj]
 ```ruby
 class MyJob < ActiveJob::Base
@@ -17,6 +85,13 @@ class MyJob < ActiveJob::Base
   end
 end
 ```
+- Logging has been redesigned to allow for pluggable log formatters:
+```ruby
+Sidekiq.configure_server do |config|
+  config.log_formatter = Sidekiq::Logger::Formatters::JSON.new
+end
+```
+See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for more details.
 - **BREAKING CHANGE** Validate proper usage of the `REDIS_PROVIDER`
   variable.  This variable is meant to hold the name of the environment
   variable which contains your Redis URL, so that you can switch Redis

data/Ent-Changes.md

@@ -4,6 +4,12 @@
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+HEAD
+-------------
+
+- Periodic job registration API adjusted to avoid loading classes in
+  initializer [#4271]
+
 2.0.0
 -------------
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sidekiq (6.0.0)
+    sidekiq (6.0.1)
       connection_pool (>= 2.2.2)
       rack (>= 2.0.0)
       rack-protection (>= 2.0.0)
@@ -97,7 +97,7 @@ GEM
     nokogiri (1.10.4)
       mini_portile2 (~> 2.4.0)
     parallel (1.17.0)
-    parser (2.6.3.0)
+    parser (2.6.4.1)
       ast (~> 2.4.0)
     pry (0.12.2)
       coderay (~> 1.1.0)
@@ -164,7 +164,7 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.1)
-    standard (0.1.2)
+    standard (0.1.4)
       rubocop (~> 0.72.0)
       rubocop-performance (~> 1.4.0)
     thor (0.20.3)

data/Pro-Changes.md

@@ -4,12 +4,20 @@
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+5.0.1
+---------
+
+- Rejigger batch failures UI to add direct links to retries and scheduled jobs [#4209]
+- Delete batch data with `UNLINK` [#4155]
+- Fix bug where a scheduled job can lose its scheduled time when using reliable push [#4267]
+- Sidekiq::JobSet#scan and #find_job APIs have been promoted to Sidekiq OSS. [#4259]
+
 5.0.0
 ---------
 
 - There is no significant migration from Sidekiq Pro 4.0 to 5.0
   but make sure you read the [update notes for Sidekiq
-6.0](/mperham/sidekiq/blob/master/6.0-Upgrade.md).
+6.0](https://github.com/mperham/sidekiq/blob/master/6.0-Upgrade.md).
 - Removed various deprecated APIs and associated warnings.
 - **BREAKING CHANGE** Remove the `Sidekiq::Batch::Status#dead_jobs` API in favor of
   `Sidekiq::Batch::Status#dead_jids`. [#4217]

data/README.md

@@ -3,6 +3,7 @@ Sidekiq
 
 [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
 [![Code Climate](https://codeclimate.com/github/mperham/sidekiq.svg)](https://codeclimate.com/github/mperham/sidekiq)
+[![Test Coverage](https://codeclimate.com/github/mperham/sidekiq/badges/coverage.svg)](https://codeclimate.com/github/mperham/sidekiq/coverage)
 [![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
 [![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
 

data/bin/sidekiqload

@@ -5,7 +5,8 @@
 $TESTING = false
 
 #require 'ruby-prof'
-Bundler.require(:default)
+require 'bundler/setup'
+Bundler.require(:default, :load_test)
 
 require_relative '../lib/sidekiq/cli'
 require_relative '../lib/sidekiq/launcher'
@@ -102,17 +103,20 @@ iter.times do
 end
 Sidekiq.logger.error "Created #{count*iter} jobs"
 
+start = Time.now
+
 Monitoring = Thread.new do
   watchdog("monitor thread") do
     while true
-      sleep 0.5
+      sleep 0.2
       qsize = Sidekiq.redis do |conn|
         conn.llen "queue:default"
       end
       total = qsize
-      Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
+      #Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
       if total == 0
-        Sidekiq.logger.error("Done, now here's the latency for three jobs")
+        Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
+        Sidekiq.logger.error("Now here's the latency for three jobs")
 
         LoadWorker.perform_async(1, Time.now.to_f)
         LoadWorker.perform_async(2, Time.now.to_f)

data/bin/sidekiqmon

@@ -2,8 +2,7 @@
 
 require 'sidekiq/monitor'
 
-if ARGV[0] == 'status'
-  Sidekiq::Monitor::Status.new.display(ARGV[1])
-else
-  Sidekiq::Monitor.print_usage
-end
+section = "all"
+section = ARGV[0] if ARGV.size == 1
+
+Sidekiq::Monitor::Status.new.display(section)

data/lib/generators/sidekiq/worker_generator.rb

@@ -16,6 +16,8 @@ module Sidekiq
       end
 
       def create_test_file
+        return unless test_framework
+
         if defined?(RSpec)
           create_worker_spec
         else
@@ -42,6 +44,14 @@ module Sidekiq
         )
         template "worker_test.rb.erb", template_file
       end
+
+      def file_name
+        @_file_name ||= super.sub(/_?worker\z/i, "")
+      end
+
+      def test_framework
+        ::Rails.application.config.generators.options[:rails][:test_framework]
+      end
     end
   end
 end

data/lib/sidekiq.rb

@@ -192,6 +192,7 @@ module Sidekiq
 
   def self.log_formatter=(log_formatter)
     @log_formatter = log_formatter
+    logger.formatter = log_formatter
   end
 
   def self.logger
@@ -199,6 +200,13 @@ module Sidekiq
   end
 
   def self.logger=(logger)
+    if logger.nil?
+      self.logger.level = Logger::FATAL
+      return self.logger
+    end
+
+    logger.extend(Sidekiq::LoggingUtils)
+
     @logger = logger
   end
 

data/lib/sidekiq/api.rb

@@ -2,23 +2,11 @@
 
 require "sidekiq"
 
-module Sidekiq
-  module RedisScanner
-    def sscan(conn, key)
-      cursor = "0"
-      result = []
-      loop do
-        cursor, values = conn.sscan(key, cursor)
-        result.push(*values)
-        break if cursor == "0"
-      end
-      result
-    end
-  end
+require "zlib"
+require "base64"
 
+module Sidekiq
   class Stats
-    include RedisScanner
-
     def initialize
       fetch_stats!
     end
@@ -77,11 +65,11 @@ module Sidekiq
       }
 
       processes = Sidekiq.redis { |conn|
-        sscan(conn, "processes")
+        conn.sscan_each("processes").to_a
       }
 
       queues = Sidekiq.redis { |conn|
-        sscan(conn, "queues")
+        conn.sscan_each("queues").to_a
       }
 
       pipe2_res = Sidekiq.redis { |conn|
@@ -142,11 +130,9 @@ module Sidekiq
     end
 
     class Queues
-      include RedisScanner
-
       def lengths
         Sidekiq.redis do |conn|
-          queues = sscan(conn, "queues")
+          queues = conn.sscan_each("queues").to_a
 
           lengths = conn.pipelined {
             queues.each do |queue|
@@ -225,13 +211,12 @@ module Sidekiq
   #
   class Queue
     include Enumerable
-    extend RedisScanner
 
     ##
     # Return all known queues within Redis.
     #
     def self.all
-      Sidekiq.redis { |c| sscan(c, "queues") }.sort.map { |q| Sidekiq::Queue.new(q) }
+      Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
@@ -349,7 +334,7 @@ module Sidekiq
                    end
                  when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
                    job_class = @item["wrapped"] || args[0]
-                   if job_class == "ActionMailer::DeliveryJob"
+                   if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
                      # MailerClass#mailer_method
                      args[0]["arguments"][0..1].join("#")
                    else
@@ -372,6 +357,9 @@ module Sidekiq
                   if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
                     # remove MailerClass, mailer_method and 'deliver_now'
                     job_args.drop(3)
+                  elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
+                    # remove MailerClass, mailer_method and 'deliver_now'
+                    job_args.drop(3).first["args"]
                   else
                     job_args
                   end
@@ -400,6 +388,20 @@ module Sidekiq
       Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
     end
 
+    def tags
+      self["tags"] || []
+    end
+
+    def error_backtrace
+      # Cache nil values
+      if defined?(@error_backtrace)
+        @error_backtrace
+      else
+        value = self["error_backtrace"]
+        @error_backtrace = value && uncompress_backtrace(value)
+      end
+    end
+
     attr_reader :queue
 
     def latency
@@ -433,6 +435,17 @@ module Sidekiq
       Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
       default
     end
+
+    def uncompress_backtrace(backtrace)
+      if backtrace.is_a?(Array)
+        # Handle old jobs with previous backtrace format
+        backtrace
+      else
+        decoded = Base64.decode64(backtrace)
+        uncompressed = Zlib::Inflate.inflate(decoded)
+        Marshal.load(uncompressed)
+      end
+    end
   end
 
   class SortedEntry < Job
@@ -540,6 +553,17 @@ module Sidekiq
       Sidekiq.redis { |c| c.zcard(name) }
     end
 
+    def scan(match, count = 100)
+      return to_enum(:scan, match) unless block_given?
+
+      match = "*#{match}*" unless match.include?("*")
+      Sidekiq.redis do |conn|
+        conn.zscan_each(name, match: match, count: count) do |entry, score|
+          yield SortedEntry.new(self, score, entry)
+        end
+      end
+    end
+
     def clear
       Sidekiq.redis do |conn|
         conn.del(name)
@@ -576,28 +600,40 @@ module Sidekiq
       end
     end
 
+    ##
+    # Fetch jobs that match a given time or Range. Job ID is an
+    # optional second argument.
     def fetch(score, jid = nil)
+      begin_score, end_score =
+        if score.is_a?(Range)
+          [score.first, score.last]
+        else
+          [score, score]
+        end
+
       elements = Sidekiq.redis { |conn|
-        conn.zrangebyscore(name, score, score)
+        conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
       }
 
       elements.each_with_object([]) do |element, result|
-        entry = SortedEntry.new(self, score, element)
-        if jid
-          result << entry if entry.jid == jid
-        else
-          result << entry
-        end
+        data, job_score = element
+        entry = SortedEntry.new(self, job_score, data)
+        result << entry if jid.nil? || entry.jid == jid
       end
     end
 
     ##
     # Find the job with the given JID within this sorted set.
-    #
-    # This is a slow, inefficient operation.  Do not use under
-    # normal conditions.  Sidekiq Pro contains a faster version.
+    # This is a slower O(n) operation.  Do not use for app logic.
     def find_job(jid)
-      detect { |j| j.jid == jid }
+      Sidekiq.redis do |conn|
+        conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
+          job = JSON.parse(entry)
+          matched = job["jid"] == jid
+          return SortedEntry.new(self, score, entry) if matched
+        end
+      end
+      nil
     end
 
     def delete_by_value(name, value)
@@ -720,7 +756,6 @@ module Sidekiq
   #
   class ProcessSet
     include Enumerable
-    include RedisScanner
 
     def initialize(clean_plz = true)
       cleanup if clean_plz
@@ -731,7 +766,7 @@ module Sidekiq
     def cleanup
       count = 0
       Sidekiq.redis do |conn|
-        procs = sscan(conn, "processes").sort
+        procs = conn.sscan_each("processes").to_a.sort
         heartbeats = conn.pipelined {
           procs.each do |key|
             conn.hget(key, "info")
@@ -751,7 +786,7 @@ module Sidekiq
     end
 
     def each
-      procs = Sidekiq.redis { |conn| sscan(conn, "processes") }.sort
+      procs = Sidekiq.redis { |conn| conn.sscan_each("processes").to_a }.sort
 
       Sidekiq.redis do |conn|
         # We're making a tradeoff here between consuming more memory instead of
@@ -885,11 +920,10 @@ module Sidekiq
   #
   class Workers
     include Enumerable
-    include RedisScanner
 
     def each
       Sidekiq.redis do |conn|
-        procs = sscan(conn, "processes")
+        procs = conn.sscan_each("processes").to_a
         procs.sort.each do |key|
           valid, workers = conn.pipelined {
             conn.exists(key)
@@ -911,7 +945,7 @@ module Sidekiq
     # which can easily get out of sync with crashy processes.
     def size
       Sidekiq.redis do |conn|
-        procs = sscan(conn, "processes")
+        procs = conn.sscan_each("processes").to_a
         if procs.empty?
           0
         else