sidekiq 6.1.2 → 6.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a80310735fb458e7332a5ecc86072248e0777e1951f77f14965c06313c5b2f8
-  data.tar.gz: bfd09dc57c4506ca9c4d74dd9a38e88ef18700aafaf92814d662c6df4e4b2e5b
+  metadata.gz: eafcca9d632a83e39c7e69e47d104493f32b07974d58724347181ea1d9c47511
+  data.tar.gz: '08b987ddd7686f813f29a24d9d9027093f97307e7a22e24c355ed8b3d4c49362'
 SHA512:
-  metadata.gz: c0e416fe575874c2051e88c64a6cbb30ad2ba4929ca33487ca1323b1963101d3d08ed43b8edbe874bccfa677521b0745baf589b451c7c56ff41b7bc70ee4f88d
-  data.tar.gz: 4d1cd6d03768f9844bd8edd9bdb545a743084322f4edc01682e04b0990b0fb6aa3f1c6ad4f9a87c1b810a07e03b08eae0405ad2291b42d58eebb257bed7a60f4
+  metadata.gz: 615b18ce20c57301cb4117e6fbeab847b6ba41377ed1ba22d98e32e2df579f1cef1138b0c2757a51a988b0ecef83e47992754fbdb5b96a6f6d653a294755e36c
+  data.tar.gz: c3f3a44a85fc7a1c9405f59bba34842892c8a23e49a3ebac8dce5ba35b48738a9c5ba71635b671d2c6b9b711e5692d780a71ebf4a0b3c66ba8ac95c8ab9f319b

data/5.0-Upgrade.md

@@ -14,7 +14,7 @@ versions of Ruby and Rails and adds support for RTL languages in the Web UI.
   integrated as Sidekiq middleware; the logging/retry logic had to be pulled out
   too.  Sidekiq 4.2 had a hack to make it work but this redesign provides
   a cleaner integration. [#3235]
-* The Delayed Extensions `delay`, `delay_in` and `delay_until` APIs are
+* The Delayed Extensions `delay`, `delay_for` and `delay_until` APIs are
   no longer available by default.  The extensions allow you to marshal
   job arguments as YAML, leading to cases where job payloads could be many
   100s of KB or larger if not careful, leading to Redis networking

data/Changes.md

@@ -2,6 +2,12 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
+6.1.3
+---------
+
+- Warn if Redis is configured to evict data under memory pressure [#4752]
+- Add process RSS on the Busy page [#4717]
+
 6.1.2
 ---------
 
@@ -150,7 +156,7 @@ assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs_for(FooMailer).size
 
 This release has major breaking changes.  Read and test carefully in production.
 
-- With Rails 6.0.1+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
+- With Rails 6.0.2+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
   features/internals like the retry subsystem. [#4213, pirj]
 ```ruby
 class MyJob < ActiveJob::Base

data/Ent-Changes.md

@@ -4,6 +4,34 @@
 
 Please see [sidekiq.org](https://sidekiq.org) for more details and how to buy.
 
+2.2.1
+-------------
+
+- Support configurable timezones for periodic jobs [#4749]
+- Handle edge case leading to negative expiry in uniqueness [#4763]
+
+2.2.0
+-------------
+
+- Add new **leaky bucket** rate limiter [#4414]
+  This allows clients to burst up to X calls before throttling
+  back to X calls per Y seconds. To limit the user to 60 calls
+  per minute:
+```ruby
+leaker = Sidekiq::Limiter.leaky("shopify", 60, :minute)
+leaker.within_limit do
+  ...
+end
+```
+  See the Rate Limiting wiki page for more detail.
+- Rate limiters may now customize their reschedule count [#4725]
+  To disable rate limit reschedules, use `reschedule: 0`.
+```ruby
+Sidekiq::Limiter.concurrent("somename", 5, reschedule: 0)
+```
+- Allow filtering by name in the Rate Limiter UI [#4695]
+- Add IT locale
+
 2.1.2
 -------------
 

data/Gemfile.lock

@@ -91,12 +91,12 @@ GEM
     mimemagic (0.3.5)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
-    minitest (5.14.1)
+    minitest (5.14.3)
     nio4r (2.5.2)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    parallel (1.20.1)
+    parser (2.7.2.0)
       ast (~> 2.4.1)
     rack (2.2.3)
     rack-test (1.1.0)
@@ -129,29 +129,30 @@ GEM
       thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
-    redis (4.2.1)
-    redis-namespace (1.8.0)
+    redis (4.2.5)
+    redis-namespace (1.6.0)
       redis (>= 3.0.4)
-    regexp_parser (1.7.1)
+    regexp_parser (2.0.0)
     rexml (3.2.4)
-    rubocop (0.89.1)
+    rubocop (1.4.2)
       parallel (~> 1.10)
-      parser (>= 2.7.1.1)
+      parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8)
       rexml
-      rubocop-ast (>= 0.3.0, < 1.0)
+      rubocop-ast (>= 1.1.1)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.3.0)
-      parser (>= 2.7.1.4)
-    rubocop-performance (1.7.1)
-      rubocop (>= 0.82.0)
+    rubocop-ast (1.3.0)
+      parser (>= 2.7.1.5)
+    rubocop-performance (1.9.1)
+      rubocop (>= 0.90.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.10.1)
     simplecov (0.19.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
-    simplecov-html (0.12.2)
+    simplecov-html (0.12.3)
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -160,9 +161,9 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
-    standard (0.5.2)
-      rubocop (~> 0.89.1)
-      rubocop-performance (~> 1.7.1)
+    standard (0.10.2)
+      rubocop (= 1.4.2)
+      rubocop-performance (= 1.9.1)
     thor (1.0.1)
     thread_safe (0.3.6)
     toxiproxy (1.0.3)

data/Pro-Changes.md

@@ -4,6 +4,14 @@
 
 Please see [sidekiq.org](https://sidekiq.org/) for more details and how to buy.
 
+5.2.1
+---------
+
+- Propagate death callbacks to parent batches [#4774]
+- Allow customization of Batch linger to quickly reclaim memory in Redis [#4772]
+- Fix disappearing processes in Busy due to super_fetch initialization when used in
+  tandem with `SIDEKIQ_PRELOAD_APP=1` in `sidekiqswarm`. [#4733]
+
 5.2.0
 ---------
 

data/lib/sidekiq/api.rb

@@ -85,10 +85,10 @@ module Sidekiq
 
       default_queue_latency = if (entry = pipe1_res[6].first)
         job = begin
-                Sidekiq.load_json(entry)
-              rescue
-                {}
-              end
+          Sidekiq.load_json(entry)
+        rescue
+          {}
+        end
         now = Time.now.to_f
         thence = job["enqueued_at"] || now
         now - thence
@@ -791,19 +791,22 @@ module Sidekiq
         # you'll be happier this way
         conn.pipelined do
           procs.each do |key|
-            conn.hmget(key, "info", "busy", "beat", "quiet")
+            conn.hmget(key, "info", "busy", "beat", "quiet", "rss")
           end
         end
       }
 
-      result.each do |info, busy, at_s, quiet|
+      result.each do |info, busy, at_s, quiet, rss|
         # If a process is stopped between when we query Redis for `procs` and
         # when we query for `result`, we will have an item in `result` that is
         # composed of `nil` values.
         next if info.nil?
 
         hash = Sidekiq.load_json(info)
-        yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
+        yield Process.new(hash.merge("busy" => busy.to_i,
+                                     "beat" => at_s.to_f,
+                                     "quiet" => quiet,
+                                     "rss" => rss))
       end
     end
 

data/lib/sidekiq/cli.rb

@@ -59,9 +59,22 @@ module Sidekiq
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
-      ver = Sidekiq.redis_info["redis_version"]
+      info = Sidekiq.redis_info
+      ver = info["redis_version"]
       raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
+      maxmemory_policy = info["maxmemory_policy"]
+      if maxmemory_policy != "noeviction"
+        logger.warn <<~EOM
+
+
+          WARNING: Your Redis instance will evict Sidekiq data under heavy load.
+          The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
+          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+
+        EOM
+      end
+
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
       cursize = Sidekiq.redis_pool.size

data/lib/sidekiq/client.rb

@@ -19,7 +19,7 @@ module Sidekiq
     #
     def middleware(&block)
       @chain ||= Sidekiq.client_middleware
-      if block_given?
+      if block
         @chain = @chain.dup
         yield @chain
       end

data/lib/sidekiq/job_retry.rb

@@ -61,6 +61,7 @@ module Sidekiq
   #
   class JobRetry
     class Handled < ::RuntimeError; end
+
     class Skip < Handled; end
 
     include Sidekiq::Util

data/lib/sidekiq/launcher.rb

@@ -154,12 +154,17 @@ module Sidekiq
         end
 
         fails = procd = 0
+        kb = memory_usage(::Process.pid)
 
         _, exists, _, _, msg = Sidekiq.redis { |conn|
           conn.multi {
             conn.sadd("processes", key)
             conn.exists?(key)
-            conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
+            conn.hmset(key, "info", to_json,
+              "busy", curstate.size,
+              "beat", Time.now.to_f,
+              "quiet", @done,
+              "rss", kb)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
           }
@@ -180,6 +185,26 @@ module Sidekiq
       end
     end
 
+    MEMORY_GRABBER = case RUBY_PLATFORM
+    when /linux/
+      ->(pid) {
+        IO.readlines("/proc/#{$$}/status").each do |line|
+          next unless line.start_with?("VmRSS:")
+          break line.split[1].to_i
+        end
+      }
+    when /darwin|bsd/
+      ->(pid) {
+        `ps -o pid,rss -p #{pid}`.lines.last.split.last.to_i
+      }
+    else
+      ->(pid) { 0 }
+    end
+
+    def memory_usage(pid)
+      MEMORY_GRABBER.call(pid)
+    end
+
     def to_data
       @data ||= begin
         {

data/lib/sidekiq/logger.rb

@@ -6,10 +6,11 @@ require "time"
 module Sidekiq
   module Context
     def self.with(hash)
+      orig_context = current.dup
       current.merge!(hash)
       yield
     ensure
-      hash.each_key { |key| current.delete(key) }
+      Thread.current[:sidekiq_context] = orig_context
     end
 
     def self.current
@@ -89,7 +90,7 @@ module Sidekiq
       return true if @logdev.nil? || severity < level
 
       if message.nil?
-        if block_given?
+        if block
           message = yield
         else
           message = progname

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.1.2"
+  VERSION = "6.1.3"
 end

data/lib/sidekiq/web/application.rb

@@ -316,7 +316,7 @@ module Sidekiq
     end
 
     def self.helpers(mod = nil, &block)
-      if block_given?
+      if block
         WebAction.class_eval(&block)
       else
         WebAction.send(:include, mod)

data/lib/sidekiq/web/csrf_protection.rb

@@ -90,13 +90,11 @@ module Sidekiq
         end
 
         sess = session(env)
-
-        # Checks that Rack::Session::Cookie did not return empty session
-        # object in case the digest verification failed
-        return false if sess.empty?
-
         localtoken = sess[:csrf]
 
+        # Checks that Rack::Session::Cookie actualy contains the csrf toekn
+        return false if localtoken.nil?
+
         # Rotate the session token after every use
         sess[:csrf] = SecureRandom.base64(TOKEN_LENGTH)
 

data/lib/sidekiq/web/helpers.rb

@@ -258,7 +258,19 @@ module Sidekiq
       end
     end
 
+    def format_memory(rss_kb)
+      return "" if rss_kb.nil? || rss_kb == 0
+
+      if rss_kb < 100_000
+        "#{number_with_delimiter(rss_kb)} KB"
+      else
+        "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
+      end
+    end
+
     def number_with_delimiter(number)
+      return "" if number.nil?
+
       begin
         Float(number)
       rescue ArgumentError, TypeError

data/sidekiq.gemspec

@@ -5,7 +5,7 @@ Gem::Specification.new do |gem|
   gem.email = ["mperham@gmail.com"]
   gem.summary = "Simple, efficient background processing for Ruby"
   gem.description = "Simple, efficient background processing for Ruby."
-  gem.homepage = "http://sidekiq.org"
+  gem.homepage = "https://sidekiq.org"
   gem.license = "LGPL-3.0"
 
   gem.executables = ["sidekiq", "sidekiqmon"]

data/web/assets/stylesheets/application-dark.css

@@ -75,6 +75,12 @@ a.btn {
   color: #000;
 }
 
+input {
+  background-color: #444;
+  color: #ccc;
+  padding: 3px;
+}
+
 .summary_bar .summary {
   background-color: #222;
   border: 1px solid #555;

data/web/assets/stylesheets/application.css

@@ -1152,3 +1152,15 @@ div.interval-slider input {
 .delete-confirm {
   width: 20%;
 }
+
+.circled {
+  border-color: #333;
+  color: #eee;
+  background-color: #b1003e;
+  border-radius: 50%;
+  text-align: center;
+  vertical-align: middle;
+  padding: 3px 7px;
+  font-size: 0.7em;
+  margin-left: 5px;
+}

data/web/locales/fr.yml

@@ -72,7 +72,7 @@ fr:
   Quiet: Clore
   StopAll: Tout arrêter
   QuietAll: Tout clore
-  PollingInterval: Interval de rafraîchissement
+  PollingInterval: Intervalle de rafraîchissement
   Plugins: Plugins
   NotYetEnqueued: Pas encore en file d'attente
   CreatedAt: Créée le

data/web/views/busy.erb

@@ -18,6 +18,7 @@
     <thead>
       <th><%= t('Name') %></th>
       <th><%= t('Started') %></th>
+      <th><%= t('RSS') %><a href="https://github.com/mperham/sidekiq/wiki/Memory#rss"><span class="circled" title="Click to learn more about RSS">?</span></a></th>
       <th><%= t('Threads') %></th>
       <th><%= t('Busy') %></th>
       <th>&nbsp;</th>
@@ -42,6 +43,7 @@
           <%= process['queues'] * ", " %>
         </td>
         <td><%= relative_time(Time.at(process['started_at'])) %></td>
+        <td><%= format_memory(process['rss'].to_i) %></td>
         <td><%= process['concurrency'] %></td>
         <td><%= process['busy'] %></td>
         <td>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 6.1.2
+  version: 6.1.3
 platform: ruby
 authors:
 - Mike Perham
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-07 00:00:00.000000000 Z
+date: 2021-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -186,7 +186,7 @@ files:
 - web/views/retry.erb
 - web/views/scheduled.erb
 - web/views/scheduled_job_info.erb
-homepage: http://sidekiq.org
+homepage: https://sidekiq.org
 licenses:
 - LGPL-3.0
 metadata: {}
@@ -205,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Simple, efficient background processing for Ruby