sidekiq 6.1.3 → 6.2.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: eafcca9d632a83e39c7e69e47d104493f32b07974d58724347181ea1d9c47511
4
- data.tar.gz: '08b987ddd7686f813f29a24d9d9027093f97307e7a22e24c355ed8b3d4c49362'
3
+ metadata.gz: 990d3e553aed906265ffa0cafb74fa4c79e0ccde957f21ccc3d09531d01e75bf
4
+ data.tar.gz: 92a68ab1ea824dc78b91610e98c3d10ce4a5794e7aebd66803dc6f29db419420
5
5
  SHA512:
6
- metadata.gz: 615b18ce20c57301cb4117e6fbeab847b6ba41377ed1ba22d98e32e2df579f1cef1138b0c2757a51a988b0ecef83e47992754fbdb5b96a6f6d653a294755e36c
7
- data.tar.gz: c3f3a44a85fc7a1c9405f59bba34842892c8a23e49a3ebac8dce5ba35b48738a9c5ba71635b671d2c6b9b711e5692d780a71ebf4a0b3c66ba8ac95c8ab9f319b
6
+ metadata.gz: 37664695dd79557af0395fe20855db1d250753dd8c874023a866c27d69e78bfa36b3f70501dad18a6ac64673e62fa8fdf413c77d0052a9ca0d59f25a17df41f9
7
+ data.tar.gz: 4d6ef75d6eb4be49e8b8d21d56bd0e300a284a4a6db9a0f0357724206b5e1b81b8bf75b53d8442319fd5fe5a9f08a467d75c59289d82d6aa70a00aa0d5102445
data/Changes.md CHANGED
@@ -2,6 +2,37 @@
2
2
 
3
3
  [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
4
4
 
5
+ 6.2.0
6
+ ---------
7
+
8
+ - Store Redis RTT and log if poor [#4824]
9
+ - Add process/thread stats to Busy page [#4806]
10
+ - Improve Web UI on mobile devices [#4840]
11
+ - **Refactor Web UI session usage** [#4804]
12
+ Numerous people have hit "Forbidden" errors and struggled with Sidekiq's
13
+ Web UI session requirement. If you have code in your initializer for
14
+ Web sessions, it's quite possible it will need to be removed. Here's
15
+ an overview:
16
+ ```
17
+ Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
18
+ make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so
19
+ Sidekiq can reuse the Rails session:
20
+
21
+ Rails.application.routes.draw do
22
+ mount Sidekiq::Web => "/sidekiq"
23
+ ....
24
+ end
25
+
26
+ If this is a bare Rack app, use a session middleware before Sidekiq::Web:
27
+
28
+ # first, use IRB to create a shared secret key for sessions and commit it
29
+ require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
30
+
31
+ # now, update your Rack app to include the secret with a session cookie middleware
32
+ use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
33
+ run Sidekiq::Web
34
+ ```
35
+
5
36
  6.1.3
6
37
  ---------
7
38
 
data/lib/sidekiq/api.rb CHANGED
@@ -791,12 +791,12 @@ module Sidekiq
791
791
  # you'll be happier this way
792
792
  conn.pipelined do
793
793
  procs.each do |key|
794
- conn.hmget(key, "info", "busy", "beat", "quiet", "rss")
794
+ conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
795
795
  end
796
796
  end
797
797
  }
798
798
 
799
- result.each do |info, busy, at_s, quiet, rss|
799
+ result.each do |info, busy, at_s, quiet, rss, rtt|
800
800
  # If a process is stopped between when we query Redis for `procs` and
801
801
  # when we query for `result`, we will have an item in `result` that is
802
802
  # composed of `nil` values.
@@ -806,7 +806,8 @@ module Sidekiq
806
806
  yield Process.new(hash.merge("busy" => busy.to_i,
807
807
  "beat" => at_s.to_f,
808
808
  "quiet" => quiet,
809
- "rss" => rss))
809
+ "rss" => rss.to_i,
810
+ "rtt_us" => rtt.to_i))
810
811
  end
811
812
  end
812
813
 
@@ -818,6 +819,17 @@ module Sidekiq
818
819
  Sidekiq.redis { |conn| conn.scard("processes") }
819
820
  end
820
821
 
822
+ # Total number of threads available to execute jobs.
823
+ # For Sidekiq Enterprise customers this number (in production) must be
824
+ # less than or equal to your licensed concurrency.
825
+ def total_concurrency
826
+ sum { |x| x["concurrency"] }
827
+ end
828
+
829
+ def total_rss
830
+ sum { |x| x["rss"] || 0 }
831
+ end
832
+
821
833
  # Returns the identity of the current cluster leader or "" if no leader.
822
834
  # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
823
835
  # or Sidekiq Pro.
data/lib/sidekiq/fetch.rb CHANGED
@@ -36,7 +36,15 @@ module Sidekiq
36
36
  end
37
37
 
38
38
  def retrieve_work
39
- work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
39
+ qs = queues_cmd
40
+ # 4825 Sidekiq Pro with all queues paused will return an
41
+ # empty set of queues with a trailing TIMEOUT value.
42
+ if qs.size <= 1
43
+ sleep(2)
44
+ return nil
45
+ end
46
+
47
+ work = Sidekiq.redis { |conn| conn.brpop(*qs) }
40
48
  UnitOfWork.new(*work) if work
41
49
  end
42
50
 
@@ -153,6 +153,8 @@ module Sidekiq
153
153
  end
154
154
  end
155
155
 
156
+ rtt = check_rtt
157
+
156
158
  fails = procd = 0
157
159
  kb = memory_usage(::Process.pid)
158
160
 
@@ -163,6 +165,7 @@ module Sidekiq
163
165
  conn.hmset(key, "info", to_json,
164
166
  "busy", curstate.size,
165
167
  "beat", Time.now.to_f,
168
+ "rtt_us", rtt,
166
169
  "quiet", @done,
167
170
  "rss", kb)
168
171
  conn.expire(key, 60)
@@ -185,6 +188,29 @@ module Sidekiq
185
188
  end
186
189
  end
187
190
 
191
+ RTT_WARNING_LEVEL = 50_000
192
+
193
+ def check_rtt
194
+ a = b = 0
195
+ Sidekiq.redis do |x|
196
+ a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
197
+ x.ping
198
+ b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
199
+ end
200
+ rtt = b - a
201
+ # Ideal RTT for Redis is < 1000µs
202
+ # Workable is < 10,000µs
203
+ # Log a warning if it's a disaster.
204
+ if rtt > RTT_WARNING_LEVEL
205
+ Sidekiq.logger.warn <<-EOM
206
+ Your Redis network connection is performing extremely poorly.
207
+ Current RTT is #{rtt} µs, ideally this should be < 1000.
208
+ Ensure Redis is running in the same AZ or datacenter as Sidekiq.
209
+ EOM
210
+ end
211
+ rtt
212
+ end
213
+
188
214
  MEMORY_GRABBER = case RUBY_PLATFORM
189
215
  when /linux/
190
216
  ->(pid) {
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Sidekiq
4
- VERSION = "6.1.3"
4
+ VERSION = "6.2.0"
5
5
  end
data/lib/sidekiq/web.rb CHANGED
@@ -13,10 +13,8 @@ require "sidekiq/web/application"
13
13
  require "sidekiq/web/csrf_protection"
14
14
 
15
15
  require "rack/content_length"
16
-
17
16
  require "rack/builder"
18
- require "rack/file"
19
- require "rack/session/cookie"
17
+ require "rack/static"
20
18
 
21
19
  module Sidekiq
22
20
  class Web
@@ -40,14 +38,6 @@ module Sidekiq
40
38
  self
41
39
  end
42
40
 
43
- def middlewares
44
- @middlewares ||= []
45
- end
46
-
47
- def use(*middleware_args, &block)
48
- middlewares << [middleware_args, block]
49
- end
50
-
51
41
  def default_tabs
52
42
  DEFAULT_TABS
53
43
  end
@@ -73,32 +63,45 @@ module Sidekiq
73
63
  opts.each { |key| set(key, false) }
74
64
  end
75
65
 
76
- # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
66
+ def middlewares
67
+ @middlewares ||= []
68
+ end
69
+
70
+ def use(*args, &block)
71
+ middlewares << [args, block]
72
+ end
73
+
77
74
  def set(attribute, value)
78
75
  send(:"#{attribute}=", value)
79
76
  end
80
77
 
81
- attr_accessor :app_url, :session_secret, :redis_pool, :sessions
78
+ def sessions=(val)
79
+ puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
80
+ end
81
+
82
+ def session_secret=(val)
83
+ puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
84
+ end
85
+
86
+ attr_accessor :app_url, :redis_pool
82
87
  attr_writer :locales, :views
83
88
  end
84
89
 
85
90
  def self.inherited(child)
86
91
  child.app_url = app_url
87
- child.session_secret = session_secret
88
92
  child.redis_pool = redis_pool
89
- child.sessions = sessions
90
93
  end
91
94
 
92
95
  def settings
93
96
  self.class.settings
94
97
  end
95
98
 
96
- def use(*middleware_args, &block)
97
- middlewares << [middleware_args, block]
99
+ def middlewares
100
+ @middlewares ||= self.class.middlewares
98
101
  end
99
102
 
100
- def middlewares
101
- @middlewares ||= Web.middlewares.dup
103
+ def use(*args, &block)
104
+ middlewares << [args, block]
102
105
  end
103
106
 
104
107
  def call(env)
@@ -126,18 +129,8 @@ module Sidekiq
126
129
  send(:"#{attribute}=", value)
127
130
  end
128
131
 
129
- # Default values
130
- set :sessions, true
131
-
132
- attr_writer :sessions
133
-
134
- def sessions
135
- unless instance_variable_defined?("@sessions")
136
- @sessions = self.class.sessions
137
- @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
138
- end
139
-
140
- @sessions
132
+ def sessions=(val)
133
+ puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
141
134
  end
142
135
 
143
136
  def self.register(extension)
@@ -146,57 +139,19 @@ module Sidekiq
146
139
 
147
140
  private
148
141
 
149
- def using?(middleware)
150
- middlewares.any? do |(m, _)|
151
- m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
152
- end
153
- end
154
-
155
- def build_sessions
156
- middlewares = self.middlewares
157
-
158
- s = sessions
159
-
160
- # turn on CSRF protection if sessions are enabled and this is not the test env
161
- if s && !using?(CsrfProtection) && ENV["RACK_ENV"] != "test"
162
- middlewares.unshift [[CsrfProtection], nil]
163
- end
164
-
165
- if s && !using?(::Rack::Session::Cookie)
166
- unless (secret = Web.session_secret)
167
- require "securerandom"
168
- secret = SecureRandom.hex(64)
169
- end
170
-
171
- options = {secret: secret}
172
- options = options.merge(s.to_hash) if s.respond_to? :to_hash
173
-
174
- middlewares.unshift [[::Rack::Session::Cookie, options], nil]
175
- end
176
-
177
- # Since Sidekiq::WebApplication no longer calculates its own
178
- # Content-Length response header, we must ensure that the Rack middleware
179
- # that does this is loaded
180
- unless using? ::Rack::ContentLength
181
- middlewares.unshift [[::Rack::ContentLength], nil]
182
- end
183
- end
184
-
185
142
  def build
186
- build_sessions
187
-
188
- middlewares = self.middlewares
189
143
  klass = self.class
144
+ m = middlewares
190
145
 
191
146
  ::Rack::Builder.new do
192
- %w[stylesheets javascripts images].each do |asset_dir|
193
- map "/#{asset_dir}" do
194
- run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
195
- end
196
- end
197
-
198
- middlewares.each { |middleware, block| use(*middleware, &block) }
199
-
147
+ use Rack::Static, :urls => ["/stylesheets", "/images", "/javascripts"],
148
+ :root => ASSETS,
149
+ :cascade => true,
150
+ :header_rules => [
151
+ [:all, {'Cache-Control' => 'public, max-age=86400'}],
152
+ ]
153
+ m.each { |middleware, block| use(*middleware, &block) }
154
+ use Sidekiq::Web::CsrfProtection unless $TESTING
200
155
  run WebApplication.new(klass)
201
156
  end
202
157
  end
@@ -4,7 +4,6 @@ module Sidekiq
4
4
  class WebApplication
5
5
  extend WebRouter
6
6
 
7
- CONTENT_LENGTH = "Content-Length"
8
7
  REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
9
8
  CSP_HEADER = [
10
9
  "default-src 'self' https: http:",
@@ -42,6 +41,13 @@ module Sidekiq
42
41
  # nothing, backwards compatibility
43
42
  end
44
43
 
44
+ head "/" do
45
+ # HEAD / is the cheapest heartbeat possible,
46
+ # it hits Redis to ensure connectivity
47
+ Sidekiq.redis { |c| c.llen("queue:default") }
48
+ ""
49
+ end
50
+
45
51
  get "/" do
46
52
  @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
47
53
  stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
@@ -66,7 +66,28 @@ module Sidekiq
66
66
  end
67
67
 
68
68
  def session(env)
69
- env["rack.session"] || fail("you need to set up a session middleware *before* #{self.class}")
69
+ env["rack.session"] || fail(<<~EOM)
70
+ Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
71
+ make sure you mount Sidekiq::Web *inside* your application routes:
72
+
73
+
74
+ Rails.application.routes.draw do
75
+ mount Sidekiq::Web => "/sidekiq"
76
+ ....
77
+ end
78
+
79
+
80
+ If this is a bare Rack app, use a session middleware before Sidekiq::Web:
81
+
82
+
83
+ # first, use IRB to create a shared secret key for sessions and commit it
84
+ require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
85
+
86
+
87
+ # now use the secret with a session cookie middleware
88
+ use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
89
+ run Sidekiq::Web
90
+ EOM
70
91
  end
71
92
 
72
93
  def accept?(env)
@@ -22,6 +22,14 @@ module Sidekiq
22
22
  end
23
23
  end
24
24
 
25
+ def singularize(str, count)
26
+ if count == 1 && str.respond_to?(:singularize) # rails
27
+ str.singularize
28
+ else
29
+ str
30
+ end
31
+ end
32
+
25
33
  def clear_caches
26
34
  @strings = nil
27
35
  @locale_files = nil
@@ -158,8 +166,7 @@ module Sidekiq
158
166
 
159
167
  def redis_connection
160
168
  Sidekiq.redis do |conn|
161
- c = conn.connection
162
- "redis://#{c[:location]}/#{c[:db]}"
169
+ conn.connection[:id]
163
170
  end
164
171
  end
165
172
 
@@ -259,12 +266,14 @@ module Sidekiq
259
266
  end
260
267
 
261
268
  def format_memory(rss_kb)
262
- return "" if rss_kb.nil? || rss_kb == 0
269
+ return "0" if rss_kb.nil? || rss_kb == 0
263
270
 
264
271
  if rss_kb < 100_000
265
272
  "#{number_with_delimiter(rss_kb)} KB"
266
- else
273
+ elsif rss_kb < 10_000_000
267
274
  "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
275
+ else
276
+ "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
268
277
  end
269
278
  end
270
279
 
@@ -15,6 +15,10 @@ module Sidekiq
15
15
  REQUEST_METHOD = "REQUEST_METHOD"
16
16
  PATH_INFO = "PATH_INFO"
17
17
 
18
+ def head(path, &block)
19
+ route(HEAD, path, &block)
20
+ end
21
+
18
22
  def get(path, &block)
19
23
  route(GET, path, &block)
20
24
  end
@@ -39,7 +43,6 @@ module Sidekiq
39
43
  @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
40
44
 
41
45
  @routes[method] << WebRoute.new(method, path, block)
42
- @routes[HEAD] << WebRoute.new(method, path, block) if method == GET
43
46
  end
44
47
 
45
48
  def match(env)