sidekiq 6.1.3 → 6.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: eafcca9d632a83e39c7e69e47d104493f32b07974d58724347181ea1d9c47511
4
- data.tar.gz: '08b987ddd7686f813f29a24d9d9027093f97307e7a22e24c355ed8b3d4c49362'
3
+ metadata.gz: 990d3e553aed906265ffa0cafb74fa4c79e0ccde957f21ccc3d09531d01e75bf
4
+ data.tar.gz: 92a68ab1ea824dc78b91610e98c3d10ce4a5794e7aebd66803dc6f29db419420
5
5
  SHA512:
6
- metadata.gz: 615b18ce20c57301cb4117e6fbeab847b6ba41377ed1ba22d98e32e2df579f1cef1138b0c2757a51a988b0ecef83e47992754fbdb5b96a6f6d653a294755e36c
7
- data.tar.gz: c3f3a44a85fc7a1c9405f59bba34842892c8a23e49a3ebac8dce5ba35b48738a9c5ba71635b671d2c6b9b711e5692d780a71ebf4a0b3c66ba8ac95c8ab9f319b
6
+ metadata.gz: 37664695dd79557af0395fe20855db1d250753dd8c874023a866c27d69e78bfa36b3f70501dad18a6ac64673e62fa8fdf413c77d0052a9ca0d59f25a17df41f9
7
+ data.tar.gz: 4d6ef75d6eb4be49e8b8d21d56bd0e300a284a4a6db9a0f0357724206b5e1b81b8bf75b53d8442319fd5fe5a9f08a467d75c59289d82d6aa70a00aa0d5102445
data/Changes.md CHANGED
@@ -2,6 +2,37 @@
2
2
 
3
3
  [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
4
4
 
5
+ 6.2.0
6
+ ---------
7
+
8
+ - Store Redis RTT and log if poor [#4824]
9
+ - Add process/thread stats to Busy page [#4806]
10
+ - Improve Web UI on mobile devices [#4840]
11
+ - **Refactor Web UI session usage** [#4804]
12
+ Numerous people have hit "Forbidden" errors and struggled with Sidekiq's
13
+ Web UI session requirement. If you have code in your initializer for
14
+ Web sessions, it's quite possible it will need to be removed. Here's
15
+ an overview:
16
+ ```
17
+ Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
18
+ make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so
19
+ Sidekiq can reuse the Rails session:
20
+
21
+ Rails.application.routes.draw do
22
+ mount Sidekiq::Web => "/sidekiq"
23
+ ....
24
+ end
25
+
26
+ If this is a bare Rack app, use a session middleware before Sidekiq::Web:
27
+
28
+ # first, use IRB to create a shared secret key for sessions and commit it
29
+ require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
30
+
31
+ # now, update your Rack app to include the secret with a session cookie middleware
32
+ use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
33
+ run Sidekiq::Web
34
+ ```
35
+
5
36
  6.1.3
6
37
  ---------
7
38
 
data/lib/sidekiq/api.rb CHANGED
@@ -791,12 +791,12 @@ module Sidekiq
791
791
  # you'll be happier this way
792
792
  conn.pipelined do
793
793
  procs.each do |key|
794
- conn.hmget(key, "info", "busy", "beat", "quiet", "rss")
794
+ conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
795
795
  end
796
796
  end
797
797
  }
798
798
 
799
- result.each do |info, busy, at_s, quiet, rss|
799
+ result.each do |info, busy, at_s, quiet, rss, rtt|
800
800
  # If a process is stopped between when we query Redis for `procs` and
801
801
  # when we query for `result`, we will have an item in `result` that is
802
802
  # composed of `nil` values.
@@ -806,7 +806,8 @@ module Sidekiq
806
806
  yield Process.new(hash.merge("busy" => busy.to_i,
807
807
  "beat" => at_s.to_f,
808
808
  "quiet" => quiet,
809
- "rss" => rss))
809
+ "rss" => rss.to_i,
810
+ "rtt_us" => rtt.to_i))
810
811
  end
811
812
  end
812
813
 
@@ -818,6 +819,17 @@ module Sidekiq
818
819
  Sidekiq.redis { |conn| conn.scard("processes") }
819
820
  end
820
821
 
822
+ # Total number of threads available to execute jobs.
823
+ # For Sidekiq Enterprise customers this number (in production) must be
824
+ # less than or equal to your licensed concurrency.
825
+ def total_concurrency
826
+ sum { |x| x["concurrency"] }
827
+ end
828
+
829
+ def total_rss
830
+ sum { |x| x["rss"] || 0 }
831
+ end
832
+
821
833
  # Returns the identity of the current cluster leader or "" if no leader.
822
834
  # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
823
835
  # or Sidekiq Pro.
data/lib/sidekiq/fetch.rb CHANGED
@@ -36,7 +36,15 @@ module Sidekiq
36
36
  end
37
37
 
38
38
  def retrieve_work
39
- work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
39
+ qs = queues_cmd
40
+ # 4825 Sidekiq Pro with all queues paused will return an
41
+ # empty set of queues with a trailing TIMEOUT value.
42
+ if qs.size <= 1
43
+ sleep(2)
44
+ return nil
45
+ end
46
+
47
+ work = Sidekiq.redis { |conn| conn.brpop(*qs) }
40
48
  UnitOfWork.new(*work) if work
41
49
  end
42
50
 
@@ -153,6 +153,8 @@ module Sidekiq
153
153
  end
154
154
  end
155
155
 
156
+ rtt = check_rtt
157
+
156
158
  fails = procd = 0
157
159
  kb = memory_usage(::Process.pid)
158
160
 
@@ -163,6 +165,7 @@ module Sidekiq
163
165
  conn.hmset(key, "info", to_json,
164
166
  "busy", curstate.size,
165
167
  "beat", Time.now.to_f,
168
+ "rtt_us", rtt,
166
169
  "quiet", @done,
167
170
  "rss", kb)
168
171
  conn.expire(key, 60)
@@ -185,6 +188,29 @@ module Sidekiq
185
188
  end
186
189
  end
187
190
 
191
+ RTT_WARNING_LEVEL = 50_000
192
+
193
+ def check_rtt
194
+ a = b = 0
195
+ Sidekiq.redis do |x|
196
+ a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
197
+ x.ping
198
+ b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
199
+ end
200
+ rtt = b - a
201
+ # Ideal RTT for Redis is < 1000µs
202
+ # Workable is < 10,000µs
203
+ # Log a warning if it's a disaster.
204
+ if rtt > RTT_WARNING_LEVEL
205
+ Sidekiq.logger.warn <<-EOM
206
+ Your Redis network connection is performing extremely poorly.
207
+ Current RTT is #{rtt} µs, ideally this should be < 1000.
208
+ Ensure Redis is running in the same AZ or datacenter as Sidekiq.
209
+ EOM
210
+ end
211
+ rtt
212
+ end
213
+
188
214
  MEMORY_GRABBER = case RUBY_PLATFORM
189
215
  when /linux/
190
216
  ->(pid) {
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Sidekiq
4
- VERSION = "6.1.3"
4
+ VERSION = "6.2.0"
5
5
  end
data/lib/sidekiq/web.rb CHANGED
@@ -13,10 +13,8 @@ require "sidekiq/web/application"
13
13
  require "sidekiq/web/csrf_protection"
14
14
 
15
15
  require "rack/content_length"
16
-
17
16
  require "rack/builder"
18
- require "rack/file"
19
- require "rack/session/cookie"
17
+ require "rack/static"
20
18
 
21
19
  module Sidekiq
22
20
  class Web
@@ -40,14 +38,6 @@ module Sidekiq
40
38
  self
41
39
  end
42
40
 
43
- def middlewares
44
- @middlewares ||= []
45
- end
46
-
47
- def use(*middleware_args, &block)
48
- middlewares << [middleware_args, block]
49
- end
50
-
51
41
  def default_tabs
52
42
  DEFAULT_TABS
53
43
  end
@@ -73,32 +63,45 @@ module Sidekiq
73
63
  opts.each { |key| set(key, false) }
74
64
  end
75
65
 
76
- # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
66
+ def middlewares
67
+ @middlewares ||= []
68
+ end
69
+
70
+ def use(*args, &block)
71
+ middlewares << [args, block]
72
+ end
73
+
77
74
  def set(attribute, value)
78
75
  send(:"#{attribute}=", value)
79
76
  end
80
77
 
81
- attr_accessor :app_url, :session_secret, :redis_pool, :sessions
78
+ def sessions=(val)
79
+ puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
80
+ end
81
+
82
+ def session_secret=(val)
83
+ puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
84
+ end
85
+
86
+ attr_accessor :app_url, :redis_pool
82
87
  attr_writer :locales, :views
83
88
  end
84
89
 
85
90
  def self.inherited(child)
86
91
  child.app_url = app_url
87
- child.session_secret = session_secret
88
92
  child.redis_pool = redis_pool
89
- child.sessions = sessions
90
93
  end
91
94
 
92
95
  def settings
93
96
  self.class.settings
94
97
  end
95
98
 
96
- def use(*middleware_args, &block)
97
- middlewares << [middleware_args, block]
99
+ def middlewares
100
+ @middlewares ||= self.class.middlewares
98
101
  end
99
102
 
100
- def middlewares
101
- @middlewares ||= Web.middlewares.dup
103
+ def use(*args, &block)
104
+ middlewares << [args, block]
102
105
  end
103
106
 
104
107
  def call(env)
@@ -126,18 +129,8 @@ module Sidekiq
126
129
  send(:"#{attribute}=", value)
127
130
  end
128
131
 
129
- # Default values
130
- set :sessions, true
131
-
132
- attr_writer :sessions
133
-
134
- def sessions
135
- unless instance_variable_defined?("@sessions")
136
- @sessions = self.class.sessions
137
- @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
138
- end
139
-
140
- @sessions
132
+ def sessions=(val)
133
+ puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
141
134
  end
142
135
 
143
136
  def self.register(extension)
@@ -146,57 +139,19 @@ module Sidekiq
146
139
 
147
140
  private
148
141
 
149
- def using?(middleware)
150
- middlewares.any? do |(m, _)|
151
- m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
152
- end
153
- end
154
-
155
- def build_sessions
156
- middlewares = self.middlewares
157
-
158
- s = sessions
159
-
160
- # turn on CSRF protection if sessions are enabled and this is not the test env
161
- if s && !using?(CsrfProtection) && ENV["RACK_ENV"] != "test"
162
- middlewares.unshift [[CsrfProtection], nil]
163
- end
164
-
165
- if s && !using?(::Rack::Session::Cookie)
166
- unless (secret = Web.session_secret)
167
- require "securerandom"
168
- secret = SecureRandom.hex(64)
169
- end
170
-
171
- options = {secret: secret}
172
- options = options.merge(s.to_hash) if s.respond_to? :to_hash
173
-
174
- middlewares.unshift [[::Rack::Session::Cookie, options], nil]
175
- end
176
-
177
- # Since Sidekiq::WebApplication no longer calculates its own
178
- # Content-Length response header, we must ensure that the Rack middleware
179
- # that does this is loaded
180
- unless using? ::Rack::ContentLength
181
- middlewares.unshift [[::Rack::ContentLength], nil]
182
- end
183
- end
184
-
185
142
  def build
186
- build_sessions
187
-
188
- middlewares = self.middlewares
189
143
  klass = self.class
144
+ m = middlewares
190
145
 
191
146
  ::Rack::Builder.new do
192
- %w[stylesheets javascripts images].each do |asset_dir|
193
- map "/#{asset_dir}" do
194
- run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
195
- end
196
- end
197
-
198
- middlewares.each { |middleware, block| use(*middleware, &block) }
199
-
147
+ use Rack::Static, :urls => ["/stylesheets", "/images", "/javascripts"],
148
+ :root => ASSETS,
149
+ :cascade => true,
150
+ :header_rules => [
151
+ [:all, {'Cache-Control' => 'public, max-age=86400'}],
152
+ ]
153
+ m.each { |middleware, block| use(*middleware, &block) }
154
+ use Sidekiq::Web::CsrfProtection unless $TESTING
200
155
  run WebApplication.new(klass)
201
156
  end
202
157
  end
@@ -4,7 +4,6 @@ module Sidekiq
4
4
  class WebApplication
5
5
  extend WebRouter
6
6
 
7
- CONTENT_LENGTH = "Content-Length"
8
7
  REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
9
8
  CSP_HEADER = [
10
9
  "default-src 'self' https: http:",
@@ -42,6 +41,13 @@ module Sidekiq
42
41
  # nothing, backwards compatibility
43
42
  end
44
43
 
44
+ head "/" do
45
+ # HEAD / is the cheapest heartbeat possible,
46
+ # it hits Redis to ensure connectivity
47
+ Sidekiq.redis { |c| c.llen("queue:default") }
48
+ ""
49
+ end
50
+
45
51
  get "/" do
46
52
  @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
47
53
  stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
@@ -66,7 +66,28 @@ module Sidekiq
66
66
  end
67
67
 
68
68
  def session(env)
69
- env["rack.session"] || fail("you need to set up a session middleware *before* #{self.class}")
69
+ env["rack.session"] || fail(<<~EOM)
70
+ Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
71
+ make sure you mount Sidekiq::Web *inside* your application routes:
72
+
73
+
74
+ Rails.application.routes.draw do
75
+ mount Sidekiq::Web => "/sidekiq"
76
+ ....
77
+ end
78
+
79
+
80
+ If this is a bare Rack app, use a session middleware before Sidekiq::Web:
81
+
82
+
83
+ # first, use IRB to create a shared secret key for sessions and commit it
84
+ require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
85
+
86
+
87
+ # now use the secret with a session cookie middleware
88
+ use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
89
+ run Sidekiq::Web
90
+ EOM
70
91
  end
71
92
 
72
93
  def accept?(env)
@@ -22,6 +22,14 @@ module Sidekiq
22
22
  end
23
23
  end
24
24
 
25
+ def singularize(str, count)
26
+ if count == 1 && str.respond_to?(:singularize) # rails
27
+ str.singularize
28
+ else
29
+ str
30
+ end
31
+ end
32
+
25
33
  def clear_caches
26
34
  @strings = nil
27
35
  @locale_files = nil
@@ -158,8 +166,7 @@ module Sidekiq
158
166
 
159
167
  def redis_connection
160
168
  Sidekiq.redis do |conn|
161
- c = conn.connection
162
- "redis://#{c[:location]}/#{c[:db]}"
169
+ conn.connection[:id]
163
170
  end
164
171
  end
165
172
 
@@ -259,12 +266,14 @@ module Sidekiq
259
266
  end
260
267
 
261
268
  def format_memory(rss_kb)
262
- return "" if rss_kb.nil? || rss_kb == 0
269
+ return "0" if rss_kb.nil? || rss_kb == 0
263
270
 
264
271
  if rss_kb < 100_000
265
272
  "#{number_with_delimiter(rss_kb)} KB"
266
- else
273
+ elsif rss_kb < 10_000_000
267
274
  "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
275
+ else
276
+ "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
268
277
  end
269
278
  end
270
279
 
@@ -15,6 +15,10 @@ module Sidekiq
15
15
  REQUEST_METHOD = "REQUEST_METHOD"
16
16
  PATH_INFO = "PATH_INFO"
17
17
 
18
+ def head(path, &block)
19
+ route(HEAD, path, &block)
20
+ end
21
+
18
22
  def get(path, &block)
19
23
  route(GET, path, &block)
20
24
  end
@@ -39,7 +43,6 @@ module Sidekiq
39
43
  @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
40
44
 
41
45
  @routes[method] << WebRoute.new(method, path, block)
42
- @routes[HEAD] << WebRoute.new(method, path, block) if method == GET
43
46
  end
44
47
 
45
48
  def match(env)