sidekiq 5.2.9 → 6.4.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (106) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +318 -1
  3. data/LICENSE +3 -3
  4. data/README.md +23 -34
  5. data/bin/sidekiq +27 -3
  6. data/bin/sidekiqload +67 -61
  7. data/bin/sidekiqmon +8 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +335 -267
  13. data/lib/sidekiq/cli.rb +164 -182
  14. data/lib/sidekiq/client.rb +58 -61
  15. data/lib/sidekiq/delay.rb +7 -6
  16. data/lib/sidekiq/exception_handler.rb +10 -12
  17. data/lib/sidekiq/extensions/action_mailer.rb +13 -22
  18. data/lib/sidekiq/extensions/active_record.rb +13 -10
  19. data/lib/sidekiq/extensions/class_methods.rb +14 -11
  20. data/lib/sidekiq/extensions/generic_proxy.rb +6 -4
  21. data/lib/sidekiq/fetch.rb +40 -32
  22. data/lib/sidekiq/job.rb +13 -0
  23. data/lib/sidekiq/job_logger.rb +33 -7
  24. data/lib/sidekiq/job_retry.rb +70 -71
  25. data/lib/sidekiq/job_util.rb +65 -0
  26. data/lib/sidekiq/launcher.rb +161 -71
  27. data/lib/sidekiq/logger.rb +170 -0
  28. data/lib/sidekiq/manager.rb +17 -21
  29. data/lib/sidekiq/middleware/chain.rb +20 -8
  30. data/lib/sidekiq/middleware/current_attributes.rb +57 -0
  31. data/lib/sidekiq/middleware/i18n.rb +5 -7
  32. data/lib/sidekiq/monitor.rb +133 -0
  33. data/lib/sidekiq/paginator.rb +20 -16
  34. data/lib/sidekiq/processor.rb +71 -70
  35. data/lib/sidekiq/rails.rb +40 -37
  36. data/lib/sidekiq/redis_connection.rb +48 -48
  37. data/lib/sidekiq/scheduled.rb +62 -28
  38. data/lib/sidekiq/sd_notify.rb +149 -0
  39. data/lib/sidekiq/systemd.rb +24 -0
  40. data/lib/sidekiq/testing/inline.rb +2 -1
  41. data/lib/sidekiq/testing.rb +36 -27
  42. data/lib/sidekiq/util.rb +57 -15
  43. data/lib/sidekiq/version.rb +2 -1
  44. data/lib/sidekiq/web/action.rb +15 -11
  45. data/lib/sidekiq/web/application.rb +88 -75
  46. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  47. data/lib/sidekiq/web/helpers.rb +109 -92
  48. data/lib/sidekiq/web/router.rb +23 -19
  49. data/lib/sidekiq/web.rb +61 -105
  50. data/lib/sidekiq/worker.rb +247 -105
  51. data/lib/sidekiq.rb +77 -44
  52. data/sidekiq.gemspec +23 -16
  53. data/web/assets/images/apple-touch-icon.png +0 -0
  54. data/web/assets/javascripts/application.js +83 -64
  55. data/web/assets/javascripts/dashboard.js +54 -73
  56. data/web/assets/stylesheets/application-dark.css +143 -0
  57. data/web/assets/stylesheets/application-rtl.css +0 -4
  58. data/web/assets/stylesheets/application.css +45 -232
  59. data/web/locales/ar.yml +8 -2
  60. data/web/locales/de.yml +14 -2
  61. data/web/locales/en.yml +6 -1
  62. data/web/locales/es.yml +18 -2
  63. data/web/locales/fr.yml +10 -3
  64. data/web/locales/ja.yml +7 -1
  65. data/web/locales/lt.yml +83 -0
  66. data/web/locales/pl.yml +4 -4
  67. data/web/locales/ru.yml +4 -0
  68. data/web/locales/vi.yml +83 -0
  69. data/web/views/_footer.erb +1 -1
  70. data/web/views/_job_info.erb +3 -2
  71. data/web/views/_poll_link.erb +2 -5
  72. data/web/views/_summary.erb +7 -7
  73. data/web/views/busy.erb +54 -20
  74. data/web/views/dashboard.erb +22 -14
  75. data/web/views/dead.erb +3 -3
  76. data/web/views/layout.erb +3 -1
  77. data/web/views/morgue.erb +9 -6
  78. data/web/views/queue.erb +19 -10
  79. data/web/views/queues.erb +10 -2
  80. data/web/views/retries.erb +11 -8
  81. data/web/views/retry.erb +3 -3
  82. data/web/views/scheduled.erb +5 -2
  83. metadata +34 -64
  84. data/.circleci/config.yml +0 -61
  85. data/.github/contributing.md +0 -32
  86. data/.github/issue_template.md +0 -11
  87. data/.gitignore +0 -15
  88. data/.travis.yml +0 -11
  89. data/3.0-Upgrade.md +0 -70
  90. data/4.0-Upgrade.md +0 -53
  91. data/5.0-Upgrade.md +0 -56
  92. data/COMM-LICENSE +0 -97
  93. data/Ent-Changes.md +0 -238
  94. data/Gemfile +0 -23
  95. data/Pro-2.0-Upgrade.md +0 -138
  96. data/Pro-3.0-Upgrade.md +0 -44
  97. data/Pro-4.0-Upgrade.md +0 -35
  98. data/Pro-Changes.md +0 -759
  99. data/Rakefile +0 -9
  100. data/bin/sidekiqctl +0 -20
  101. data/code_of_conduct.md +0 -50
  102. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  103. data/lib/sidekiq/core_ext.rb +0 -1
  104. data/lib/sidekiq/ctl.rb +0 -221
  105. data/lib/sidekiq/logging.rb +0 -122
  106. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
@@ -1,40 +1,48 @@
1
1
  # frozen_string_literal: true
2
- require 'uri'
3
- require 'set'
4
- require 'yaml'
5
- require 'cgi'
2
+
3
+ require "uri"
4
+ require "set"
5
+ require "yaml"
6
+ require "cgi"
6
7
 
7
8
  module Sidekiq
8
9
  # This is not a public API
9
10
  module WebHelpers
10
11
  def strings(lang)
11
- @@strings ||= {}
12
- @@strings[lang] ||= begin
13
- # Allow sidekiq-web extensions to add locale paths
14
- # so extensions can be localized
15
- settings.locales.each_with_object({}) do |path, global|
16
- find_locale_files(lang).each do |file|
17
- strs = YAML.load(File.open(file))
18
- global.merge!(strs[lang])
19
- end
12
+ @strings ||= {}
13
+
14
+ # Allow sidekiq-web extensions to add locale paths
15
+ # so extensions can be localized
16
+ @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
17
+ find_locale_files(lang).each do |file|
18
+ strs = YAML.load(File.open(file))
19
+ global.merge!(strs[lang])
20
20
  end
21
21
  end
22
22
  end
23
23
 
24
+ def singularize(str, count)
25
+ if count == 1 && str.respond_to?(:singularize) # rails
26
+ str.singularize
27
+ else
28
+ str
29
+ end
30
+ end
31
+
24
32
  def clear_caches
25
- @@strings = nil
26
- @@locale_files = nil
27
- @@available_locales = nil
33
+ @strings = nil
34
+ @locale_files = nil
35
+ @available_locales = nil
28
36
  end
29
37
 
30
38
  def locale_files
31
- @@locale_files ||= settings.locales.flat_map do |path|
39
+ @locale_files ||= settings.locales.flat_map { |path|
32
40
  Dir["#{path}/*.yml"]
33
- end
41
+ }
34
42
  end
35
43
 
36
44
  def available_locales
37
- @@available_locales ||= locale_files.map { |path| File.basename(path, '.yml') }.uniq
45
+ @available_locales ||= locale_files.map { |path| File.basename(path, ".yml") }.uniq
38
46
  end
39
47
 
40
48
  def find_locale_files(lang)
@@ -62,33 +70,25 @@ module Sidekiq
62
70
  @head_html.join if defined?(@head_html)
63
71
  end
64
72
 
65
- def poll_path
66
- if current_path != '' && params['poll']
67
- root_path + current_path
68
- else
69
- ""
70
- end
71
- end
72
-
73
73
  def text_direction
74
- get_locale['TextDirection'] || 'ltr'
74
+ get_locale["TextDirection"] || "ltr"
75
75
  end
76
76
 
77
77
  def rtl?
78
- text_direction == 'rtl'
78
+ text_direction == "rtl"
79
79
  end
80
80
 
81
81
  # See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
82
82
  def user_preferred_languages
83
- languages = env['HTTP_ACCEPT_LANGUAGE']
84
- languages.to_s.downcase.gsub(/\s+/, '').split(',').map do |language|
85
- locale, quality = language.split(';q=', 2)
86
- locale = nil if locale == '*' # Ignore wildcards
83
+ languages = env["HTTP_ACCEPT_LANGUAGE"]
84
+ languages.to_s.downcase.gsub(/\s+/, "").split(",").map { |language|
85
+ locale, quality = language.split(";q=", 2)
86
+ locale = nil if locale == "*" # Ignore wildcards
87
87
  quality = quality ? quality.to_f : 1.0
88
88
  [locale, quality]
89
- end.sort do |(_, left), (_, right)|
89
+ }.sort { |(_, left), (_, right)|
90
90
  right <=> left
91
- end.map(&:first).compact
91
+ }.map(&:first).compact
92
92
  end
93
93
 
94
94
  # Given an Accept-Language header like "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2"
@@ -97,31 +97,38 @@ module Sidekiq
97
97
  # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
98
98
  def locale
99
99
  @locale ||= begin
100
- matched_locale = user_preferred_languages.map do |preferred|
101
- preferred_language = preferred.split('-', 2).first
100
+ matched_locale = user_preferred_languages.map { |preferred|
101
+ preferred_language = preferred.split("-", 2).first
102
102
 
103
- lang_group = available_locales.select do |available|
104
- preferred_language == available.split('-', 2).first
105
- end
103
+ lang_group = available_locales.select { |available|
104
+ preferred_language == available.split("-", 2).first
105
+ }
106
106
 
107
107
  lang_group.find { |lang| lang == preferred } || lang_group.min_by(&:length)
108
- end.compact.first
108
+ }.compact.first
109
109
 
110
- matched_locale || 'en'
110
+ matched_locale || "en"
111
111
  end
112
112
  end
113
113
 
114
+ # within is used by Sidekiq Pro
115
+ def display_tags(job, within = nil)
116
+ job.tags.map { |tag|
117
+ "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
118
+ }.join(" ")
119
+ end
120
+
114
121
  # mperham/sidekiq#3243
115
122
  def unfiltered?
116
- yield unless env['PATH_INFO'].start_with?("/filter/")
123
+ yield unless env["PATH_INFO"].start_with?("/filter/")
117
124
  end
118
125
 
119
126
  def get_locale
120
127
  strings(locale)
121
128
  end
122
129
 
123
- def t(msg, options={})
124
- string = get_locale[msg] || strings('en')[msg] || msg
130
+ def t(msg, options = {})
131
+ string = get_locale[msg] || strings("en")[msg] || msg
125
132
  if options.empty?
126
133
  string
127
134
  else
@@ -129,6 +136,10 @@ module Sidekiq
129
136
  end
130
137
  end
131
138
 
139
+ def sort_direction_label
140
+ params[:direction] == "asc" ? "&uarr;" : "&darr;"
141
+ end
142
+
132
143
  def workers
133
144
  @workers ||= Sidekiq::Workers.new
134
145
  end
@@ -141,16 +152,9 @@ module Sidekiq
141
152
  @stats ||= Sidekiq::Stats.new
142
153
  end
143
154
 
144
- def retries_with_score(score)
145
- Sidekiq.redis do |conn|
146
- conn.zrangebyscore('retry', score, score)
147
- end.map { |msg| Sidekiq.load_json(msg) }
148
- end
149
-
150
155
  def redis_connection
151
156
  Sidekiq.redis do |conn|
152
- c = conn.connection
153
- "redis://#{c[:location]}/#{c[:db]}"
157
+ conn.connection[:id]
154
158
  end
155
159
  end
156
160
 
@@ -163,24 +167,24 @@ module Sidekiq
163
167
  end
164
168
 
165
169
  def root_path
166
- "#{env['SCRIPT_NAME']}/"
170
+ "#{env["SCRIPT_NAME"]}/"
167
171
  end
168
172
 
169
173
  def current_path
170
- @current_path ||= request.path_info.gsub(/^\//,'')
174
+ @current_path ||= request.path_info.gsub(/^\//, "")
171
175
  end
172
176
 
173
177
  def current_status
174
- workers.size == 0 ? 'idle' : 'active'
178
+ workers.size == 0 ? "idle" : "active"
175
179
  end
176
180
 
177
181
  def relative_time(time)
178
182
  stamp = time.getutc.iso8601
179
- %{<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>}
183
+ %(<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>)
180
184
  end
181
185
 
182
186
  def job_params(job, score)
183
- "#{score}-#{job['jid']}"
187
+ "#{score}-#{job["jid"]}"
184
188
  end
185
189
 
186
190
  def parse_params(params)
@@ -188,18 +192,19 @@ module Sidekiq
188
192
  [score.to_f, jid]
189
193
  end
190
194
 
191
- SAFE_QPARAMS = %w(page poll)
195
+ SAFE_QPARAMS = %w[page direction]
192
196
 
193
197
  # Merge options with current params, filter safe params, and stringify to query string
194
198
  def qparams(options)
195
- # stringify
196
- options.keys.each do |key|
197
- options[key.to_s] = options.delete(key)
198
- end
199
+ stringified_options = options.transform_keys(&:to_s)
199
200
 
200
- params.merge(options).map do |key, value|
201
+ to_query_string(params.merge(stringified_options))
202
+ end
203
+
204
+ def to_query_string(params)
205
+ params.map { |key, value|
201
206
  SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
202
- end.compact.join("&")
207
+ }.compact.join("&")
203
208
  end
204
209
 
205
210
  def truncate(text, truncate_after_chars = 2000)
@@ -207,40 +212,38 @@ module Sidekiq
207
212
  end
208
213
 
209
214
  def display_args(args, truncate_after_chars = 2000)
210
- return "Invalid job payload, args is nil" if args == nil
211
- return "Invalid job payload, args must be an Array, not #{args.class.name}" if !args.is_a?(Array)
215
+ return "Invalid job payload, args is nil" if args.nil?
216
+ return "Invalid job payload, args must be an Array, not #{args.class.name}" unless args.is_a?(Array)
212
217
 
213
218
  begin
214
- args.map do |arg|
219
+ args.map { |arg|
215
220
  h(truncate(to_display(arg), truncate_after_chars))
216
- end.join(", ")
221
+ }.join(", ")
217
222
  rescue
218
223
  "Illegal job arguments: #{h args.inspect}"
219
224
  end
220
225
  end
221
226
 
222
227
  def csrf_tag
223
- "<input type='hidden' name='authenticity_token' value='#{session[:csrf]}'/>"
228
+ "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
224
229
  end
225
230
 
226
231
  def to_display(arg)
232
+ arg.inspect
233
+ rescue
227
234
  begin
228
- arg.inspect
229
- rescue
230
- begin
231
- arg.to_s
232
- rescue => ex
233
- "Cannot display argument: [#{ex.class.name}] #{ex.message}"
234
- end
235
+ arg.to_s
236
+ rescue => ex
237
+ "Cannot display argument: [#{ex.class.name}] #{ex.message}"
235
238
  end
236
239
  end
237
240
 
238
- RETRY_JOB_KEYS = Set.new(%w(
241
+ RETRY_JOB_KEYS = Set.new(%w[
239
242
  queue class args retry_count retried_at failed_at
240
243
  jid error_message error_class backtrace
241
244
  error_backtrace enqueued_at retry wrapped
242
- created_at
243
- ))
245
+ created_at tags display_class
246
+ ])
244
247
 
245
248
  def retry_extra_items(retry_job)
246
249
  @retry_extra_items ||= {}.tap do |extra|
@@ -250,15 +253,29 @@ module Sidekiq
250
253
  end
251
254
  end
252
255
 
256
+ def format_memory(rss_kb)
257
+ return "0" if rss_kb.nil? || rss_kb == 0
258
+
259
+ if rss_kb < 100_000
260
+ "#{number_with_delimiter(rss_kb)} KB"
261
+ elsif rss_kb < 10_000_000
262
+ "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
263
+ else
264
+ "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
265
+ end
266
+ end
267
+
253
268
  def number_with_delimiter(number)
269
+ return "" if number.nil?
270
+
254
271
  begin
255
272
  Float(number)
256
273
  rescue ArgumentError, TypeError
257
274
  return number
258
275
  end
259
276
 
260
- options = {delimiter: ',', separator: '.'}
261
- parts = number.to_s.to_str.split('.')
277
+ options = {delimiter: ",", separator: "."}
278
+ parts = number.to_s.to_str.split(".")
262
279
  parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
263
280
  parts.join(options[:separator])
264
281
  end
@@ -266,8 +283,8 @@ module Sidekiq
266
283
  def h(text)
267
284
  ::Rack::Utils.escape_html(text)
268
285
  rescue ArgumentError => e
269
- raise unless e.message.eql?('invalid byte sequence in UTF-8')
270
- text.encode!('UTF-16', 'UTF-8', invalid: :replace, replace: '').encode!('UTF-8', 'UTF-16')
286
+ raise unless e.message.eql?("invalid byte sequence in UTF-8")
287
+ text.encode!("UTF-16", "UTF-8", invalid: :replace, replace: "").encode!("UTF-8", "UTF-16")
271
288
  retry
272
289
  end
273
290
 
@@ -284,7 +301,7 @@ module Sidekiq
284
301
  end
285
302
 
286
303
  def environment_title_prefix
287
- environment = Sidekiq.options[:environment] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
304
+ environment = Sidekiq.options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
288
305
 
289
306
  "[#{environment.upcase}] " unless environment == "production"
290
307
  end
@@ -294,30 +311,30 @@ module Sidekiq
294
311
  end
295
312
 
296
313
  def server_utc_time
297
- Time.now.utc.strftime('%H:%M:%S UTC')
314
+ Time.now.utc.strftime("%H:%M:%S UTC")
298
315
  end
299
316
 
300
317
  def redis_connection_and_namespace
301
318
  @redis_connection_and_namespace ||= begin
302
- namespace_suffix = namespace == nil ? '' : "##{namespace}"
319
+ namespace_suffix = namespace.nil? ? "" : "##{namespace}"
303
320
  "#{redis_connection}#{namespace_suffix}"
304
321
  end
305
322
  end
306
323
 
307
324
  def retry_or_delete_or_kill(job, params)
308
- if params['retry']
325
+ if params["retry"]
309
326
  job.retry
310
- elsif params['delete']
327
+ elsif params["delete"]
311
328
  job.delete
312
- elsif params['kill']
329
+ elsif params["kill"]
313
330
  job.kill
314
331
  end
315
332
  end
316
333
 
317
334
  def delete_or_add_queue(job, params)
318
- if params['delete']
335
+ if params["delete"]
319
336
  job.delete
320
- elsif params['add_to_queue']
337
+ elsif params["add_to_queue"]
321
338
  job.add_to_queue
322
339
  end
323
340
  end
@@ -1,18 +1,23 @@
1
1
  # frozen_string_literal: true
2
- require 'rack'
2
+
3
+ require "rack"
3
4
 
4
5
  module Sidekiq
5
6
  module WebRouter
6
- GET = 'GET'
7
- DELETE = 'DELETE'
8
- POST = 'POST'
9
- PUT = 'PUT'
10
- PATCH = 'PATCH'
11
- HEAD = 'HEAD'
12
-
13
- ROUTE_PARAMS = 'rack.route_params'
14
- REQUEST_METHOD = 'REQUEST_METHOD'
15
- PATH_INFO = 'PATH_INFO'
7
+ GET = "GET"
8
+ DELETE = "DELETE"
9
+ POST = "POST"
10
+ PUT = "PUT"
11
+ PATCH = "PATCH"
12
+ HEAD = "HEAD"
13
+
14
+ ROUTE_PARAMS = "rack.route_params"
15
+ REQUEST_METHOD = "REQUEST_METHOD"
16
+ PATH_INFO = "PATH_INFO"
17
+
18
+ def head(path, &block)
19
+ route(HEAD, path, &block)
20
+ end
16
21
 
17
22
  def get(path, &block)
18
23
  route(GET, path, &block)
@@ -35,10 +40,9 @@ module Sidekiq
35
40
  end
36
41
 
37
42
  def route(method, path, &block)
38
- @routes ||= { GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => [] }
43
+ @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
39
44
 
40
45
  @routes[method] << WebRoute.new(method, path, block)
41
- @routes[HEAD] << WebRoute.new(method, path, block) if method == GET
42
46
  end
43
47
 
44
48
  def match(env)
@@ -50,7 +54,8 @@ module Sidekiq
50
54
  path_info = "/" if path_info == ""
51
55
 
52
56
  @routes[request_method].each do |route|
53
- if params = route.match(request_method, path_info)
57
+ params = route.match(request_method, path_info)
58
+ if params
54
59
  env[ROUTE_PARAMS] = params
55
60
 
56
61
  return WebAction.new(env, route.block)
@@ -64,7 +69,7 @@ module Sidekiq
64
69
  class WebRoute
65
70
  attr_accessor :request_method, :pattern, :block, :name
66
71
 
67
- NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^\.:$\/]+)/
72
+ NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^.:$\/]+)/
68
73
 
69
74
  def initialize(request_method, pattern, block)
70
75
  @request_method = request_method
@@ -77,7 +82,7 @@ module Sidekiq
77
82
  end
78
83
 
79
84
  def compile
80
- if pattern.match(NAMED_SEGMENTS_PATTERN)
85
+ if pattern.match?(NAMED_SEGMENTS_PATTERN)
81
86
  p = pattern.gsub(NAMED_SEGMENTS_PATTERN, '/\1(?<\2>[^$/]+)')
82
87
 
83
88
  Regexp.new("\\A#{p}\\Z")
@@ -91,9 +96,8 @@ module Sidekiq
91
96
  when String
92
97
  {} if path == matcher
93
98
  else
94
- if path_match = path.match(matcher)
95
- Hash[path_match.names.map(&:to_sym).zip(path_match.captures)]
96
- end
99
+ path_match = path.match(matcher)
100
+ path_match&.named_captures&.transform_keys(&:to_sym)
97
101
  end
98
102
  end
99
103
  end