sidekiq 5.2.7 → 6.0.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +0 -2
- data/.standard.yml +20 -0
- data/6.0-Upgrade.md +70 -0
- data/Changes.md +34 -0
- data/Ent-2.0-Upgrade.md +37 -0
- data/Ent-Changes.md +12 -0
- data/Gemfile +12 -11
- data/Gemfile.lock +196 -0
- data/Pro-5.0-Upgrade.md +25 -0
- data/Pro-Changes.md +12 -3
- data/README.md +16 -30
- data/Rakefile +5 -4
- data/bin/sidekiqload +26 -22
- data/bin/sidekiqmon +9 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
- data/lib/generators/sidekiq/worker_generator.rb +12 -14
- data/lib/sidekiq.rb +53 -42
- data/lib/sidekiq/api.rb +138 -151
- data/lib/sidekiq/cli.rb +97 -162
- data/lib/sidekiq/client.rb +45 -46
- data/lib/sidekiq/delay.rb +5 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +10 -20
- data/lib/sidekiq/extensions/active_record.rb +9 -7
- data/lib/sidekiq/extensions/class_methods.rb +9 -7
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
- data/lib/sidekiq/fetch.rb +5 -6
- data/lib/sidekiq/job_logger.rb +37 -7
- data/lib/sidekiq/job_retry.rb +45 -58
- data/lib/sidekiq/launcher.rb +59 -51
- data/lib/sidekiq/logger.rb +69 -0
- data/lib/sidekiq/manager.rb +7 -9
- data/lib/sidekiq/middleware/chain.rb +3 -2
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +148 -0
- data/lib/sidekiq/paginator.rb +11 -12
- data/lib/sidekiq/processor.rb +52 -49
- data/lib/sidekiq/rails.rb +23 -29
- data/lib/sidekiq/redis_connection.rb +31 -37
- data/lib/sidekiq/scheduled.rb +17 -19
- data/lib/sidekiq/testing.rb +22 -23
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/util.rb +17 -14
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web.rb +41 -49
- data/lib/sidekiq/web/action.rb +14 -10
- data/lib/sidekiq/web/application.rb +60 -57
- data/lib/sidekiq/web/helpers.rb +66 -67
- data/lib/sidekiq/web/router.rb +17 -14
- data/lib/sidekiq/worker.rb +124 -97
- data/sidekiq.gemspec +16 -16
- data/web/assets/javascripts/dashboard.js +2 -21
- data/web/locales/ja.yml +2 -1
- metadata +19 -29
- data/.travis.yml +0 -11
- data/bin/sidekiqctl +0 -20
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
require "sidekiq/util"
|
5
|
+
require "sidekiq/api"
|
5
6
|
|
6
7
|
module Sidekiq
|
7
8
|
module Scheduled
|
8
|
-
SETS = %w
|
9
|
+
SETS = %w[retry schedule]
|
9
10
|
|
10
11
|
class Enq
|
11
|
-
def enqueue_jobs(now=Time.now.to_f.to_s, sorted_sets=SETS)
|
12
|
+
def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = SETS)
|
12
13
|
# A job's "score" in Redis is the time at which it should be processed.
|
13
14
|
# Just check Redis for the set of jobs with a timestamp before now.
|
14
15
|
Sidekiq.redis do |conn|
|
@@ -17,14 +18,14 @@ module Sidekiq
|
|
17
18
|
# We need to go through the list one at a time to reduce the risk of something
|
18
19
|
# going wrong between the time jobs are popped from the scheduled queue and when
|
19
20
|
# they are pushed onto a work queue and losing the jobs.
|
20
|
-
while job = conn.zrangebyscore(sorted_set,
|
21
|
+
while (job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first)
|
21
22
|
|
22
23
|
# Pop item off the queue and add it to the work queue. If the job can't be popped from
|
23
24
|
# the queue, it's because another process already popped it so we can move on to the
|
24
25
|
# next one.
|
25
26
|
if conn.zrem(sorted_set, job)
|
26
27
|
Sidekiq::Client.push(Sidekiq.load_json(job))
|
27
|
-
Sidekiq
|
28
|
+
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
28
29
|
end
|
29
30
|
end
|
30
31
|
end
|
@@ -61,26 +62,24 @@ module Sidekiq
|
|
61
62
|
end
|
62
63
|
|
63
64
|
def start
|
64
|
-
@thread ||= safe_thread("scheduler")
|
65
|
+
@thread ||= safe_thread("scheduler") {
|
65
66
|
initial_wait
|
66
67
|
|
67
|
-
|
68
|
+
until @done
|
68
69
|
enqueue
|
69
70
|
wait
|
70
71
|
end
|
71
72
|
Sidekiq.logger.info("Scheduler exiting...")
|
72
|
-
|
73
|
+
}
|
73
74
|
end
|
74
75
|
|
75
76
|
def enqueue
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
handle_exception(ex)
|
83
|
-
end
|
77
|
+
@enq.enqueue_jobs
|
78
|
+
rescue => ex
|
79
|
+
# Most likely a problem with redis networking.
|
80
|
+
# Punt and try again at the next interval
|
81
|
+
logger.error ex.message
|
82
|
+
handle_exception(ex)
|
84
83
|
end
|
85
84
|
|
86
85
|
private
|
@@ -168,7 +167,6 @@ module Sidekiq
|
|
168
167
|
@sleeper.pop(total)
|
169
168
|
rescue Timeout::Error
|
170
169
|
end
|
171
|
-
|
172
170
|
end
|
173
171
|
end
|
174
172
|
end
|
data/lib/sidekiq/testing.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'securerandom'
|
3
|
-
require 'sidekiq'
|
4
2
|
|
5
|
-
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq"
|
6
5
|
|
6
|
+
module Sidekiq
|
7
7
|
class Testing
|
8
8
|
class << self
|
9
9
|
attr_accessor :__test_mode
|
10
10
|
|
11
11
|
def __set_test_mode(mode)
|
12
12
|
if block_given?
|
13
|
-
current_mode =
|
13
|
+
current_mode = __test_mode
|
14
14
|
begin
|
15
15
|
self.__test_mode = mode
|
16
16
|
yield
|
@@ -35,19 +35,19 @@ module Sidekiq
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def enabled?
|
38
|
-
|
38
|
+
__test_mode != :disable
|
39
39
|
end
|
40
40
|
|
41
41
|
def disabled?
|
42
|
-
|
42
|
+
__test_mode == :disable
|
43
43
|
end
|
44
44
|
|
45
45
|
def fake?
|
46
|
-
|
46
|
+
__test_mode == :fake
|
47
47
|
end
|
48
48
|
|
49
49
|
def inline?
|
50
|
-
|
50
|
+
__test_mode == :inline
|
51
51
|
end
|
52
52
|
|
53
53
|
def server_middleware
|
@@ -57,7 +57,7 @@ module Sidekiq
|
|
57
57
|
end
|
58
58
|
|
59
59
|
def constantize(str)
|
60
|
-
names = str.split(
|
60
|
+
names = str.split("::")
|
61
61
|
names.shift if names.empty? || names.first.empty?
|
62
62
|
|
63
63
|
names.inject(Object) do |constant, name|
|
@@ -77,14 +77,14 @@ module Sidekiq
|
|
77
77
|
if Sidekiq::Testing.fake?
|
78
78
|
payloads.each do |job|
|
79
79
|
job = Sidekiq.load_json(Sidekiq.dump_json(job))
|
80
|
-
job
|
81
|
-
Queues.push(job[
|
80
|
+
job["enqueued_at"] = Time.now.to_f unless job["at"]
|
81
|
+
Queues.push(job["queue"], job["class"], job)
|
82
82
|
end
|
83
83
|
true
|
84
84
|
elsif Sidekiq::Testing.inline?
|
85
85
|
payloads.each do |job|
|
86
|
-
klass = Sidekiq::Testing.constantize(job[
|
87
|
-
job[
|
86
|
+
klass = Sidekiq::Testing.constantize(job["class"])
|
87
|
+
job["id"] ||= SecureRandom.hex(12)
|
88
88
|
job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
|
89
89
|
klass.process_job(job_hash)
|
90
90
|
end
|
@@ -255,27 +255,26 @@ module Sidekiq
|
|
255
255
|
# Then I should receive a welcome email to "foo@example.com"
|
256
256
|
#
|
257
257
|
module ClassMethods
|
258
|
-
|
259
258
|
# Queue for this worker
|
260
259
|
def queue
|
261
|
-
|
260
|
+
get_sidekiq_options["queue"]
|
262
261
|
end
|
263
262
|
|
264
263
|
# Jobs queued for this worker
|
265
264
|
def jobs
|
266
|
-
Queues.jobs_by_worker[
|
265
|
+
Queues.jobs_by_worker[to_s]
|
267
266
|
end
|
268
267
|
|
269
268
|
# Clear all jobs for this worker
|
270
269
|
def clear
|
271
|
-
Queues.clear_for(queue,
|
270
|
+
Queues.clear_for(queue, to_s)
|
272
271
|
end
|
273
272
|
|
274
273
|
# Drain and run all jobs for this worker
|
275
274
|
def drain
|
276
275
|
while jobs.any?
|
277
276
|
next_job = jobs.first
|
278
|
-
Queues.delete_for(next_job["jid"], next_job["queue"],
|
277
|
+
Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
|
279
278
|
process_job(next_job)
|
280
279
|
end
|
281
280
|
end
|
@@ -284,16 +283,16 @@ module Sidekiq
|
|
284
283
|
def perform_one
|
285
284
|
raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
|
286
285
|
next_job = jobs.first
|
287
|
-
Queues.delete_for(next_job["jid"], queue,
|
286
|
+
Queues.delete_for(next_job["jid"], queue, to_s)
|
288
287
|
process_job(next_job)
|
289
288
|
end
|
290
289
|
|
291
290
|
def process_job(job)
|
292
291
|
worker = new
|
293
|
-
worker.jid = job[
|
294
|
-
worker.bid = job[
|
295
|
-
Sidekiq::Testing.server_middleware.invoke(worker, job, job[
|
296
|
-
execute_job(worker, job[
|
292
|
+
worker.jid = job["jid"]
|
293
|
+
worker.bid = job["bid"] if worker.respond_to?(:bid=)
|
294
|
+
Sidekiq::Testing.server_middleware.invoke(worker, job, job["queue"]) do
|
295
|
+
execute_job(worker, job["args"])
|
297
296
|
end
|
298
297
|
end
|
299
298
|
|
data/lib/sidekiq/util.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
2
|
+
|
3
|
+
require "socket"
|
4
|
+
require "securerandom"
|
5
|
+
require "sidekiq/exception_handler"
|
5
6
|
|
6
7
|
module Sidekiq
|
7
8
|
##
|
@@ -15,13 +16,13 @@ module Sidekiq
|
|
15
16
|
def watchdog(last_words)
|
16
17
|
yield
|
17
18
|
rescue Exception => ex
|
18
|
-
handle_exception(ex, {
|
19
|
+
handle_exception(ex, {context: last_words})
|
19
20
|
raise ex
|
20
21
|
end
|
21
22
|
|
22
23
|
def safe_thread(name, &block)
|
23
24
|
Thread.new do
|
24
|
-
Thread.current
|
25
|
+
Thread.current.name = name
|
25
26
|
watchdog(name, &block)
|
26
27
|
end
|
27
28
|
end
|
@@ -34,8 +35,12 @@ module Sidekiq
|
|
34
35
|
Sidekiq.redis(&block)
|
35
36
|
end
|
36
37
|
|
38
|
+
def tid
|
39
|
+
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
40
|
+
end
|
41
|
+
|
37
42
|
def hostname
|
38
|
-
ENV[
|
43
|
+
ENV["DYNO"] || Socket.gethostname
|
39
44
|
end
|
40
45
|
|
41
46
|
def process_nonce
|
@@ -43,22 +48,20 @@ module Sidekiq
|
|
43
48
|
end
|
44
49
|
|
45
50
|
def identity
|
46
|
-
@@identity ||= "#{hostname}:#{
|
51
|
+
@@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
|
47
52
|
end
|
48
53
|
|
49
|
-
def fire_event(event, options={})
|
54
|
+
def fire_event(event, options = {})
|
50
55
|
reverse = options[:reverse]
|
51
56
|
reraise = options[:reraise]
|
52
57
|
|
53
58
|
arr = Sidekiq.options[:lifecycle_events][event]
|
54
59
|
arr.reverse! if reverse
|
55
60
|
arr.each do |block|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
raise ex if reraise
|
61
|
-
end
|
61
|
+
block.call
|
62
|
+
rescue => ex
|
63
|
+
handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
|
64
|
+
raise ex if reraise
|
62
65
|
end
|
63
66
|
arr.clear
|
64
67
|
end
|
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web.rb
CHANGED
@@ -1,20 +1,21 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'erb'
|
3
2
|
|
4
|
-
require
|
5
|
-
require 'sidekiq/api'
|
6
|
-
require 'sidekiq/paginator'
|
7
|
-
require 'sidekiq/web/helpers'
|
3
|
+
require "erb"
|
8
4
|
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
5
|
+
require "sidekiq"
|
6
|
+
require "sidekiq/api"
|
7
|
+
require "sidekiq/paginator"
|
8
|
+
require "sidekiq/web/helpers"
|
12
9
|
|
13
|
-
require
|
10
|
+
require "sidekiq/web/router"
|
11
|
+
require "sidekiq/web/action"
|
12
|
+
require "sidekiq/web/application"
|
14
13
|
|
15
|
-
require
|
16
|
-
|
17
|
-
require
|
14
|
+
require "rack/protection"
|
15
|
+
|
16
|
+
require "rack/builder"
|
17
|
+
require "rack/file"
|
18
|
+
require "rack/session/cookie"
|
18
19
|
|
19
20
|
module Sidekiq
|
20
21
|
class Web
|
@@ -25,12 +26,12 @@ module Sidekiq
|
|
25
26
|
ASSETS = "#{ROOT}/assets"
|
26
27
|
|
27
28
|
DEFAULT_TABS = {
|
28
|
-
"Dashboard" =>
|
29
|
-
"Busy"
|
30
|
-
"Queues"
|
31
|
-
"Retries"
|
32
|
-
"Scheduled" =>
|
33
|
-
"Dead"
|
29
|
+
"Dashboard" => "",
|
30
|
+
"Busy" => "busy",
|
31
|
+
"Queues" => "queues",
|
32
|
+
"Retries" => "retries",
|
33
|
+
"Scheduled" => "scheduled",
|
34
|
+
"Dead" => "morgue",
|
34
35
|
}
|
35
36
|
|
36
37
|
class << self
|
@@ -64,11 +65,11 @@ module Sidekiq
|
|
64
65
|
end
|
65
66
|
|
66
67
|
def enable(*opts)
|
67
|
-
opts.each {|key| set(key, true) }
|
68
|
+
opts.each { |key| set(key, true) }
|
68
69
|
end
|
69
70
|
|
70
71
|
def disable(*opts)
|
71
|
-
opts.each {|key| set(key, false) }
|
72
|
+
opts.each { |key| set(key, false) }
|
72
73
|
end
|
73
74
|
|
74
75
|
# Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
|
@@ -81,10 +82,10 @@ module Sidekiq
|
|
81
82
|
end
|
82
83
|
|
83
84
|
def self.inherited(child)
|
84
|
-
child.app_url =
|
85
|
-
child.session_secret =
|
86
|
-
child.redis_pool =
|
87
|
-
child.sessions =
|
85
|
+
child.app_url = app_url
|
86
|
+
child.session_secret = session_secret
|
87
|
+
child.redis_pool = redis_pool
|
88
|
+
child.sessions = sessions
|
88
89
|
end
|
89
90
|
|
90
91
|
def settings
|
@@ -113,11 +114,11 @@ module Sidekiq
|
|
113
114
|
end
|
114
115
|
|
115
116
|
def enable(*opts)
|
116
|
-
opts.each {|key| set(key, true) }
|
117
|
+
opts.each { |key| set(key, true) }
|
117
118
|
end
|
118
119
|
|
119
120
|
def disable(*opts)
|
120
|
-
opts.each {|key| set(key, false) }
|
121
|
+
opts.each { |key| set(key, false) }
|
121
122
|
end
|
122
123
|
|
123
124
|
def set(attribute, value)
|
@@ -145,28 +146,28 @@ module Sidekiq
|
|
145
146
|
private
|
146
147
|
|
147
148
|
def using?(middleware)
|
148
|
-
middlewares.any? do |(m,_)|
|
149
|
-
m.
|
149
|
+
middlewares.any? do |(m, _)|
|
150
|
+
m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
|
150
151
|
end
|
151
152
|
end
|
152
153
|
|
153
154
|
def build_sessions
|
154
155
|
middlewares = self.middlewares
|
155
156
|
|
156
|
-
unless using?(::Rack::Protection) || ENV[
|
157
|
-
middlewares.unshift [[::Rack::Protection, {
|
157
|
+
unless using?(::Rack::Protection) || ENV["RACK_ENV"] == "test"
|
158
|
+
middlewares.unshift [[::Rack::Protection, {use: :authenticity_token}], nil]
|
158
159
|
end
|
159
160
|
|
160
161
|
s = sessions
|
161
162
|
return unless s
|
162
163
|
|
163
164
|
unless using? ::Rack::Session::Cookie
|
164
|
-
unless secret = Web.session_secret
|
165
|
-
require
|
165
|
+
unless (secret = Web.session_secret)
|
166
|
+
require "securerandom"
|
166
167
|
secret = SecureRandom.hex(64)
|
167
168
|
end
|
168
169
|
|
169
|
-
options = {
|
170
|
+
options = {secret: secret}
|
170
171
|
options = options.merge(s.to_hash) if s.respond_to? :to_hash
|
171
172
|
|
172
173
|
middlewares.unshift [[::Rack::Session::Cookie, options], nil]
|
@@ -180,13 +181,13 @@ module Sidekiq
|
|
180
181
|
klass = self.class
|
181
182
|
|
182
183
|
::Rack::Builder.new do
|
183
|
-
%w
|
184
|
+
%w[stylesheets javascripts images].each do |asset_dir|
|
184
185
|
map "/#{asset_dir}" do
|
185
|
-
run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {
|
186
|
+
run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
|
186
187
|
end
|
187
188
|
end
|
188
189
|
|
189
|
-
middlewares.each {|middleware, block| use(*middleware, &block) }
|
190
|
+
middlewares.each { |middleware, block| use(*middleware, &block) }
|
190
191
|
|
191
192
|
run WebApplication.new(klass)
|
192
193
|
end
|
@@ -196,18 +197,9 @@ module Sidekiq
|
|
196
197
|
Sidekiq::WebApplication.helpers WebHelpers
|
197
198
|
Sidekiq::WebApplication.helpers Sidekiq::Paginator
|
198
199
|
|
199
|
-
Sidekiq::WebAction.class_eval
|
200
|
-
|
201
|
-
|
202
|
-
if defined?(::ActionDispatch::Request::Session) &&
|
203
|
-
!::ActionDispatch::Request::Session.method_defined?(:each)
|
204
|
-
# mperham/sidekiq#2460
|
205
|
-
# Rack apps can't reuse the Rails session store without
|
206
|
-
# this monkeypatch, fixed in Rails 5.
|
207
|
-
class ActionDispatch::Request::Session
|
208
|
-
def each(&block)
|
209
|
-
hash = self.to_hash
|
210
|
-
hash.each(&block)
|
200
|
+
Sidekiq::WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
201
|
+
def _render
|
202
|
+
#{ERB.new(File.read(Web::LAYOUT)).src}
|
211
203
|
end
|
212
|
-
|
204
|
+
RUBY
|
213
205
|
end
|