sidekiq 3.4.1 → 4.0.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +3 -2
- data/4.0-Upgrade.md +50 -0
- data/COMM-LICENSE +55 -45
- data/Changes.md +85 -1
- data/Ent-Changes.md +79 -0
- data/Gemfile +7 -1
- data/Pro-2.0-Upgrade.md +2 -2
- data/Pro-3.0-Upgrade.md +46 -0
- data/Pro-Changes.md +60 -2
- data/README.md +20 -16
- data/bin/sidekiq +4 -0
- data/bin/sidekiqctl +8 -2
- data/bin/sidekiqload +167 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +2 -2
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +5 -5
- data/lib/sidekiq/api.rb +43 -33
- data/lib/sidekiq/cli.rb +41 -42
- data/lib/sidekiq/client.rb +5 -10
- data/lib/sidekiq/fetch.rb +35 -111
- data/lib/sidekiq/launcher.rb +102 -42
- data/lib/sidekiq/manager.rb +80 -180
- data/lib/sidekiq/middleware/server/logging.rb +13 -8
- data/lib/sidekiq/middleware/server/retry_jobs.rb +6 -6
- data/lib/sidekiq/processor.rb +126 -97
- data/lib/sidekiq/redis_connection.rb +23 -5
- data/lib/sidekiq/scheduled.rb +47 -26
- data/lib/sidekiq/testing.rb +139 -17
- data/lib/sidekiq/util.rb +20 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web.rb +17 -1
- data/lib/sidekiq/web_helpers.rb +33 -5
- data/lib/sidekiq/worker.rb +16 -0
- data/lib/sidekiq.rb +37 -14
- data/sidekiq.gemspec +10 -11
- data/test/helper.rb +45 -10
- data/test/test_actors.rb +137 -0
- data/test/test_api.rb +417 -384
- data/test/test_cli.rb +29 -59
- data/test/test_client.rb +60 -135
- data/test/test_extensions.rb +29 -23
- data/test/test_fetch.rb +2 -57
- data/test/test_launcher.rb +80 -0
- data/test/test_logging.rb +1 -1
- data/test/test_manager.rb +16 -131
- data/test/test_middleware.rb +3 -5
- data/test/test_processor.rb +110 -76
- data/test/test_rails.rb +21 -0
- data/test/test_redis_connection.rb +0 -1
- data/test/test_retry.rb +114 -162
- data/test/test_scheduled.rb +11 -17
- data/test/test_scheduling.rb +20 -42
- data/test/test_sidekiq.rb +46 -16
- data/test/test_testing.rb +80 -20
- data/test/test_testing_fake.rb +83 -8
- data/test/test_testing_inline.rb +3 -3
- data/test/test_util.rb +16 -0
- data/test/test_web.rb +28 -9
- data/test/test_web_helpers.rb +3 -2
- data/web/assets/images/favicon.ico +0 -0
- data/web/assets/javascripts/application.js +6 -1
- data/web/assets/javascripts/dashboard.js +2 -8
- data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +14 -14
- data/web/assets/stylesheets/application.css +33 -56
- data/web/locales/de.yml +1 -1
- data/web/locales/en.yml +2 -0
- data/web/locales/fr.yml +2 -2
- data/web/locales/ja.yml +10 -1
- data/web/locales/{no.yml → nb.yml} +10 -2
- data/web/locales/uk.yml +76 -0
- data/web/views/_footer.erb +2 -7
- data/web/views/_job_info.erb +5 -1
- data/web/views/_nav.erb +2 -2
- data/web/views/_poll_js.erb +5 -0
- data/web/views/{_poll.erb → _poll_link.erb} +0 -3
- data/web/views/busy.erb +2 -1
- data/web/views/dead.erb +1 -0
- data/web/views/layout.erb +2 -0
- data/web/views/morgue.erb +3 -0
- data/web/views/queue.erb +1 -0
- data/web/views/queues.erb +1 -0
- data/web/views/retries.erb +3 -0
- data/web/views/retry.erb +1 -0
- data/web/views/scheduled.erb +1 -0
- data/web/views/scheduled_job_info.erb +1 -0
- metadata +75 -55
- data/lib/sidekiq/actor.rb +0 -39
- data/test/test_worker_generator.rb +0 -17
data/test/test_api.rb
CHANGED
@@ -1,493 +1,526 @@
|
|
1
1
|
require_relative 'helper'
|
2
|
+
require 'sidekiq/api'
|
3
|
+
require 'active_job'
|
4
|
+
require 'action_mailer'
|
2
5
|
|
3
6
|
class TestApi < Sidekiq::Test
|
4
|
-
|
5
|
-
describe "stats" do
|
7
|
+
describe 'api' do
|
6
8
|
before do
|
7
|
-
Sidekiq.redis = REDIS
|
8
9
|
Sidekiq.redis {|c| c.flushdb }
|
9
10
|
end
|
10
11
|
|
11
|
-
describe "
|
12
|
+
describe "stats" do
|
12
13
|
it "is initially zero" do
|
13
14
|
s = Sidekiq::Stats.new
|
14
15
|
assert_equal 0, s.processed
|
16
|
+
assert_equal 0, s.failed
|
17
|
+
assert_equal 0, s.enqueued
|
15
18
|
end
|
16
19
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
20
|
+
describe "processed" do
|
21
|
+
it "returns number of processed jobs" do
|
22
|
+
Sidekiq.redis { |conn| conn.set("stat:processed", 5) }
|
23
|
+
s = Sidekiq::Stats.new
|
24
|
+
assert_equal 5, s.processed
|
25
|
+
end
|
21
26
|
end
|
22
|
-
end
|
23
27
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
+
describe "failed" do
|
29
|
+
it "returns number of failed jobs" do
|
30
|
+
Sidekiq.redis { |conn| conn.set("stat:failed", 5) }
|
31
|
+
s = Sidekiq::Stats.new
|
32
|
+
assert_equal 5, s.failed
|
33
|
+
end
|
28
34
|
end
|
29
35
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
+
describe "reset" do
|
37
|
+
before do
|
38
|
+
Sidekiq.redis do |conn|
|
39
|
+
conn.set('stat:processed', 5)
|
40
|
+
conn.set('stat:failed', 10)
|
41
|
+
end
|
42
|
+
end
|
36
43
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
44
|
+
it 'will reset all stats by default' do
|
45
|
+
Sidekiq::Stats.new.reset
|
46
|
+
s = Sidekiq::Stats.new
|
47
|
+
assert_equal 0, s.failed
|
48
|
+
assert_equal 0, s.processed
|
42
49
|
end
|
43
|
-
end
|
44
50
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
assert_equal
|
49
|
-
assert_equal
|
51
|
+
it 'can reset individual stats' do
|
52
|
+
Sidekiq::Stats.new.reset('failed')
|
53
|
+
s = Sidekiq::Stats.new
|
54
|
+
assert_equal 0, s.failed
|
55
|
+
assert_equal 5, s.processed
|
50
56
|
end
|
51
|
-
end
|
52
57
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
assert_equal
|
57
|
-
assert_equal
|
58
|
+
it 'can accept anything that responds to #to_s' do
|
59
|
+
Sidekiq::Stats.new.reset(:failed)
|
60
|
+
s = Sidekiq::Stats.new
|
61
|
+
assert_equal 0, s.failed
|
62
|
+
assert_equal 5, s.processed
|
58
63
|
end
|
59
|
-
end
|
60
64
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
assert_equal
|
65
|
-
assert_equal
|
65
|
+
it 'ignores anything other than "failed" or "processed"' do
|
66
|
+
Sidekiq::Stats.new.reset((1..10).to_a, ['failed'])
|
67
|
+
s = Sidekiq::Stats.new
|
68
|
+
assert_equal 0, s.failed
|
69
|
+
assert_equal 5, s.processed
|
66
70
|
end
|
67
71
|
end
|
68
72
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
assert_equal
|
73
|
-
assert_equal '0', conn.get('stat:failed')
|
73
|
+
describe "queues" do
|
74
|
+
it "is initially empty" do
|
75
|
+
s = Sidekiq::Stats::Queues.new
|
76
|
+
assert_equal 0, s.lengths.size
|
74
77
|
end
|
75
|
-
end
|
76
|
-
end
|
77
78
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
79
|
+
it "returns a hash of queue and size in order" do
|
80
|
+
Sidekiq.redis do |conn|
|
81
|
+
conn.rpush 'queue:foo', '{}'
|
82
|
+
conn.sadd 'queues', 'foo'
|
82
83
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
end
|
84
|
+
3.times { conn.rpush 'queue:bar', '{}' }
|
85
|
+
conn.sadd 'queues', 'bar'
|
86
|
+
end
|
87
87
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
conn.sadd 'queues', 'foo'
|
88
|
+
s = Sidekiq::Stats::Queues.new
|
89
|
+
assert_equal ({ "foo" => 1, "bar" => 3 }), s.lengths
|
90
|
+
assert_equal "bar", s.lengths.first.first
|
92
91
|
|
93
|
-
|
94
|
-
conn.sadd 'queues', 'bar'
|
92
|
+
assert_equal Sidekiq::Stats.new.queues, Sidekiq::Stats::Queues.new.lengths
|
95
93
|
end
|
96
|
-
|
97
|
-
s = Sidekiq::Stats::Queues.new
|
98
|
-
assert_equal ({ "foo" => 1, "bar" => 3 }), s.lengths
|
99
|
-
assert_equal "bar", s.lengths.first.first
|
100
94
|
end
|
101
|
-
end
|
102
95
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
96
|
+
describe "enqueued" do
|
97
|
+
it "returns total enqueued jobs" do
|
98
|
+
Sidekiq.redis do |conn|
|
99
|
+
conn.rpush 'queue:foo', '{}'
|
100
|
+
conn.sadd 'queues', 'foo'
|
108
101
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
conn.sadd 'queues', 'foo'
|
102
|
+
3.times { conn.rpush 'queue:bar', '{}' }
|
103
|
+
conn.sadd 'queues', 'bar'
|
104
|
+
end
|
113
105
|
|
114
|
-
|
115
|
-
|
106
|
+
s = Sidekiq::Stats.new
|
107
|
+
assert_equal 4, s.enqueued
|
116
108
|
end
|
117
|
-
|
118
|
-
s = Sidekiq::Stats.new
|
119
|
-
assert_equal 4, s.enqueued
|
120
109
|
end
|
121
|
-
end
|
122
110
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
c.incrby("stat:processed:2012-12-26", 6)
|
130
|
-
c.incrby("stat:processed:2012-12-27", 2)
|
131
|
-
end
|
132
|
-
Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
|
133
|
-
s = Sidekiq::Stats::History.new(2)
|
134
|
-
assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.processed
|
135
|
-
|
136
|
-
s = Sidekiq::Stats::History.new(3)
|
137
|
-
assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.processed
|
111
|
+
describe "over time" do
|
112
|
+
before do
|
113
|
+
require 'active_support/core_ext/time/conversions'
|
114
|
+
@before = Time::DATE_FORMATS[:default]
|
115
|
+
Time::DATE_FORMATS[:default] = "%d/%m/%Y %H:%M:%S"
|
116
|
+
end
|
138
117
|
|
139
|
-
|
140
|
-
|
141
|
-
end
|
118
|
+
after do
|
119
|
+
Time::DATE_FORMATS[:default] = @before
|
142
120
|
end
|
143
|
-
end
|
144
121
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
122
|
+
describe "processed" do
|
123
|
+
it 'retrieves hash of dates' do
|
124
|
+
Sidekiq.redis do |c|
|
125
|
+
c.incrby("stat:processed:2012-12-24", 4)
|
126
|
+
c.incrby("stat:processed:2012-12-25", 1)
|
127
|
+
c.incrby("stat:processed:2012-12-26", 6)
|
128
|
+
c.incrby("stat:processed:2012-12-27", 2)
|
129
|
+
end
|
130
|
+
Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
|
131
|
+
s = Sidekiq::Stats::History.new(2)
|
132
|
+
assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1 }, s.processed)
|
133
|
+
|
134
|
+
s = Sidekiq::Stats::History.new(3)
|
135
|
+
assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
|
136
|
+
|
137
|
+
s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
|
138
|
+
assert_equal({ "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
|
139
|
+
end
|
152
140
|
end
|
153
|
-
|
154
|
-
s = Sidekiq::Stats::History.new(2)
|
155
|
-
assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.failed
|
156
|
-
|
157
|
-
s = Sidekiq::Stats::History.new(3)
|
158
|
-
assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
|
141
|
+
end
|
159
142
|
|
160
|
-
|
161
|
-
|
143
|
+
describe "failed" do
|
144
|
+
it 'retrieves hash of dates' do
|
145
|
+
Sidekiq.redis do |c|
|
146
|
+
c.incrby("stat:failed:2012-12-24", 4)
|
147
|
+
c.incrby("stat:failed:2012-12-25", 1)
|
148
|
+
c.incrby("stat:failed:2012-12-26", 6)
|
149
|
+
c.incrby("stat:failed:2012-12-27", 2)
|
150
|
+
end
|
151
|
+
Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
|
152
|
+
s = Sidekiq::Stats::History.new(2)
|
153
|
+
assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.failed
|
154
|
+
|
155
|
+
s = Sidekiq::Stats::History.new(3)
|
156
|
+
assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
|
157
|
+
|
158
|
+
s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
|
159
|
+
assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
|
160
|
+
end
|
162
161
|
end
|
163
162
|
end
|
164
163
|
end
|
165
164
|
end
|
166
|
-
end
|
167
|
-
|
168
|
-
describe 'with an empty database' do
|
169
|
-
include Sidekiq::Util
|
170
165
|
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
166
|
+
describe 'with an empty database' do
|
167
|
+
it 'shows queue as empty' do
|
168
|
+
q = Sidekiq::Queue.new
|
169
|
+
assert_equal 0, q.size
|
170
|
+
assert_equal 0, q.latency
|
171
|
+
end
|
175
172
|
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
assert_equal 0, q.latency
|
180
|
-
end
|
173
|
+
before do
|
174
|
+
ActiveJob::Base.queue_adapter = :sidekiq
|
175
|
+
ActiveJob::Base.logger = nil
|
181
176
|
|
182
|
-
|
183
|
-
|
184
|
-
|
177
|
+
class ApiMailer < ActionMailer::Base
|
178
|
+
def test_email(*)
|
179
|
+
end
|
180
|
+
end
|
185
181
|
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
182
|
+
class ApiJob < ActiveJob::Base
|
183
|
+
def perform(*)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
191
187
|
|
192
|
-
|
193
|
-
|
194
|
-
assert_equal [1, 'mike'], job.args
|
195
|
-
assert_equal Time.new(2012, 12, 26), job.enqueued_at
|
188
|
+
class ApiWorker
|
189
|
+
include Sidekiq::Worker
|
196
190
|
end
|
197
191
|
|
198
|
-
|
192
|
+
it 'can enumerate jobs' do
|
193
|
+
q = Sidekiq::Queue.new
|
194
|
+
Time.stub(:now, Time.new(2012, 12, 26)) do
|
195
|
+
ApiWorker.perform_async(1, 'mike')
|
196
|
+
assert_equal ['TestApi::ApiWorker'], q.map(&:klass)
|
199
197
|
|
200
|
-
|
201
|
-
|
202
|
-
|
198
|
+
job = q.first
|
199
|
+
assert_equal 24, job.jid.size
|
200
|
+
assert_equal [1, 'mike'], job.args
|
201
|
+
assert_equal Time.new(2012, 12, 26), job.enqueued_at
|
202
|
+
end
|
203
|
+
assert q.latency > 10_000_000
|
203
204
|
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
x = q.first
|
208
|
-
assert_equal "TestApi::ApiWorker.foo", x.display_class
|
209
|
-
assert_equal [1,2,3], x.display_args
|
210
|
-
end
|
205
|
+
q = Sidekiq::Queue.new('other')
|
206
|
+
assert_equal 0, q.size
|
207
|
+
end
|
211
208
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
209
|
+
it 'has no enqueued_at time for jobs enqueued in the future' do
|
210
|
+
job_id = ApiWorker.perform_in(100, 1, 'foo')
|
211
|
+
job = Sidekiq::ScheduledSet.new.find_job(job_id)
|
212
|
+
assert_nil job.enqueued_at
|
213
|
+
end
|
216
214
|
|
217
|
-
|
218
|
-
|
219
|
-
|
215
|
+
it 'unwraps delayed jobs' do
|
216
|
+
Sidekiq::Queue.delay.foo(1,2,3)
|
217
|
+
q = Sidekiq::Queue.new
|
218
|
+
x = q.first
|
219
|
+
assert_equal "Sidekiq::Queue.foo", x.display_class
|
220
|
+
assert_equal [1,2,3], x.display_args
|
221
|
+
end
|
220
222
|
|
221
|
-
|
222
|
-
|
223
|
-
|
223
|
+
it 'unwraps ActiveJob jobs' do
|
224
|
+
ApiJob.perform_later(1, 2, 3)
|
225
|
+
q = Sidekiq::Queue.new
|
226
|
+
x = q.first
|
227
|
+
assert_equal "TestApi::ApiJob", x.display_class
|
228
|
+
assert_equal [1,2,3], x.display_args
|
229
|
+
end
|
224
230
|
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
refute_nil queued_job
|
233
|
-
assert_equal queued_job.jid, job_id
|
234
|
-
assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
|
235
|
-
refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
|
236
|
-
end
|
231
|
+
it 'unwraps ActionMailer jobs' do
|
232
|
+
ApiMailer.test_email(1, 2, 3).deliver_later
|
233
|
+
q = Sidekiq::Queue.new('mailers')
|
234
|
+
x = q.first
|
235
|
+
assert_equal "TestApi::ApiMailer#test_email", x.display_class
|
236
|
+
assert_equal [1,2,3], x.display_args
|
237
|
+
end
|
237
238
|
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
(remain_id, job_id) = jids
|
244
|
-
job = Sidekiq::ScheduledSet.new.find_job(job_id)
|
245
|
-
q = Sidekiq::Queue.new
|
246
|
-
job.add_to_queue
|
247
|
-
queued_job = q.find_job(job_id)
|
248
|
-
refute_nil queued_job
|
249
|
-
assert_equal queued_job.jid, job_id
|
250
|
-
assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
|
251
|
-
refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
|
252
|
-
end
|
239
|
+
it 'has no enqueued_at time for jobs enqueued in the future' do
|
240
|
+
job_id = ApiWorker.perform_in(100, 1, 'foo')
|
241
|
+
job = Sidekiq::ScheduledSet.new.find_job(job_id)
|
242
|
+
assert_nil job.enqueued_at
|
243
|
+
end
|
253
244
|
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
245
|
+
it 'can delete jobs' do
|
246
|
+
q = Sidekiq::Queue.new
|
247
|
+
ApiWorker.perform_async(1, 'mike')
|
248
|
+
assert_equal 1, q.size
|
249
|
+
|
250
|
+
x = q.first
|
251
|
+
assert_equal "TestApi::ApiWorker", x.display_class
|
252
|
+
assert_equal [1,'mike'], x.display_args
|
261
253
|
|
262
|
-
|
263
|
-
|
264
|
-
51.times do
|
265
|
-
ApiWorker.perform_in(100, 'aaron')
|
254
|
+
assert_equal [true], q.map(&:delete)
|
255
|
+
assert_equal 0, q.size
|
266
256
|
end
|
267
|
-
set = Sidekiq::ScheduledSet.new
|
268
|
-
set.map(&:delete)
|
269
|
-
assert_equal set.size, 0
|
270
|
-
end
|
271
257
|
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
258
|
+
it "can move scheduled job to queue" do
|
259
|
+
remain_id = ApiWorker.perform_in(100, 1, 'jason')
|
260
|
+
job_id = ApiWorker.perform_in(100, 1, 'jason')
|
261
|
+
job = Sidekiq::ScheduledSet.new.find_job(job_id)
|
262
|
+
q = Sidekiq::Queue.new
|
263
|
+
job.add_to_queue
|
264
|
+
queued_job = q.find_job(job_id)
|
265
|
+
refute_nil queued_job
|
266
|
+
assert_equal queued_job.jid, job_id
|
267
|
+
assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
|
268
|
+
refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
|
276
269
|
end
|
277
|
-
q = Sidekiq::Queue.new
|
278
|
-
q.map(&:delete)
|
279
|
-
assert_equal q.size, 0
|
280
|
-
end
|
281
270
|
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
271
|
+
it "handles multiple scheduled jobs when moving to queue" do
|
272
|
+
jids = Sidekiq::Client.push_bulk('class' => ApiWorker,
|
273
|
+
'args' => [[1, 'jason'], [2, 'jason']],
|
274
|
+
'at' => Time.now.to_f)
|
275
|
+
assert_equal 2, jids.size
|
276
|
+
(remain_id, job_id) = jids
|
277
|
+
job = Sidekiq::ScheduledSet.new.find_job(job_id)
|
278
|
+
q = Sidekiq::Queue.new
|
279
|
+
job.add_to_queue
|
280
|
+
queued_job = q.find_job(job_id)
|
281
|
+
refute_nil queued_job
|
282
|
+
assert_equal queued_job.jid, job_id
|
283
|
+
assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
|
284
|
+
refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
|
285
|
+
end
|
289
286
|
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
287
|
+
it 'can find job by id in sorted sets' do
|
288
|
+
job_id = ApiWorker.perform_in(100, 1, 'jason')
|
289
|
+
job = Sidekiq::ScheduledSet.new.find_job(job_id)
|
290
|
+
refute_nil job
|
291
|
+
assert_equal job_id, job.jid
|
292
|
+
assert_in_delta job.latency, 0.0, 0.1
|
293
|
+
end
|
294
294
|
|
295
|
-
|
296
|
-
|
297
|
-
|
295
|
+
it 'can remove jobs when iterating over a sorted set' do
|
296
|
+
# scheduled jobs must be greater than SortedSet#each underlying page size
|
297
|
+
51.times do
|
298
|
+
ApiWorker.perform_in(100, 'aaron')
|
299
|
+
end
|
300
|
+
set = Sidekiq::ScheduledSet.new
|
301
|
+
set.map(&:delete)
|
302
|
+
assert_equal set.size, 0
|
298
303
|
end
|
299
|
-
end
|
300
304
|
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
305
|
+
it 'can remove jobs when iterating over a queue' do
|
306
|
+
# initial queue size must be greater than Queue#each underlying page size
|
307
|
+
51.times do
|
308
|
+
ApiWorker.perform_async(1, 'aaron')
|
309
|
+
end
|
310
|
+
q = Sidekiq::Queue.new
|
311
|
+
q.map(&:delete)
|
312
|
+
assert_equal q.size, 0
|
313
|
+
end
|
308
314
|
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
315
|
+
it 'can find job by id in queues' do
|
316
|
+
q = Sidekiq::Queue.new
|
317
|
+
job_id = ApiWorker.perform_async(1, 'jason')
|
318
|
+
job = q.find_job(job_id)
|
319
|
+
refute_nil job
|
320
|
+
assert_equal job_id, job.jid
|
321
|
+
end
|
316
322
|
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
323
|
+
it 'can clear a queue' do
|
324
|
+
q = Sidekiq::Queue.new
|
325
|
+
2.times { ApiWorker.perform_async(1, 'mike') }
|
326
|
+
q.clear
|
321
327
|
|
322
|
-
|
323
|
-
|
328
|
+
Sidekiq.redis do |conn|
|
329
|
+
refute conn.smembers('queues').include?('foo')
|
330
|
+
refute conn.exists('queue:foo')
|
331
|
+
end
|
332
|
+
end
|
324
333
|
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
334
|
+
it 'can fetch by score' do
|
335
|
+
same_time = Time.now.to_f
|
336
|
+
add_retry('bob1', same_time)
|
337
|
+
add_retry('bob2', same_time)
|
338
|
+
r = Sidekiq::RetrySet.new
|
339
|
+
assert_equal 2, r.fetch(same_time).size
|
340
|
+
end
|
329
341
|
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
342
|
+
it 'can fetch by score and jid' do
|
343
|
+
same_time = Time.now.to_f
|
344
|
+
add_retry('bob1', same_time)
|
345
|
+
add_retry('bob2', same_time)
|
346
|
+
r = Sidekiq::RetrySet.new
|
347
|
+
assert_equal 1, r.fetch(same_time, 'bob1').size
|
348
|
+
end
|
336
349
|
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
r = Sidekiq::RetrySet.new
|
342
|
-
assert_equal 2, r.size
|
343
|
-
Sidekiq::RetrySet.new.delete(same_time)
|
344
|
-
assert_equal 0, r.size
|
345
|
-
end
|
350
|
+
it 'shows empty retries' do
|
351
|
+
r = Sidekiq::RetrySet.new
|
352
|
+
assert_equal 0, r.size
|
353
|
+
end
|
346
354
|
|
347
|
-
|
348
|
-
|
349
|
-
add_retry('bob1', same_time)
|
350
|
-
add_retry('bob2', same_time)
|
351
|
-
r = Sidekiq::RetrySet.new
|
352
|
-
assert_equal 2, r.size
|
353
|
-
Sidekiq::RetrySet.new.delete(same_time, 'bob1')
|
354
|
-
assert_equal 1, r.size
|
355
|
-
end
|
355
|
+
it 'can enumerate retries' do
|
356
|
+
add_retry
|
356
357
|
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
r.first.retry
|
362
|
-
assert_equal 0, r.size
|
363
|
-
assert_equal 1, Sidekiq::Queue.new('default').size
|
364
|
-
job = Sidekiq::Queue.new('default').first
|
365
|
-
assert_equal 'bob', job.jid
|
366
|
-
assert_equal 1, job['retry_count']
|
367
|
-
end
|
358
|
+
r = Sidekiq::RetrySet.new
|
359
|
+
assert_equal 1, r.size
|
360
|
+
array = r.to_a
|
361
|
+
assert_equal 1, array.size
|
368
362
|
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
assert_equal 0, r.size
|
376
|
-
end
|
363
|
+
retri = array.first
|
364
|
+
assert_equal 'ApiWorker', retri.klass
|
365
|
+
assert_equal 'default', retri.queue
|
366
|
+
assert_equal 'bob', retri.jid
|
367
|
+
assert_in_delta Time.now.to_f, retri.at.to_f, 0.02
|
368
|
+
end
|
377
369
|
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
'key' => identity_string,
|
384
|
-
'identity' => identity_string,
|
385
|
-
'started_at' => Time.now.to_f - 15,
|
386
|
-
}
|
387
|
-
|
388
|
-
time = Time.now.to_f
|
389
|
-
Sidekiq.redis do |conn|
|
390
|
-
conn.multi do
|
391
|
-
conn.sadd('processes', odata['key'])
|
392
|
-
conn.hmset(odata['key'], 'info', Sidekiq.dump_json(odata), 'busy', 10, 'beat', time)
|
393
|
-
conn.sadd('processes', 'fake:pid')
|
370
|
+
it 'requires a jid to delete an entry' do
|
371
|
+
start_time = Time.now.to_f
|
372
|
+
add_retry('bob2', Time.now.to_f)
|
373
|
+
assert_raises(ArgumentError) do
|
374
|
+
Sidekiq::RetrySet.new.delete(start_time)
|
394
375
|
end
|
395
376
|
end
|
396
377
|
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
signals_string = "#{odata['key']}-signals"
|
406
|
-
assert_equal "TERM", Sidekiq.redis{|c| c.lpop(signals_string) }
|
407
|
-
assert_equal "USR1", Sidekiq.redis{|c| c.lpop(signals_string) }
|
408
|
-
end
|
409
|
-
|
410
|
-
it 'can enumerate workers' do
|
411
|
-
w = Sidekiq::Workers.new
|
412
|
-
assert_equal 0, w.size
|
413
|
-
w.each do
|
414
|
-
assert false
|
378
|
+
it 'can delete a single retry from score and jid' do
|
379
|
+
same_time = Time.now.to_f
|
380
|
+
add_retry('bob1', same_time)
|
381
|
+
add_retry('bob2', same_time)
|
382
|
+
r = Sidekiq::RetrySet.new
|
383
|
+
assert_equal 2, r.size
|
384
|
+
Sidekiq::RetrySet.new.delete(same_time, 'bob1')
|
385
|
+
assert_equal 1, r.size
|
415
386
|
end
|
416
387
|
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
388
|
+
it 'can retry a retry' do
|
389
|
+
add_retry
|
390
|
+
r = Sidekiq::RetrySet.new
|
391
|
+
assert_equal 1, r.size
|
392
|
+
r.first.retry
|
393
|
+
assert_equal 0, r.size
|
394
|
+
assert_equal 1, Sidekiq::Queue.new('default').size
|
395
|
+
job = Sidekiq::Queue.new('default').first
|
396
|
+
assert_equal 'bob', job.jid
|
397
|
+
assert_equal 1, job['retry_count']
|
422
398
|
end
|
423
399
|
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
400
|
+
it 'can clear retries' do
|
401
|
+
add_retry
|
402
|
+
add_retry('test')
|
403
|
+
r = Sidekiq::RetrySet.new
|
404
|
+
assert_equal 2, r.size
|
405
|
+
r.clear
|
406
|
+
assert_equal 0, r.size
|
428
407
|
end
|
429
408
|
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
409
|
+
it 'can enumerate processes' do
|
410
|
+
identity_string = "identity_string"
|
411
|
+
odata = {
|
412
|
+
'pid' => 123,
|
413
|
+
'hostname' => Socket.gethostname,
|
414
|
+
'key' => identity_string,
|
415
|
+
'identity' => identity_string,
|
416
|
+
'started_at' => Time.now.to_f - 15,
|
417
|
+
}
|
418
|
+
|
419
|
+
time = Time.now.to_f
|
420
|
+
Sidekiq.redis do |conn|
|
421
|
+
conn.multi do
|
422
|
+
conn.sadd('processes', odata['key'])
|
423
|
+
conn.hmset(odata['key'], 'info', Sidekiq.dump_json(odata), 'busy', 10, 'beat', time)
|
424
|
+
conn.sadd('processes', 'fake:pid')
|
425
|
+
end
|
426
|
+
end
|
427
|
+
|
428
|
+
ps = Sidekiq::ProcessSet.new.to_a
|
429
|
+
assert_equal 1, ps.size
|
430
|
+
data = ps.first
|
431
|
+
assert_equal 10, data['busy']
|
432
|
+
assert_equal time, data['beat']
|
433
|
+
assert_equal 123, data['pid']
|
434
|
+
data.quiet!
|
435
|
+
data.stop!
|
436
|
+
signals_string = "#{odata['key']}-signals"
|
437
|
+
assert_equal "TERM", Sidekiq.redis{|c| c.lpop(signals_string) }
|
438
|
+
assert_equal "USR1", Sidekiq.redis{|c| c.lpop(signals_string) }
|
435
439
|
end
|
436
440
|
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
c.hmset("b#{s}", '5678', data)
|
441
|
+
it 'can enumerate workers' do
|
442
|
+
w = Sidekiq::Workers.new
|
443
|
+
assert_equal 0, w.size
|
444
|
+
w.each do
|
445
|
+
assert false
|
443
446
|
end
|
444
|
-
end
|
445
447
|
|
446
|
-
|
447
|
-
|
448
|
+
hn = Socket.gethostname
|
449
|
+
key = "#{hn}:#{$$}"
|
450
|
+
pdata = { 'pid' => $$, 'hostname' => hn, 'started_at' => Time.now.to_i }
|
451
|
+
Sidekiq.redis do |conn|
|
452
|
+
conn.sadd('processes', key)
|
453
|
+
conn.hmset(key, 'info', Sidekiq.dump_json(pdata), 'busy', 0, 'beat', Time.now.to_f)
|
454
|
+
end
|
448
455
|
|
449
|
-
|
450
|
-
|
451
|
-
|
456
|
+
s = "#{key}:workers"
|
457
|
+
data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => Time.now.to_i })
|
458
|
+
Sidekiq.redis do |c|
|
459
|
+
c.hmset(s, '1234', data)
|
460
|
+
end
|
452
461
|
|
453
|
-
|
454
|
-
|
455
|
-
|
462
|
+
w.each do |p, x, y|
|
463
|
+
assert_equal key, p
|
464
|
+
assert_equal "1234", x
|
465
|
+
assert_equal 'default', y['queue']
|
466
|
+
assert_equal Time.now.year, Time.at(y['run_at']).year
|
467
|
+
end
|
456
468
|
|
457
|
-
|
458
|
-
|
469
|
+
s = "#{key}:workers"
|
470
|
+
data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => (Time.now.to_i - 2*60*60) })
|
471
|
+
Sidekiq.redis do |c|
|
472
|
+
c.multi do
|
473
|
+
c.hmset(s, '5678', data)
|
474
|
+
c.hmset("b#{s}", '5678', data)
|
475
|
+
end
|
476
|
+
end
|
477
|
+
|
478
|
+
assert_equal ['1234', '5678'], w.map { |_, tid, _| tid }
|
459
479
|
end
|
460
480
|
|
461
|
-
|
462
|
-
|
463
|
-
|
481
|
+
it 'can reschedule jobs' do
|
482
|
+
add_retry('foo1')
|
483
|
+
add_retry('foo2')
|
464
484
|
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
Sidekiq.redis do |conn|
|
469
|
-
conn.sadd('processes', key)
|
470
|
-
conn.hmset(key, 'info', Sidekiq.dump_json(data), 'busy', 0, 'beat', Time.now.to_f)
|
471
|
-
end
|
485
|
+
retries = Sidekiq::RetrySet.new
|
486
|
+
assert_equal 2, retries.size
|
487
|
+
refute(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
|
472
488
|
|
473
|
-
|
474
|
-
|
475
|
-
|
489
|
+
retries.each do |retri|
|
490
|
+
retri.reschedule(Time.now.to_f + 10) if retri.jid == 'foo2'
|
491
|
+
end
|
476
492
|
|
477
|
-
|
478
|
-
|
479
|
-
conn.sadd('processes', "bar:986")
|
493
|
+
assert_equal 2, retries.size
|
494
|
+
assert(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
|
480
495
|
end
|
481
496
|
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
497
|
+
it 'prunes processes which have died' do
|
498
|
+
data = { 'pid' => rand(10_000), 'hostname' => "app#{rand(1_000)}", 'started_at' => Time.now.to_f }
|
499
|
+
key = "#{data['hostname']}:#{data['pid']}"
|
500
|
+
Sidekiq.redis do |conn|
|
501
|
+
conn.sadd('processes', key)
|
502
|
+
conn.hmset(key, 'info', Sidekiq.dump_json(data), 'busy', 0, 'beat', Time.now.to_f)
|
503
|
+
end
|
486
504
|
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
505
|
+
ps = Sidekiq::ProcessSet.new
|
506
|
+
assert_equal 1, ps.size
|
507
|
+
assert_equal 1, ps.to_a.size
|
508
|
+
|
509
|
+
Sidekiq.redis do |conn|
|
510
|
+
conn.sadd('processes', "bar:987")
|
511
|
+
conn.sadd('processes', "bar:986")
|
512
|
+
end
|
513
|
+
|
514
|
+
ps = Sidekiq::ProcessSet.new
|
515
|
+
assert_equal 1, ps.size
|
516
|
+
assert_equal 1, ps.to_a.size
|
517
|
+
end
|
518
|
+
|
519
|
+
def add_retry(jid = 'bob', at = Time.now.to_f)
|
520
|
+
payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.to_f)
|
521
|
+
Sidekiq.redis do |conn|
|
522
|
+
conn.zadd('retry', at.to_s, payload)
|
523
|
+
end
|
491
524
|
end
|
492
525
|
end
|
493
526
|
end
|