sidekiq 3.4.1 → 4.0.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +3 -2
  3. data/4.0-Upgrade.md +50 -0
  4. data/COMM-LICENSE +55 -45
  5. data/Changes.md +85 -1
  6. data/Ent-Changes.md +79 -0
  7. data/Gemfile +7 -1
  8. data/Pro-2.0-Upgrade.md +2 -2
  9. data/Pro-3.0-Upgrade.md +46 -0
  10. data/Pro-Changes.md +60 -2
  11. data/README.md +20 -16
  12. data/bin/sidekiq +4 -0
  13. data/bin/sidekiqctl +8 -2
  14. data/bin/sidekiqload +167 -0
  15. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +2 -2
  16. data/lib/generators/sidekiq/templates/worker_test.rb.erb +5 -5
  17. data/lib/sidekiq/api.rb +43 -33
  18. data/lib/sidekiq/cli.rb +41 -42
  19. data/lib/sidekiq/client.rb +5 -10
  20. data/lib/sidekiq/fetch.rb +35 -111
  21. data/lib/sidekiq/launcher.rb +102 -42
  22. data/lib/sidekiq/manager.rb +80 -180
  23. data/lib/sidekiq/middleware/server/logging.rb +13 -8
  24. data/lib/sidekiq/middleware/server/retry_jobs.rb +6 -6
  25. data/lib/sidekiq/processor.rb +126 -97
  26. data/lib/sidekiq/redis_connection.rb +23 -5
  27. data/lib/sidekiq/scheduled.rb +47 -26
  28. data/lib/sidekiq/testing.rb +139 -17
  29. data/lib/sidekiq/util.rb +20 -0
  30. data/lib/sidekiq/version.rb +1 -1
  31. data/lib/sidekiq/web.rb +17 -1
  32. data/lib/sidekiq/web_helpers.rb +33 -5
  33. data/lib/sidekiq/worker.rb +16 -0
  34. data/lib/sidekiq.rb +37 -14
  35. data/sidekiq.gemspec +10 -11
  36. data/test/helper.rb +45 -10
  37. data/test/test_actors.rb +137 -0
  38. data/test/test_api.rb +417 -384
  39. data/test/test_cli.rb +29 -59
  40. data/test/test_client.rb +60 -135
  41. data/test/test_extensions.rb +29 -23
  42. data/test/test_fetch.rb +2 -57
  43. data/test/test_launcher.rb +80 -0
  44. data/test/test_logging.rb +1 -1
  45. data/test/test_manager.rb +16 -131
  46. data/test/test_middleware.rb +3 -5
  47. data/test/test_processor.rb +110 -76
  48. data/test/test_rails.rb +21 -0
  49. data/test/test_redis_connection.rb +0 -1
  50. data/test/test_retry.rb +114 -162
  51. data/test/test_scheduled.rb +11 -17
  52. data/test/test_scheduling.rb +20 -42
  53. data/test/test_sidekiq.rb +46 -16
  54. data/test/test_testing.rb +80 -20
  55. data/test/test_testing_fake.rb +83 -8
  56. data/test/test_testing_inline.rb +3 -3
  57. data/test/test_util.rb +16 -0
  58. data/test/test_web.rb +28 -9
  59. data/test/test_web_helpers.rb +3 -2
  60. data/web/assets/images/favicon.ico +0 -0
  61. data/web/assets/javascripts/application.js +6 -1
  62. data/web/assets/javascripts/dashboard.js +2 -8
  63. data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +14 -14
  64. data/web/assets/stylesheets/application.css +33 -56
  65. data/web/locales/de.yml +1 -1
  66. data/web/locales/en.yml +2 -0
  67. data/web/locales/fr.yml +2 -2
  68. data/web/locales/ja.yml +10 -1
  69. data/web/locales/{no.yml → nb.yml} +10 -2
  70. data/web/locales/uk.yml +76 -0
  71. data/web/views/_footer.erb +2 -7
  72. data/web/views/_job_info.erb +5 -1
  73. data/web/views/_nav.erb +2 -2
  74. data/web/views/_poll_js.erb +5 -0
  75. data/web/views/{_poll.erb → _poll_link.erb} +0 -3
  76. data/web/views/busy.erb +2 -1
  77. data/web/views/dead.erb +1 -0
  78. data/web/views/layout.erb +2 -0
  79. data/web/views/morgue.erb +3 -0
  80. data/web/views/queue.erb +1 -0
  81. data/web/views/queues.erb +1 -0
  82. data/web/views/retries.erb +3 -0
  83. data/web/views/retry.erb +1 -0
  84. data/web/views/scheduled.erb +1 -0
  85. data/web/views/scheduled_job_info.erb +1 -0
  86. metadata +75 -55
  87. data/lib/sidekiq/actor.rb +0 -39
  88. data/test/test_worker_generator.rb +0 -17
data/test/test_api.rb CHANGED
@@ -1,493 +1,526 @@
1
1
  require_relative 'helper'
2
+ require 'sidekiq/api'
3
+ require 'active_job'
4
+ require 'action_mailer'
2
5
 
3
6
  class TestApi < Sidekiq::Test
4
-
5
- describe "stats" do
7
+ describe 'api' do
6
8
  before do
7
- Sidekiq.redis = REDIS
8
9
  Sidekiq.redis {|c| c.flushdb }
9
10
  end
10
11
 
11
- describe "processed" do
12
+ describe "stats" do
12
13
  it "is initially zero" do
13
14
  s = Sidekiq::Stats.new
14
15
  assert_equal 0, s.processed
16
+ assert_equal 0, s.failed
17
+ assert_equal 0, s.enqueued
15
18
  end
16
19
 
17
- it "returns number of processed jobs" do
18
- Sidekiq.redis { |conn| conn.set("stat:processed", 5) }
19
- s = Sidekiq::Stats.new
20
- assert_equal 5, s.processed
20
+ describe "processed" do
21
+ it "returns number of processed jobs" do
22
+ Sidekiq.redis { |conn| conn.set("stat:processed", 5) }
23
+ s = Sidekiq::Stats.new
24
+ assert_equal 5, s.processed
25
+ end
21
26
  end
22
- end
23
27
 
24
- describe "failed" do
25
- it "is initially zero" do
26
- s = Sidekiq::Stats.new
27
- assert_equal 0, s.failed
28
+ describe "failed" do
29
+ it "returns number of failed jobs" do
30
+ Sidekiq.redis { |conn| conn.set("stat:failed", 5) }
31
+ s = Sidekiq::Stats.new
32
+ assert_equal 5, s.failed
33
+ end
28
34
  end
29
35
 
30
- it "returns number of failed jobs" do
31
- Sidekiq.redis { |conn| conn.set("stat:failed", 5) }
32
- s = Sidekiq::Stats.new
33
- assert_equal 5, s.failed
34
- end
35
- end
36
+ describe "reset" do
37
+ before do
38
+ Sidekiq.redis do |conn|
39
+ conn.set('stat:processed', 5)
40
+ conn.set('stat:failed', 10)
41
+ end
42
+ end
36
43
 
37
- describe "reset" do
38
- before do
39
- Sidekiq.redis do |conn|
40
- conn.set('stat:processed', 5)
41
- conn.set('stat:failed', 10)
44
+ it 'will reset all stats by default' do
45
+ Sidekiq::Stats.new.reset
46
+ s = Sidekiq::Stats.new
47
+ assert_equal 0, s.failed
48
+ assert_equal 0, s.processed
42
49
  end
43
- end
44
50
 
45
- it 'will reset all stats by default' do
46
- Sidekiq::Stats.new.reset
47
- Sidekiq.redis do |conn|
48
- assert_equal '0', conn.get('stat:processed')
49
- assert_equal '0', conn.get('stat:failed')
51
+ it 'can reset individual stats' do
52
+ Sidekiq::Stats.new.reset('failed')
53
+ s = Sidekiq::Stats.new
54
+ assert_equal 0, s.failed
55
+ assert_equal 5, s.processed
50
56
  end
51
- end
52
57
 
53
- it 'can reset individual stats' do
54
- Sidekiq::Stats.new.reset('failed')
55
- Sidekiq.redis do |conn|
56
- assert_equal '5', conn.get('stat:processed')
57
- assert_equal '0', conn.get('stat:failed')
58
+ it 'can accept anything that responds to #to_s' do
59
+ Sidekiq::Stats.new.reset(:failed)
60
+ s = Sidekiq::Stats.new
61
+ assert_equal 0, s.failed
62
+ assert_equal 5, s.processed
58
63
  end
59
- end
60
64
 
61
- it 'can accept anything that responds to #to_s' do
62
- Sidekiq::Stats.new.reset(:failed)
63
- Sidekiq.redis do |conn|
64
- assert_equal '5', conn.get('stat:processed')
65
- assert_equal '0', conn.get('stat:failed')
65
+ it 'ignores anything other than "failed" or "processed"' do
66
+ Sidekiq::Stats.new.reset((1..10).to_a, ['failed'])
67
+ s = Sidekiq::Stats.new
68
+ assert_equal 0, s.failed
69
+ assert_equal 5, s.processed
66
70
  end
67
71
  end
68
72
 
69
- it 'ignores anything other than "failed" or "processed"' do
70
- Sidekiq::Stats.new.reset((1..10).to_a, ['failed'])
71
- Sidekiq.redis do |conn|
72
- assert_equal '5', conn.get('stat:processed')
73
- assert_equal '0', conn.get('stat:failed')
73
+ describe "queues" do
74
+ it "is initially empty" do
75
+ s = Sidekiq::Stats::Queues.new
76
+ assert_equal 0, s.lengths.size
74
77
  end
75
- end
76
- end
77
78
 
78
- describe "queues" do
79
- it "returns all queues" do
80
- assert_equal Sidekiq::Stats.new.queues, Sidekiq::Stats::Queues.new.lengths
81
- end
79
+ it "returns a hash of queue and size in order" do
80
+ Sidekiq.redis do |conn|
81
+ conn.rpush 'queue:foo', '{}'
82
+ conn.sadd 'queues', 'foo'
82
83
 
83
- it "is initially empty" do
84
- s = Sidekiq::Stats::Queues.new
85
- assert_equal 0, s.lengths.size
86
- end
84
+ 3.times { conn.rpush 'queue:bar', '{}' }
85
+ conn.sadd 'queues', 'bar'
86
+ end
87
87
 
88
- it "returns a hash of queue and size in order" do
89
- Sidekiq.redis do |conn|
90
- conn.rpush 'queue:foo', '{}'
91
- conn.sadd 'queues', 'foo'
88
+ s = Sidekiq::Stats::Queues.new
89
+ assert_equal ({ "foo" => 1, "bar" => 3 }), s.lengths
90
+ assert_equal "bar", s.lengths.first.first
92
91
 
93
- 3.times { conn.rpush 'queue:bar', '{}' }
94
- conn.sadd 'queues', 'bar'
92
+ assert_equal Sidekiq::Stats.new.queues, Sidekiq::Stats::Queues.new.lengths
95
93
  end
96
-
97
- s = Sidekiq::Stats::Queues.new
98
- assert_equal ({ "foo" => 1, "bar" => 3 }), s.lengths
99
- assert_equal "bar", s.lengths.first.first
100
94
  end
101
- end
102
95
 
103
- describe "enqueued" do
104
- it "is initially empty" do
105
- s = Sidekiq::Stats.new
106
- assert_equal 0, s.enqueued
107
- end
96
+ describe "enqueued" do
97
+ it "returns total enqueued jobs" do
98
+ Sidekiq.redis do |conn|
99
+ conn.rpush 'queue:foo', '{}'
100
+ conn.sadd 'queues', 'foo'
108
101
 
109
- it "returns total enqueued jobs" do
110
- Sidekiq.redis do |conn|
111
- conn.rpush 'queue:foo', '{}'
112
- conn.sadd 'queues', 'foo'
102
+ 3.times { conn.rpush 'queue:bar', '{}' }
103
+ conn.sadd 'queues', 'bar'
104
+ end
113
105
 
114
- 3.times { conn.rpush 'queue:bar', '{}' }
115
- conn.sadd 'queues', 'bar'
106
+ s = Sidekiq::Stats.new
107
+ assert_equal 4, s.enqueued
116
108
  end
117
-
118
- s = Sidekiq::Stats.new
119
- assert_equal 4, s.enqueued
120
109
  end
121
- end
122
110
 
123
- describe "over time" do
124
- describe "processed" do
125
- it 'retrieves hash of dates' do
126
- Sidekiq.redis do |c|
127
- c.incrby("stat:processed:2012-12-24", 4)
128
- c.incrby("stat:processed:2012-12-25", 1)
129
- c.incrby("stat:processed:2012-12-26", 6)
130
- c.incrby("stat:processed:2012-12-27", 2)
131
- end
132
- Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
133
- s = Sidekiq::Stats::History.new(2)
134
- assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.processed
135
-
136
- s = Sidekiq::Stats::History.new(3)
137
- assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.processed
111
+ describe "over time" do
112
+ before do
113
+ require 'active_support/core_ext/time/conversions'
114
+ @before = Time::DATE_FORMATS[:default]
115
+ Time::DATE_FORMATS[:default] = "%d/%m/%Y %H:%M:%S"
116
+ end
138
117
 
139
- s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
140
- assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.processed
141
- end
118
+ after do
119
+ Time::DATE_FORMATS[:default] = @before
142
120
  end
143
- end
144
121
 
145
- describe "failed" do
146
- it 'retrieves hash of dates' do
147
- Sidekiq.redis do |c|
148
- c.incrby("stat:failed:2012-12-24", 4)
149
- c.incrby("stat:failed:2012-12-25", 1)
150
- c.incrby("stat:failed:2012-12-26", 6)
151
- c.incrby("stat:failed:2012-12-27", 2)
122
+ describe "processed" do
123
+ it 'retrieves hash of dates' do
124
+ Sidekiq.redis do |c|
125
+ c.incrby("stat:processed:2012-12-24", 4)
126
+ c.incrby("stat:processed:2012-12-25", 1)
127
+ c.incrby("stat:processed:2012-12-26", 6)
128
+ c.incrby("stat:processed:2012-12-27", 2)
129
+ end
130
+ Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
131
+ s = Sidekiq::Stats::History.new(2)
132
+ assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1 }, s.processed)
133
+
134
+ s = Sidekiq::Stats::History.new(3)
135
+ assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
136
+
137
+ s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
138
+ assert_equal({ "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
139
+ end
152
140
  end
153
- Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
154
- s = Sidekiq::Stats::History.new(2)
155
- assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.failed
156
-
157
- s = Sidekiq::Stats::History.new(3)
158
- assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
141
+ end
159
142
 
160
- s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
161
- assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
143
+ describe "failed" do
144
+ it 'retrieves hash of dates' do
145
+ Sidekiq.redis do |c|
146
+ c.incrby("stat:failed:2012-12-24", 4)
147
+ c.incrby("stat:failed:2012-12-25", 1)
148
+ c.incrby("stat:failed:2012-12-26", 6)
149
+ c.incrby("stat:failed:2012-12-27", 2)
150
+ end
151
+ Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
152
+ s = Sidekiq::Stats::History.new(2)
153
+ assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.failed
154
+
155
+ s = Sidekiq::Stats::History.new(3)
156
+ assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
157
+
158
+ s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
159
+ assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
160
+ end
162
161
  end
163
162
  end
164
163
  end
165
164
  end
166
- end
167
-
168
- describe 'with an empty database' do
169
- include Sidekiq::Util
170
165
 
171
- before do
172
- Sidekiq.redis = REDIS
173
- Sidekiq.redis {|c| c.flushdb }
174
- end
166
+ describe 'with an empty database' do
167
+ it 'shows queue as empty' do
168
+ q = Sidekiq::Queue.new
169
+ assert_equal 0, q.size
170
+ assert_equal 0, q.latency
171
+ end
175
172
 
176
- it 'shows queue as empty' do
177
- q = Sidekiq::Queue.new
178
- assert_equal 0, q.size
179
- assert_equal 0, q.latency
180
- end
173
+ before do
174
+ ActiveJob::Base.queue_adapter = :sidekiq
175
+ ActiveJob::Base.logger = nil
181
176
 
182
- class ApiWorker
183
- include Sidekiq::Worker
184
- end
177
+ class ApiMailer < ActionMailer::Base
178
+ def test_email(*)
179
+ end
180
+ end
185
181
 
186
- it 'can enumerate jobs' do
187
- q = Sidekiq::Queue.new
188
- Time.stub(:now, Time.new(2012, 12, 26)) do
189
- ApiWorker.perform_async(1, 'mike')
190
- assert_equal ['TestApi::ApiWorker'], q.map(&:klass)
182
+ class ApiJob < ActiveJob::Base
183
+ def perform(*)
184
+ end
185
+ end
186
+ end
191
187
 
192
- job = q.first
193
- assert_equal 24, job.jid.size
194
- assert_equal [1, 'mike'], job.args
195
- assert_equal Time.new(2012, 12, 26), job.enqueued_at
188
+ class ApiWorker
189
+ include Sidekiq::Worker
196
190
  end
197
191
 
198
- assert q.latency > 10_000_000
192
+ it 'can enumerate jobs' do
193
+ q = Sidekiq::Queue.new
194
+ Time.stub(:now, Time.new(2012, 12, 26)) do
195
+ ApiWorker.perform_async(1, 'mike')
196
+ assert_equal ['TestApi::ApiWorker'], q.map(&:klass)
199
197
 
200
- q = Sidekiq::Queue.new('other')
201
- assert_equal 0, q.size
202
- end
198
+ job = q.first
199
+ assert_equal 24, job.jid.size
200
+ assert_equal [1, 'mike'], job.args
201
+ assert_equal Time.new(2012, 12, 26), job.enqueued_at
202
+ end
203
+ assert q.latency > 10_000_000
203
204
 
204
- it 'unwraps delayed jobs' do
205
- ApiWorker.delay.foo(1,2,3)
206
- q = Sidekiq::Queue.new
207
- x = q.first
208
- assert_equal "TestApi::ApiWorker.foo", x.display_class
209
- assert_equal [1,2,3], x.display_args
210
- end
205
+ q = Sidekiq::Queue.new('other')
206
+ assert_equal 0, q.size
207
+ end
211
208
 
212
- it 'can delete jobs' do
213
- q = Sidekiq::Queue.new
214
- ApiWorker.perform_async(1, 'mike')
215
- assert_equal 1, q.size
209
+ it 'has no enqueued_at time for jobs enqueued in the future' do
210
+ job_id = ApiWorker.perform_in(100, 1, 'foo')
211
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
212
+ assert_nil job.enqueued_at
213
+ end
216
214
 
217
- x = q.first
218
- assert_equal "TestApi::ApiWorker", x.display_class
219
- assert_equal [1,'mike'], x.display_args
215
+ it 'unwraps delayed jobs' do
216
+ Sidekiq::Queue.delay.foo(1,2,3)
217
+ q = Sidekiq::Queue.new
218
+ x = q.first
219
+ assert_equal "Sidekiq::Queue.foo", x.display_class
220
+ assert_equal [1,2,3], x.display_args
221
+ end
220
222
 
221
- assert_equal [true], q.map(&:delete)
222
- assert_equal 0, q.size
223
- end
223
+ it 'unwraps ActiveJob jobs' do
224
+ ApiJob.perform_later(1, 2, 3)
225
+ q = Sidekiq::Queue.new
226
+ x = q.first
227
+ assert_equal "TestApi::ApiJob", x.display_class
228
+ assert_equal [1,2,3], x.display_args
229
+ end
224
230
 
225
- it "can move scheduled job to queue" do
226
- remain_id = ApiWorker.perform_in(100, 1, 'jason')
227
- job_id = ApiWorker.perform_in(100, 1, 'jason')
228
- job = Sidekiq::ScheduledSet.new.find_job(job_id)
229
- q = Sidekiq::Queue.new
230
- job.add_to_queue
231
- queued_job = q.find_job(job_id)
232
- refute_nil queued_job
233
- assert_equal queued_job.jid, job_id
234
- assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
235
- refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
236
- end
231
+ it 'unwraps ActionMailer jobs' do
232
+ ApiMailer.test_email(1, 2, 3).deliver_later
233
+ q = Sidekiq::Queue.new('mailers')
234
+ x = q.first
235
+ assert_equal "TestApi::ApiMailer#test_email", x.display_class
236
+ assert_equal [1,2,3], x.display_args
237
+ end
237
238
 
238
- it "handles multiple scheduled jobs when moving to queue" do
239
- jids = Sidekiq::Client.push_bulk('class' => ApiWorker,
240
- 'args' => [[1, 'jason'], [2, 'jason']],
241
- 'at' => Time.now.to_f)
242
- assert_equal 2, jids.size
243
- (remain_id, job_id) = jids
244
- job = Sidekiq::ScheduledSet.new.find_job(job_id)
245
- q = Sidekiq::Queue.new
246
- job.add_to_queue
247
- queued_job = q.find_job(job_id)
248
- refute_nil queued_job
249
- assert_equal queued_job.jid, job_id
250
- assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
251
- refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
252
- end
239
+ it 'has no enqueued_at time for jobs enqueued in the future' do
240
+ job_id = ApiWorker.perform_in(100, 1, 'foo')
241
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
242
+ assert_nil job.enqueued_at
243
+ end
253
244
 
254
- it 'can find job by id in sorted sets' do
255
- job_id = ApiWorker.perform_in(100, 1, 'jason')
256
- job = Sidekiq::ScheduledSet.new.find_job(job_id)
257
- refute_nil job
258
- assert_equal job_id, job.jid
259
- assert_in_delta job.latency, 0.0, 0.1
260
- end
245
+ it 'can delete jobs' do
246
+ q = Sidekiq::Queue.new
247
+ ApiWorker.perform_async(1, 'mike')
248
+ assert_equal 1, q.size
249
+
250
+ x = q.first
251
+ assert_equal "TestApi::ApiWorker", x.display_class
252
+ assert_equal [1,'mike'], x.display_args
261
253
 
262
- it 'can remove jobs when iterating over a sorted set' do
263
- # scheduled jobs must be greater than SortedSet#each underlying page size
264
- 51.times do
265
- ApiWorker.perform_in(100, 'aaron')
254
+ assert_equal [true], q.map(&:delete)
255
+ assert_equal 0, q.size
266
256
  end
267
- set = Sidekiq::ScheduledSet.new
268
- set.map(&:delete)
269
- assert_equal set.size, 0
270
- end
271
257
 
272
- it 'can remove jobs when iterating over a queue' do
273
- # initial queue size must be greater than Queue#each underlying page size
274
- 51.times do
275
- ApiWorker.perform_async(1, 'aaron')
258
+ it "can move scheduled job to queue" do
259
+ remain_id = ApiWorker.perform_in(100, 1, 'jason')
260
+ job_id = ApiWorker.perform_in(100, 1, 'jason')
261
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
262
+ q = Sidekiq::Queue.new
263
+ job.add_to_queue
264
+ queued_job = q.find_job(job_id)
265
+ refute_nil queued_job
266
+ assert_equal queued_job.jid, job_id
267
+ assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
268
+ refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
276
269
  end
277
- q = Sidekiq::Queue.new
278
- q.map(&:delete)
279
- assert_equal q.size, 0
280
- end
281
270
 
282
- it 'can find job by id in queues' do
283
- q = Sidekiq::Queue.new
284
- job_id = ApiWorker.perform_async(1, 'jason')
285
- job = q.find_job(job_id)
286
- refute_nil job
287
- assert_equal job_id, job.jid
288
- end
271
+ it "handles multiple scheduled jobs when moving to queue" do
272
+ jids = Sidekiq::Client.push_bulk('class' => ApiWorker,
273
+ 'args' => [[1, 'jason'], [2, 'jason']],
274
+ 'at' => Time.now.to_f)
275
+ assert_equal 2, jids.size
276
+ (remain_id, job_id) = jids
277
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
278
+ q = Sidekiq::Queue.new
279
+ job.add_to_queue
280
+ queued_job = q.find_job(job_id)
281
+ refute_nil queued_job
282
+ assert_equal queued_job.jid, job_id
283
+ assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
284
+ refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
285
+ end
289
286
 
290
- it 'can clear a queue' do
291
- q = Sidekiq::Queue.new
292
- 2.times { ApiWorker.perform_async(1, 'mike') }
293
- q.clear
287
+ it 'can find job by id in sorted sets' do
288
+ job_id = ApiWorker.perform_in(100, 1, 'jason')
289
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
290
+ refute_nil job
291
+ assert_equal job_id, job.jid
292
+ assert_in_delta job.latency, 0.0, 0.1
293
+ end
294
294
 
295
- Sidekiq.redis do |conn|
296
- refute conn.smembers('queues').include?('foo')
297
- refute conn.exists('queue:foo')
295
+ it 'can remove jobs when iterating over a sorted set' do
296
+ # scheduled jobs must be greater than SortedSet#each underlying page size
297
+ 51.times do
298
+ ApiWorker.perform_in(100, 'aaron')
299
+ end
300
+ set = Sidekiq::ScheduledSet.new
301
+ set.map(&:delete)
302
+ assert_equal set.size, 0
298
303
  end
299
- end
300
304
 
301
- it 'can fetch by score' do
302
- same_time = Time.now.to_f
303
- add_retry('bob1', same_time)
304
- add_retry('bob2', same_time)
305
- r = Sidekiq::RetrySet.new
306
- assert_equal 2, r.fetch(same_time).size
307
- end
305
+ it 'can remove jobs when iterating over a queue' do
306
+ # initial queue size must be greater than Queue#each underlying page size
307
+ 51.times do
308
+ ApiWorker.perform_async(1, 'aaron')
309
+ end
310
+ q = Sidekiq::Queue.new
311
+ q.map(&:delete)
312
+ assert_equal q.size, 0
313
+ end
308
314
 
309
- it 'can fetch by score and jid' do
310
- same_time = Time.now.to_f
311
- add_retry('bob1', same_time)
312
- add_retry('bob2', same_time)
313
- r = Sidekiq::RetrySet.new
314
- assert_equal 1, r.fetch(same_time, 'bob1').size
315
- end
315
+ it 'can find job by id in queues' do
316
+ q = Sidekiq::Queue.new
317
+ job_id = ApiWorker.perform_async(1, 'jason')
318
+ job = q.find_job(job_id)
319
+ refute_nil job
320
+ assert_equal job_id, job.jid
321
+ end
316
322
 
317
- it 'shows empty retries' do
318
- r = Sidekiq::RetrySet.new
319
- assert_equal 0, r.size
320
- end
323
+ it 'can clear a queue' do
324
+ q = Sidekiq::Queue.new
325
+ 2.times { ApiWorker.perform_async(1, 'mike') }
326
+ q.clear
321
327
 
322
- it 'can enumerate retries' do
323
- add_retry
328
+ Sidekiq.redis do |conn|
329
+ refute conn.smembers('queues').include?('foo')
330
+ refute conn.exists('queue:foo')
331
+ end
332
+ end
324
333
 
325
- r = Sidekiq::RetrySet.new
326
- assert_equal 1, r.size
327
- array = r.to_a
328
- assert_equal 1, array.size
334
+ it 'can fetch by score' do
335
+ same_time = Time.now.to_f
336
+ add_retry('bob1', same_time)
337
+ add_retry('bob2', same_time)
338
+ r = Sidekiq::RetrySet.new
339
+ assert_equal 2, r.fetch(same_time).size
340
+ end
329
341
 
330
- retri = array.first
331
- assert_equal 'ApiWorker', retri.klass
332
- assert_equal 'default', retri.queue
333
- assert_equal 'bob', retri.jid
334
- assert_in_delta Time.now.to_f, retri.at.to_f, 0.02
335
- end
342
+ it 'can fetch by score and jid' do
343
+ same_time = Time.now.to_f
344
+ add_retry('bob1', same_time)
345
+ add_retry('bob2', same_time)
346
+ r = Sidekiq::RetrySet.new
347
+ assert_equal 1, r.fetch(same_time, 'bob1').size
348
+ end
336
349
 
337
- it 'can delete multiple retries from score' do
338
- same_time = Time.now.to_f
339
- add_retry('bob1', same_time)
340
- add_retry('bob2', same_time)
341
- r = Sidekiq::RetrySet.new
342
- assert_equal 2, r.size
343
- Sidekiq::RetrySet.new.delete(same_time)
344
- assert_equal 0, r.size
345
- end
350
+ it 'shows empty retries' do
351
+ r = Sidekiq::RetrySet.new
352
+ assert_equal 0, r.size
353
+ end
346
354
 
347
- it 'can delete a single retry from score and jid' do
348
- same_time = Time.now.to_f
349
- add_retry('bob1', same_time)
350
- add_retry('bob2', same_time)
351
- r = Sidekiq::RetrySet.new
352
- assert_equal 2, r.size
353
- Sidekiq::RetrySet.new.delete(same_time, 'bob1')
354
- assert_equal 1, r.size
355
- end
355
+ it 'can enumerate retries' do
356
+ add_retry
356
357
 
357
- it 'can retry a retry' do
358
- add_retry
359
- r = Sidekiq::RetrySet.new
360
- assert_equal 1, r.size
361
- r.first.retry
362
- assert_equal 0, r.size
363
- assert_equal 1, Sidekiq::Queue.new('default').size
364
- job = Sidekiq::Queue.new('default').first
365
- assert_equal 'bob', job.jid
366
- assert_equal 1, job['retry_count']
367
- end
358
+ r = Sidekiq::RetrySet.new
359
+ assert_equal 1, r.size
360
+ array = r.to_a
361
+ assert_equal 1, array.size
368
362
 
369
- it 'can clear retries' do
370
- add_retry
371
- add_retry('test')
372
- r = Sidekiq::RetrySet.new
373
- assert_equal 2, r.size
374
- r.clear
375
- assert_equal 0, r.size
376
- end
363
+ retri = array.first
364
+ assert_equal 'ApiWorker', retri.klass
365
+ assert_equal 'default', retri.queue
366
+ assert_equal 'bob', retri.jid
367
+ assert_in_delta Time.now.to_f, retri.at.to_f, 0.02
368
+ end
377
369
 
378
- it 'can enumerate processes' do
379
- identity_string = "identity_string"
380
- odata = {
381
- 'pid' => 123,
382
- 'hostname' => hostname,
383
- 'key' => identity_string,
384
- 'identity' => identity_string,
385
- 'started_at' => Time.now.to_f - 15,
386
- }
387
-
388
- time = Time.now.to_f
389
- Sidekiq.redis do |conn|
390
- conn.multi do
391
- conn.sadd('processes', odata['key'])
392
- conn.hmset(odata['key'], 'info', Sidekiq.dump_json(odata), 'busy', 10, 'beat', time)
393
- conn.sadd('processes', 'fake:pid')
370
+ it 'requires a jid to delete an entry' do
371
+ start_time = Time.now.to_f
372
+ add_retry('bob2', Time.now.to_f)
373
+ assert_raises(ArgumentError) do
374
+ Sidekiq::RetrySet.new.delete(start_time)
394
375
  end
395
376
  end
396
377
 
397
- ps = Sidekiq::ProcessSet.new.to_a
398
- assert_equal 1, ps.size
399
- data = ps.first
400
- assert_equal 10, data['busy']
401
- assert_equal time, data['beat']
402
- assert_equal 123, data['pid']
403
- data.quiet!
404
- data.stop!
405
- signals_string = "#{odata['key']}-signals"
406
- assert_equal "TERM", Sidekiq.redis{|c| c.lpop(signals_string) }
407
- assert_equal "USR1", Sidekiq.redis{|c| c.lpop(signals_string) }
408
- end
409
-
410
- it 'can enumerate workers' do
411
- w = Sidekiq::Workers.new
412
- assert_equal 0, w.size
413
- w.each do
414
- assert false
378
+ it 'can delete a single retry from score and jid' do
379
+ same_time = Time.now.to_f
380
+ add_retry('bob1', same_time)
381
+ add_retry('bob2', same_time)
382
+ r = Sidekiq::RetrySet.new
383
+ assert_equal 2, r.size
384
+ Sidekiq::RetrySet.new.delete(same_time, 'bob1')
385
+ assert_equal 1, r.size
415
386
  end
416
387
 
417
- key = "#{hostname}:#{$$}"
418
- pdata = { 'pid' => $$, 'hostname' => hostname, 'started_at' => Time.now.to_i }
419
- Sidekiq.redis do |conn|
420
- conn.sadd('processes', key)
421
- conn.hmset(key, 'info', Sidekiq.dump_json(pdata), 'busy', 0, 'beat', Time.now.to_f)
388
+ it 'can retry a retry' do
389
+ add_retry
390
+ r = Sidekiq::RetrySet.new
391
+ assert_equal 1, r.size
392
+ r.first.retry
393
+ assert_equal 0, r.size
394
+ assert_equal 1, Sidekiq::Queue.new('default').size
395
+ job = Sidekiq::Queue.new('default').first
396
+ assert_equal 'bob', job.jid
397
+ assert_equal 1, job['retry_count']
422
398
  end
423
399
 
424
- s = "#{key}:workers"
425
- data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => Time.now.to_i })
426
- Sidekiq.redis do |c|
427
- c.hmset(s, '1234', data)
400
+ it 'can clear retries' do
401
+ add_retry
402
+ add_retry('test')
403
+ r = Sidekiq::RetrySet.new
404
+ assert_equal 2, r.size
405
+ r.clear
406
+ assert_equal 0, r.size
428
407
  end
429
408
 
430
- w.each do |p, x, y|
431
- assert_equal key, p
432
- assert_equal "1234", x
433
- assert_equal 'default', y['queue']
434
- assert_equal Time.now.year, Time.at(y['run_at']).year
409
+ it 'can enumerate processes' do
410
+ identity_string = "identity_string"
411
+ odata = {
412
+ 'pid' => 123,
413
+ 'hostname' => Socket.gethostname,
414
+ 'key' => identity_string,
415
+ 'identity' => identity_string,
416
+ 'started_at' => Time.now.to_f - 15,
417
+ }
418
+
419
+ time = Time.now.to_f
420
+ Sidekiq.redis do |conn|
421
+ conn.multi do
422
+ conn.sadd('processes', odata['key'])
423
+ conn.hmset(odata['key'], 'info', Sidekiq.dump_json(odata), 'busy', 10, 'beat', time)
424
+ conn.sadd('processes', 'fake:pid')
425
+ end
426
+ end
427
+
428
+ ps = Sidekiq::ProcessSet.new.to_a
429
+ assert_equal 1, ps.size
430
+ data = ps.first
431
+ assert_equal 10, data['busy']
432
+ assert_equal time, data['beat']
433
+ assert_equal 123, data['pid']
434
+ data.quiet!
435
+ data.stop!
436
+ signals_string = "#{odata['key']}-signals"
437
+ assert_equal "TERM", Sidekiq.redis{|c| c.lpop(signals_string) }
438
+ assert_equal "USR1", Sidekiq.redis{|c| c.lpop(signals_string) }
435
439
  end
436
440
 
437
- s = "#{key}:workers"
438
- data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => (Time.now.to_i - 2*60*60) })
439
- Sidekiq.redis do |c|
440
- c.multi do
441
- c.hmset(s, '5678', data)
442
- c.hmset("b#{s}", '5678', data)
441
+ it 'can enumerate workers' do
442
+ w = Sidekiq::Workers.new
443
+ assert_equal 0, w.size
444
+ w.each do
445
+ assert false
443
446
  end
444
- end
445
447
 
446
- assert_equal ['1234', '5678'], w.map { |_, tid, _| tid }
447
- end
448
+ hn = Socket.gethostname
449
+ key = "#{hn}:#{$$}"
450
+ pdata = { 'pid' => $$, 'hostname' => hn, 'started_at' => Time.now.to_i }
451
+ Sidekiq.redis do |conn|
452
+ conn.sadd('processes', key)
453
+ conn.hmset(key, 'info', Sidekiq.dump_json(pdata), 'busy', 0, 'beat', Time.now.to_f)
454
+ end
448
455
 
449
- it 'can reschedule jobs' do
450
- add_retry('foo1')
451
- add_retry('foo2')
456
+ s = "#{key}:workers"
457
+ data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => Time.now.to_i })
458
+ Sidekiq.redis do |c|
459
+ c.hmset(s, '1234', data)
460
+ end
452
461
 
453
- retries = Sidekiq::RetrySet.new
454
- assert_equal 2, retries.size
455
- refute(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
462
+ w.each do |p, x, y|
463
+ assert_equal key, p
464
+ assert_equal "1234", x
465
+ assert_equal 'default', y['queue']
466
+ assert_equal Time.now.year, Time.at(y['run_at']).year
467
+ end
456
468
 
457
- retries.each do |retri|
458
- retri.reschedule(Time.now.to_f + 10) if retri.jid == 'foo2'
469
+ s = "#{key}:workers"
470
+ data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => (Time.now.to_i - 2*60*60) })
471
+ Sidekiq.redis do |c|
472
+ c.multi do
473
+ c.hmset(s, '5678', data)
474
+ c.hmset("b#{s}", '5678', data)
475
+ end
476
+ end
477
+
478
+ assert_equal ['1234', '5678'], w.map { |_, tid, _| tid }
459
479
  end
460
480
 
461
- assert_equal 2, retries.size
462
- assert(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
463
- end
481
+ it 'can reschedule jobs' do
482
+ add_retry('foo1')
483
+ add_retry('foo2')
464
484
 
465
- it 'prunes processes which have died' do
466
- data = { 'pid' => rand(10_000), 'hostname' => "app#{rand(1_000)}", 'started_at' => Time.now.to_f }
467
- key = "#{data['hostname']}:#{data['pid']}"
468
- Sidekiq.redis do |conn|
469
- conn.sadd('processes', key)
470
- conn.hmset(key, 'info', Sidekiq.dump_json(data), 'busy', 0, 'beat', Time.now.to_f)
471
- end
485
+ retries = Sidekiq::RetrySet.new
486
+ assert_equal 2, retries.size
487
+ refute(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
472
488
 
473
- ps = Sidekiq::ProcessSet.new
474
- assert_equal 1, ps.size
475
- assert_equal 1, ps.to_a.size
489
+ retries.each do |retri|
490
+ retri.reschedule(Time.now.to_f + 10) if retri.jid == 'foo2'
491
+ end
476
492
 
477
- Sidekiq.redis do |conn|
478
- conn.sadd('processes', "bar:987")
479
- conn.sadd('processes', "bar:986")
493
+ assert_equal 2, retries.size
494
+ assert(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
480
495
  end
481
496
 
482
- ps = Sidekiq::ProcessSet.new
483
- assert_equal 1, ps.size
484
- assert_equal 1, ps.to_a.size
485
- end
497
+ it 'prunes processes which have died' do
498
+ data = { 'pid' => rand(10_000), 'hostname' => "app#{rand(1_000)}", 'started_at' => Time.now.to_f }
499
+ key = "#{data['hostname']}:#{data['pid']}"
500
+ Sidekiq.redis do |conn|
501
+ conn.sadd('processes', key)
502
+ conn.hmset(key, 'info', Sidekiq.dump_json(data), 'busy', 0, 'beat', Time.now.to_f)
503
+ end
486
504
 
487
- def add_retry(jid = 'bob', at = Time.now.to_f)
488
- payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.to_f)
489
- Sidekiq.redis do |conn|
490
- conn.zadd('retry', at.to_s, payload)
505
+ ps = Sidekiq::ProcessSet.new
506
+ assert_equal 1, ps.size
507
+ assert_equal 1, ps.to_a.size
508
+
509
+ Sidekiq.redis do |conn|
510
+ conn.sadd('processes', "bar:987")
511
+ conn.sadd('processes', "bar:986")
512
+ end
513
+
514
+ ps = Sidekiq::ProcessSet.new
515
+ assert_equal 1, ps.size
516
+ assert_equal 1, ps.to_a.size
517
+ end
518
+
519
+ def add_retry(jid = 'bob', at = Time.now.to_f)
520
+ payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.to_f)
521
+ Sidekiq.redis do |conn|
522
+ conn.zadd('retry', at.to_s, payload)
523
+ end
491
524
  end
492
525
  end
493
526
  end