canvas-jobs 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/db/migrate/20101216224513_create_delayed_jobs.rb +40 -0
  3. data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +14 -0
  4. data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +13 -0
  5. data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +14 -0
  6. data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +26 -0
  7. data/db/migrate/20110610213249_optimize_delayed_jobs.rb +40 -0
  8. data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +52 -0
  9. data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +31 -0
  10. data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +15 -0
  11. data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +80 -0
  12. data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +15 -0
  13. data/db/migrate/20120608191051_add_jobs_run_at_index.rb +15 -0
  14. data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +13 -0
  15. data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +13 -0
  16. data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +13 -0
  17. data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +13 -0
  18. data/db/migrate/20140512213941_add_source_to_jobs.rb +15 -0
  19. data/lib/canvas-jobs.rb +1 -0
  20. data/lib/delayed/backend/active_record.rb +297 -0
  21. data/lib/delayed/backend/base.rb +317 -0
  22. data/lib/delayed/backend/redis/bulk_update.lua +40 -0
  23. data/lib/delayed/backend/redis/destroy_job.lua +2 -0
  24. data/lib/delayed/backend/redis/enqueue.lua +29 -0
  25. data/lib/delayed/backend/redis/fail_job.lua +5 -0
  26. data/lib/delayed/backend/redis/find_available.lua +3 -0
  27. data/lib/delayed/backend/redis/functions.rb +57 -0
  28. data/lib/delayed/backend/redis/get_and_lock_next_available.lua +17 -0
  29. data/lib/delayed/backend/redis/includes/jobs_common.lua +203 -0
  30. data/lib/delayed/backend/redis/job.rb +481 -0
  31. data/lib/delayed/backend/redis/set_running.lua +5 -0
  32. data/lib/delayed/backend/redis/tickle_strand.lua +2 -0
  33. data/lib/delayed/batch.rb +56 -0
  34. data/lib/delayed/engine.rb +4 -0
  35. data/lib/delayed/job_tracking.rb +31 -0
  36. data/lib/delayed/lifecycle.rb +83 -0
  37. data/lib/delayed/message_sending.rb +130 -0
  38. data/lib/delayed/performable_method.rb +42 -0
  39. data/lib/delayed/periodic.rb +81 -0
  40. data/lib/delayed/pool.rb +335 -0
  41. data/lib/delayed/settings.rb +32 -0
  42. data/lib/delayed/version.rb +3 -0
  43. data/lib/delayed/worker.rb +213 -0
  44. data/lib/delayed/yaml_extensions.rb +63 -0
  45. data/lib/delayed_job.rb +40 -0
  46. data/spec/active_record_job_spec.rb +61 -0
  47. data/spec/gemfiles/32.gemfile +6 -0
  48. data/spec/gemfiles/40.gemfile +6 -0
  49. data/spec/gemfiles/41.gemfile +6 -0
  50. data/spec/gemfiles/42.gemfile +6 -0
  51. data/spec/migrate/20140924140513_add_story_table.rb +7 -0
  52. data/spec/redis_job_spec.rb +77 -0
  53. data/spec/sample_jobs.rb +26 -0
  54. data/spec/shared/delayed_batch.rb +85 -0
  55. data/spec/shared/delayed_method.rb +419 -0
  56. data/spec/shared/performable_method.rb +52 -0
  57. data/spec/shared/shared_backend.rb +836 -0
  58. data/spec/shared/worker.rb +291 -0
  59. data/spec/shared_jobs_specs.rb +13 -0
  60. data/spec/spec_helper.rb +91 -0
  61. metadata +329 -0
@@ -0,0 +1,52 @@
1
+ shared_examples_for 'Delayed::PerformableMethod' do
2
+
3
+ it "should not ignore ActiveRecord::RecordNotFound errors because they are not always permanent" do
4
+ story = Story.create :text => 'Once upon...'
5
+ p = Delayed::PerformableMethod.new(story, :tell, [])
6
+ story.destroy
7
+ lambda { YAML.load(p.to_yaml) }.should raise_error
8
+ end
9
+
10
+ it "should store the object using native YAML even if its an active record" do
11
+ story = Story.create :text => 'Once upon...'
12
+ p = Delayed::PerformableMethod.new(story, :tell, [])
13
+ p.class.should == Delayed::PerformableMethod
14
+ p.object.should == story
15
+ p.method.should == :tell
16
+ p.args.should == []
17
+ p.perform.should == 'Once upon...'
18
+ end
19
+
20
+ it "should allow class methods to be called on ActiveRecord models" do
21
+ Story.create!(:text => 'Once upon a...')
22
+ p = Delayed::PerformableMethod.new(Story, :count, [])
23
+ lambda { expect(p.send(:perform)).to eql 1 }.should_not raise_error
24
+ end
25
+
26
+ it "should allow class methods to be called" do
27
+ p = Delayed::PerformableMethod.new(StoryReader, :reverse, ["ohai"])
28
+ lambda { p.send(:perform).should == "iaho" }.should_not raise_error
29
+ end
30
+
31
+ it "should allow module methods to be called" do
32
+ p = Delayed::PerformableMethod.new(MyReverser, :reverse, ["ohai"])
33
+ lambda { p.send(:perform).should == "iaho" }.should_not raise_error
34
+ end
35
+
36
+ it "should store arguments as native YAML if they are active record objects" do
37
+ story = Story.create :text => 'Once upon...'
38
+ reader = StoryReader.new
39
+ p = Delayed::PerformableMethod.new(reader, :read, [story])
40
+ p.class.should == Delayed::PerformableMethod
41
+ p.method.should == :read
42
+ p.args.should == [story]
43
+ p.perform.should == 'Epilog: Once upon...'
44
+ end
45
+
46
+ it "should deeply de-AR-ize arguments in full name" do
47
+ story = Story.create :text => 'Once upon...'
48
+ reader = StoryReader.new
49
+ p = Delayed::PerformableMethod.new(reader, :read, [['arg1', story, { [:key, 1] => story }]])
50
+ p.full_name.should == "StoryReader#read([\"arg1\", Story.find(#{story.id}), {[:key, 1] => Story.find(#{story.id})}])"
51
+ end
52
+ end
@@ -0,0 +1,836 @@
1
+ shared_examples_for 'a backend' do
2
+ def create_job(opts = {})
3
+ Delayed::Job.enqueue(SimpleJob.new, { :queue => nil }.merge(opts))
4
+ end
5
+
6
+ before do
7
+ SimpleJob.runs = 0
8
+ end
9
+
10
+ it "should set run_at automatically if not set" do
11
+ Delayed::Job.create(:payload_object => ErrorJob.new).run_at.should_not be_nil
12
+ end
13
+
14
+ it "should not set run_at automatically if already set" do
15
+ later = Delayed::Job.db_time_now + 5.minutes
16
+ Delayed::Job.create(:payload_object => ErrorJob.new, :run_at => later).run_at.should be_within(1).of(later)
17
+ end
18
+
19
+ it "should raise ArgumentError when handler doesn't respond_to :perform" do
20
+ lambda { Delayed::Job.enqueue(Object.new) }.should raise_error(ArgumentError)
21
+ end
22
+
23
+ it "should increase count after enqueuing items" do
24
+ Delayed::Job.enqueue SimpleJob.new
25
+ Delayed::Job.jobs_count(:current).should == 1
26
+ end
27
+
28
+ it "should be able to set priority when enqueuing items" do
29
+ @job = Delayed::Job.enqueue SimpleJob.new, :priority => 5
30
+ @job.priority.should == 5
31
+ end
32
+
33
+ it "should use the default priority when enqueuing items" do
34
+ Delayed::Job.default_priority = 0
35
+ @job = Delayed::Job.enqueue SimpleJob.new
36
+ @job.priority.should == 0
37
+ Delayed::Job.default_priority = 10
38
+ @job = Delayed::Job.enqueue SimpleJob.new
39
+ @job.priority.should == 10
40
+ Delayed::Job.default_priority = 0
41
+ end
42
+
43
+ it "should be able to set run_at when enqueuing items" do
44
+ later = Delayed::Job.db_time_now + 5.minutes
45
+ @job = Delayed::Job.enqueue SimpleJob.new, :priority => 5, :run_at => later
46
+ @job.run_at.should be_within(1).of(later)
47
+ end
48
+
49
+ it "should work with jobs in modules" do
50
+ M::ModuleJob.runs = 0
51
+ job = Delayed::Job.enqueue M::ModuleJob.new
52
+ lambda { job.invoke_job }.should change { M::ModuleJob.runs }.from(0).to(1)
53
+ end
54
+
55
+ it "should raise an DeserializationError when the job class is totally unknown" do
56
+ job = Delayed::Job.new :handler => "--- !ruby/object:JobThatDoesNotExist {}"
57
+ lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
58
+ end
59
+
60
+ it "should try to load the class when it is unknown at the time of the deserialization" do
61
+ job = Delayed::Job.new :handler => "--- !ruby/object:JobThatDoesNotExist {}"
62
+ lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
63
+ end
64
+
65
+ it "should try include the namespace when loading unknown objects" do
66
+ job = Delayed::Job.new :handler => "--- !ruby/object:Delayed::JobThatDoesNotExist {}"
67
+ lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
68
+ end
69
+
70
+ it "should also try to load structs when they are unknown (raises TypeError)" do
71
+ job = Delayed::Job.new :handler => "--- !ruby/struct:JobThatDoesNotExist {}"
72
+ lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
73
+ end
74
+
75
+ it "should try include the namespace when loading unknown structs" do
76
+ job = Delayed::Job.new :handler => "--- !ruby/struct:Delayed::JobThatDoesNotExist {}"
77
+ lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
78
+ end
79
+
80
+ describe "find_available" do
81
+ it "should not find failed jobs" do
82
+ @job = create_job :attempts => 50
83
+ @job.fail!
84
+ Delayed::Job.find_available(5).should_not include(@job)
85
+ end
86
+
87
+ it "should not find jobs scheduled for the future" do
88
+ @job = create_job :run_at => (Delayed::Job.db_time_now + 1.minute)
89
+ Delayed::Job.find_available(5).should_not include(@job)
90
+ end
91
+
92
+ it "should not find jobs locked by another worker" do
93
+ @job = create_job
94
+ Delayed::Job.get_and_lock_next_available('other_worker').should == @job
95
+ Delayed::Job.find_available(5).should_not include(@job)
96
+ end
97
+
98
+ it "should find open jobs" do
99
+ @job = create_job
100
+ Delayed::Job.find_available(5).should include(@job)
101
+ end
102
+ end
103
+
104
+ context "when another worker is already performing an task, it" do
105
+
106
+ before :each do
107
+ @job = Delayed::Job.create :payload_object => SimpleJob.new
108
+ Delayed::Job.get_and_lock_next_available('worker1').should == @job
109
+ end
110
+
111
+ it "should not allow a second worker to get exclusive access" do
112
+ Delayed::Job.get_and_lock_next_available('worker2').should be_nil
113
+ end
114
+
115
+ it "should not be found by another worker" do
116
+ Delayed::Job.find_available(1).length.should == 0
117
+ end
118
+ end
119
+
120
+ context "#name" do
121
+ it "should be the class name of the job that was enqueued" do
122
+ Delayed::Job.create(:payload_object => ErrorJob.new ).name.should == 'ErrorJob'
123
+ end
124
+
125
+ it "should be the method that will be called if its a performable method object" do
126
+ @job = Story.send_later_enqueue_args(:create, no_delay: true)
127
+ @job.name.should == "Story.create"
128
+ end
129
+
130
+ it "should be the instance method that will be called if its a performable method object" do
131
+ @job = Story.create(:text => "...").send_later_enqueue_args(:save, no_delay: true)
132
+ @job.name.should == 'Story#save'
133
+ end
134
+ end
135
+
136
+ context "worker prioritization" do
137
+ it "should fetch jobs ordered by priority" do
138
+ 10.times { create_job :priority => rand(10) }
139
+ jobs = Delayed::Job.find_available(10)
140
+ jobs.size.should == 10
141
+ jobs.each_cons(2) do |a, b|
142
+ a.priority.should <= b.priority
143
+ end
144
+ end
145
+
146
+ it "should not find jobs lower than the given priority" do
147
+ job1 = create_job :priority => 5
148
+ found = Delayed::Job.get_and_lock_next_available('test1', Delayed::Settings.queue, 10, 20)
149
+ found.should be_nil
150
+ job2 = create_job :priority => 10
151
+ found = Delayed::Job.get_and_lock_next_available('test1', Delayed::Settings.queue, 10, 20)
152
+ found.should == job2
153
+ job3 = create_job :priority => 15
154
+ found = Delayed::Job.get_and_lock_next_available('test2', Delayed::Settings.queue, 10, 20)
155
+ found.should == job3
156
+ end
157
+
158
+ it "should not find jobs higher than the given priority" do
159
+ job1 = create_job :priority => 25
160
+ found = Delayed::Job.get_and_lock_next_available('test1', Delayed::Settings.queue, 10, 20)
161
+ found.should be_nil
162
+ job2 = create_job :priority => 20
163
+ found = Delayed::Job.get_and_lock_next_available('test1', Delayed::Settings.queue, 10, 20)
164
+ found.should == job2
165
+ job3 = create_job :priority => 15
166
+ found = Delayed::Job.get_and_lock_next_available('test2', Delayed::Settings.queue, 10, 20)
167
+ found.should == job3
168
+ end
169
+ end
170
+
171
+ context "clear_locks!" do
172
+ before do
173
+ @job = create_job(:locked_by => 'worker', :locked_at => Delayed::Job.db_time_now)
174
+ end
175
+
176
+ it "should clear locks for the given worker" do
177
+ Delayed::Job.clear_locks!('worker')
178
+ Delayed::Job.find_available(5).should include(@job)
179
+ end
180
+
181
+ it "should not clear locks for other workers" do
182
+ Delayed::Job.clear_locks!('worker1')
183
+ Delayed::Job.find_available(5).should_not include(@job)
184
+ end
185
+ end
186
+
187
+ context "unlock" do
188
+ before do
189
+ @job = create_job(:locked_by => 'worker', :locked_at => Delayed::Job.db_time_now)
190
+ end
191
+
192
+ it "should clear locks" do
193
+ @job.unlock
194
+ @job.locked_by.should be_nil
195
+ @job.locked_at.should be_nil
196
+ end
197
+ end
198
+
199
+ context "strands" do
200
+ it "should run strand jobs in strict order" do
201
+ job1 = create_job(:strand => 'myjobs')
202
+ job2 = create_job(:strand => 'myjobs')
203
+ Delayed::Job.get_and_lock_next_available('w1').should == job1
204
+ Delayed::Job.get_and_lock_next_available('w2').should == nil
205
+ job1.destroy
206
+ # update time since the failed lock pushed it forward
207
+ job2.run_at = 1.minute.ago
208
+ job2.save!
209
+ Delayed::Job.get_and_lock_next_available('w3').should == job2
210
+ Delayed::Job.get_and_lock_next_available('w4').should == nil
211
+ end
212
+
213
+ it "should fail to lock if an earlier job gets locked" do
214
+ job1 = create_job(:strand => 'myjobs')
215
+ job2 = create_job(:strand => 'myjobs')
216
+ Delayed::Job.find_available(2).should == [job1]
217
+ Delayed::Job.find_available(2).should == [job1]
218
+
219
+ # job1 gets locked by w1
220
+ Delayed::Job.get_and_lock_next_available('w1').should == job1
221
+
222
+ # normally w2 would now be able to lock job2, but strands prevent it
223
+ Delayed::Job.get_and_lock_next_available('w2').should be_nil
224
+
225
+ # now job1 is done
226
+ job1.destroy
227
+ # update time since the failed lock pushed it forward
228
+ job2.run_at = 1.minute.ago
229
+ job2.save!
230
+ Delayed::Job.get_and_lock_next_available('w2').should == job2
231
+ end
232
+
233
+ it "should keep strand jobs in order as they are rescheduled" do
234
+ job1 = create_job(:strand => 'myjobs')
235
+ job2 = create_job(:strand => 'myjobs')
236
+ job3 = create_job(:strand => 'myjobs')
237
+ Delayed::Job.get_and_lock_next_available('w1').should == job1
238
+ Delayed::Job.find_available(1).should == []
239
+ job1.destroy
240
+ Delayed::Job.find_available(1).should == [job2]
241
+ # move job2's time forward
242
+ job2.run_at = 1.second.ago
243
+ job2.save!
244
+ job3.run_at = 5.seconds.ago
245
+ job3.save!
246
+ # we should still get job2, not job3
247
+ Delayed::Job.get_and_lock_next_available('w1').should == job2
248
+ end
249
+
250
+ it "should allow to run the next job if a failed job is present" do
251
+ job1 = create_job(:strand => 'myjobs')
252
+ job2 = create_job(:strand => 'myjobs')
253
+ job1.fail!
254
+ Delayed::Job.get_and_lock_next_available('w1').should == job2
255
+ end
256
+
257
+ it "should not interfere with jobs with no strand" do
258
+ jobs = [create_job(:strand => nil), create_job(:strand => 'myjobs')]
259
+ locked = [Delayed::Job.get_and_lock_next_available('w1'),
260
+ Delayed::Job.get_and_lock_next_available('w2')]
261
+ jobs.should =~ locked
262
+ Delayed::Job.get_and_lock_next_available('w3').should == nil
263
+ end
264
+
265
+ it "should not interfere with jobs in other strands" do
266
+ jobs = [create_job(:strand => 'strand1'), create_job(:strand => 'strand2')]
267
+ locked = [Delayed::Job.get_and_lock_next_available('w1'),
268
+ Delayed::Job.get_and_lock_next_available('w2')]
269
+ jobs.should =~ locked
270
+ Delayed::Job.get_and_lock_next_available('w3').should == nil
271
+ end
272
+
273
+ context 'singleton' do
274
+ it "should create if there's no jobs on the strand" do
275
+ @job = create_job(:singleton => 'myjobs')
276
+ @job.should be_present
277
+ Delayed::Job.get_and_lock_next_available('w1').should == @job
278
+ end
279
+
280
+ it "should create if there's another job on the strand, but it's running" do
281
+ @job = create_job(:singleton => 'myjobs')
282
+ @job.should be_present
283
+ Delayed::Job.get_and_lock_next_available('w1').should == @job
284
+
285
+ @job2 = create_job(:singleton => 'myjobs')
286
+ @job.should be_present
287
+ @job2.should_not == @job
288
+ end
289
+
290
+ it "should not create if there's another non-running job on the strand" do
291
+ @job = create_job(:singleton => 'myjobs')
292
+ @job.should be_present
293
+
294
+ @job2 = create_job(:singleton => 'myjobs')
295
+ @job2.should == @job
296
+ end
297
+
298
+ it "should not create if there's a job running and one waiting on the strand" do
299
+ @job = create_job(:singleton => 'myjobs')
300
+ @job.should be_present
301
+ Delayed::Job.get_and_lock_next_available('w1').should == @job
302
+
303
+ @job2 = create_job(:singleton => 'myjobs')
304
+ @job2.should be_present
305
+ @job2.should_not == @job
306
+
307
+ @job3 = create_job(:singleton => 'myjobs')
308
+ @job3.should == @job2
309
+ end
310
+ end
311
+
312
+ context 'n_strand' do
313
+ it "should default to 1" do
314
+ expect(Delayed::Job).to receive(:rand).never
315
+ job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
316
+ job.strand.should == "njobs"
317
+ end
318
+
319
+ it "should pick a strand randomly out of N" do
320
+ change_setting(Delayed::Settings, :num_strands, ->(strand_name) { expect(strand_name).to eql "njobs"; "3" }) do
321
+ expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
322
+ job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
323
+ job.strand.should == "njobs:2"
324
+ end
325
+ end
326
+
327
+ context "with two parameters" do
328
+ it "should use the first param as the setting to read" do
329
+ job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
330
+ job.strand.should == "njobs/123"
331
+ change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
332
+ case strand_name
333
+ when "njobs"; 3
334
+ else nil
335
+ end
336
+ }) do
337
+ expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
338
+ job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
339
+ job.strand.should == "njobs/123:2"
340
+ end
341
+ end
342
+
343
+ it "should allow overridding the setting based on the second param" do
344
+ change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
345
+ case strand_name
346
+ when "njobs/123"; 5
347
+ else nil
348
+ end
349
+ }) do
350
+ expect(Delayed::Job).to receive(:rand).with(5).and_return(3)
351
+ job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
352
+ job.strand.should == "njobs/123:4"
353
+ job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
354
+ job.strand.should == "njobs/456"
355
+ end
356
+
357
+ change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
358
+ case strand_name
359
+ when "njobs/123"; 5
360
+ when "njobs"; 3
361
+ else nil
362
+ end
363
+ }) do
364
+ expect(Delayed::Job).to receive(:rand).with(5).and_return(2)
365
+ expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
366
+ job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
367
+ job.strand.should == "njobs/123:3"
368
+ job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
369
+ job.strand.should == "njobs/456:2"
370
+ end
371
+ end
372
+ end
373
+ end
374
+ end
375
+
376
+ context "on hold" do
377
+ it "should hold/unhold jobs" do
378
+ job1 = create_job()
379
+ job1.hold!
380
+ Delayed::Job.get_and_lock_next_available('w1').should be_nil
381
+
382
+ job1.unhold!
383
+ Delayed::Job.get_and_lock_next_available('w1').should == job1
384
+ end
385
+ end
386
+
387
+ context "periodic jobs" do
388
+ before(:each) do
389
+ # make the periodic job get scheduled in the past
390
+ @cron_time = 10.minutes.ago
391
+ allow(Delayed::Periodic).to receive(:now).and_return(@cron_time)
392
+ Delayed::Periodic.scheduled = {}
393
+ Delayed::Periodic.cron('my SimpleJob', '*/5 * * * * *') do
394
+ Delayed::Job.enqueue(SimpleJob.new)
395
+ end
396
+ end
397
+
398
+ it "should schedule jobs if they aren't scheduled yet" do
399
+ Delayed::Job.jobs_count(:current).should == 0
400
+ Delayed::Periodic.perform_audit!
401
+ Delayed::Job.jobs_count(:current).should == 1
402
+ job = Delayed::Job.get_and_lock_next_available('test1')
403
+ job.tag.should == 'periodic: my SimpleJob'
404
+ job.payload_object.should == Delayed::Periodic.scheduled['my SimpleJob']
405
+ job.run_at.should >= @cron_time
406
+ job.run_at.should <= @cron_time + 6.minutes
407
+ job.strand.should == job.tag
408
+ end
409
+
410
+ it "should schedule jobs if there are only failed jobs on the queue" do
411
+ Delayed::Job.jobs_count(:current).should == 0
412
+ expect { Delayed::Periodic.perform_audit! }.to change { Delayed::Job.jobs_count(:current) }.by(1)
413
+ Delayed::Job.jobs_count(:current).should == 1
414
+ job = Delayed::Job.get_and_lock_next_available('test1')
415
+ job.fail!
416
+ expect { Delayed::Periodic.perform_audit! }.to change{ Delayed::Job.jobs_count(:current) }.by(1)
417
+ end
418
+
419
+ it "should not schedule jobs that are already scheduled" do
420
+ Delayed::Job.jobs_count(:current).should == 0
421
+ Delayed::Periodic.perform_audit!
422
+ Delayed::Job.jobs_count(:current).should == 1
423
+ job = Delayed::Job.find_available(1).first
424
+ Delayed::Periodic.perform_audit!
425
+ Delayed::Job.jobs_count(:current).should == 1
426
+ # verify that the same job still exists, it wasn't just replaced with a new one
427
+ job.should == Delayed::Job.find_available(1).first
428
+ end
429
+
430
+ it "should schedule the next job run after performing" do
431
+ Delayed::Job.jobs_count(:current).should == 0
432
+ Delayed::Periodic.perform_audit!
433
+ Delayed::Job.jobs_count(:current).should == 1
434
+ job = Delayed::Job.get_and_lock_next_available('test')
435
+ run_job(job)
436
+
437
+ job = Delayed::Job.get_and_lock_next_available('test1')
438
+ job.tag.should == 'SimpleJob#perform'
439
+
440
+ next_scheduled = Delayed::Job.get_and_lock_next_available('test2')
441
+ next_scheduled.tag.should == 'periodic: my SimpleJob'
442
+ next_scheduled.payload_object.should be_is_a(Delayed::Periodic)
443
+ end
444
+
445
+ it "should reject duplicate named jobs" do
446
+ proc { Delayed::Periodic.cron('my SimpleJob', '*/15 * * * * *') {} }.should raise_error(ArgumentError)
447
+ end
448
+
449
+ it "should handle jobs that are no longer scheduled" do
450
+ Delayed::Periodic.perform_audit!
451
+ Delayed::Periodic.scheduled = {}
452
+ job = Delayed::Job.get_and_lock_next_available('test')
453
+ run_job(job)
454
+ # shouldn't error, and the job should now be deleted
455
+ Delayed::Job.jobs_count(:current).should == 0
456
+ end
457
+
458
+ it "should allow overriding schedules using periodic_jobs.yml" do
459
+ change_setting(Delayed::Periodic, :overrides, { 'my ChangedJob' => '*/10 * * * * *' }) do
460
+ Delayed::Periodic.scheduled = {}
461
+ Delayed::Periodic.cron('my ChangedJob', '*/5 * * * * *') do
462
+ Delayed::Job.enqueue(SimpleJob.new)
463
+ end
464
+ Delayed::Periodic.scheduled['my ChangedJob'].cron.original.should == '*/10 * * * * *'
465
+ end
466
+ end
467
+
468
+ it "should fail if the override cron line is invalid" do
469
+ change_setting(Delayed::Periodic, :overrides, { 'my ChangedJob' => '*/10 * * * * * *' }) do # extra asterisk
470
+ Delayed::Periodic.scheduled = {}
471
+ expect { Delayed::Periodic.cron('my ChangedJob', '*/5 * * * * *') do
472
+ Delayed::Job.enqueue(SimpleJob.new)
473
+ end }.to raise_error
474
+ end
475
+
476
+ expect { Delayed::Periodic.add_overrides({ 'my ChangedJob' => '*/10 * * * * * *' }) }.to raise_error
477
+ end
478
+ end
479
+
480
+ module InDelayedJobTest
481
+ def self.check_in_job
482
+ Delayed::Job.in_delayed_job?.should == true
483
+ end
484
+ end
485
+
486
+ it "should set in_delayed_job?" do
487
+ job = InDelayedJobTest.send_later_enqueue_args(:check_in_job, no_delay: true)
488
+ Delayed::Job.in_delayed_job?.should == false
489
+ job.invoke_job
490
+ Delayed::Job.in_delayed_job?.should == false
491
+ end
492
+
493
+ it "should fail on job creation if an unsaved AR object is used" do
494
+ story = Story.new :text => "Once upon..."
495
+ lambda { story.send_later(:text) }.should raise_error
496
+
497
+ reader = StoryReader.new
498
+ lambda { reader.send_later(:read, story) }.should raise_error
499
+
500
+ lambda { [story, 1, story, false].send_later(:first) }.should raise_error
501
+ end
502
+
503
+ # the sort order of current_jobs and list_jobs depends on the back-end
504
+ # implementation, so sort order isn't tested in these specs
505
+ describe "current jobs, queue size, strand_size" do
506
+ before do
507
+ @jobs = []
508
+ 3.times { @jobs << create_job(:priority => 3) }
509
+ @jobs.unshift create_job(:priority => 2)
510
+ @jobs.unshift create_job(:priority => 1)
511
+ @jobs << create_job(:priority => 3, :strand => "test1")
512
+ @future_job = create_job(:run_at => 5.hours.from_now)
513
+ 2.times { @jobs << create_job(:priority => 3) }
514
+ @jobs << create_job(:priority => 3, :strand => "test1")
515
+ @failed_job = create_job.tap { |j| j.fail! }
516
+ @other_queue_job = create_job(:queue => "another")
517
+ end
518
+
519
+ it "should return the queued jobs" do
520
+ Delayed::Job.list_jobs(:current, 100).map(&:id).sort.should == @jobs.map(&:id).sort
521
+ end
522
+
523
+ it "should paginate the returned jobs" do
524
+ @returned = []
525
+ @returned += Delayed::Job.list_jobs(:current, 3, 0)
526
+ @returned += Delayed::Job.list_jobs(:current, 4, 3)
527
+ @returned += Delayed::Job.list_jobs(:current, 100, 7)
528
+ @returned.sort_by { |j| j.id }.should == @jobs.sort_by { |j| j.id }
529
+ end
530
+
531
+ it "should return other queues" do
532
+ Delayed::Job.list_jobs(:current, 5, 0, "another").should == [@other_queue_job]
533
+ end
534
+
535
+ it "should return queue size" do
536
+ Delayed::Job.jobs_count(:current).should == @jobs.size
537
+ Delayed::Job.jobs_count(:current, "another").should == 1
538
+ Delayed::Job.jobs_count(:current, "bogus").should == 0
539
+ end
540
+
541
+ it "should return strand size" do
542
+ Delayed::Job.strand_size("test1").should == 2
543
+ Delayed::Job.strand_size("bogus").should == 0
544
+ end
545
+ end
546
+
547
+ it "should return the jobs in a strand" do
548
+ strand_jobs = []
549
+ 3.times { strand_jobs << create_job(:strand => 'test1') }
550
+ 2.times { create_job(:strand => 'test2') }
551
+ strand_jobs << create_job(:strand => 'test1', :run_at => 5.hours.from_now)
552
+ create_job
553
+
554
+ jobs = Delayed::Job.list_jobs(:strand, 3, 0, "test1")
555
+ jobs.size.should == 3
556
+
557
+ jobs += Delayed::Job.list_jobs(:strand, 3, 3, "test1")
558
+ jobs.size.should == 4
559
+
560
+ jobs.sort_by { |j| j.id }.should == strand_jobs.sort_by { |j| j.id }
561
+ end
562
+
563
+ it "should return the jobs for a tag" do
564
+ tag_jobs = []
565
+ 3.times { tag_jobs << "test".send_later_enqueue_args(:to_s, :no_delay => true) }
566
+ 2.times { "test".send_later(:to_i) }
567
+ tag_jobs << "test".send_later_enqueue_args(:to_s, :run_at => 5.hours.from_now, :no_delay => true)
568
+ tag_jobs << "test".send_later_enqueue_args(:to_s, :strand => "test1", :no_delay => true)
569
+ "test".send_later_enqueue_args(:to_i, :strand => "test1")
570
+ create_job
571
+
572
+ jobs = Delayed::Job.list_jobs(:tag, 3, 0, "String#to_s")
573
+ jobs.size.should == 3
574
+
575
+ jobs += Delayed::Job.list_jobs(:tag, 3, 3, "String#to_s")
576
+ jobs.size.should == 5
577
+
578
+ jobs.sort_by { |j| j.id }.should == tag_jobs.sort_by { |j| j.id }
579
+ end
580
+
581
+ describe "running_jobs" do
582
+ it "should return the running jobs, ordered by locked_at" do
583
+ Timecop.freeze(10.minutes.ago) { 3.times { create_job } }
584
+ j1 = Timecop.freeze(2.minutes.ago) { Delayed::Job.get_and_lock_next_available('w1') }
585
+ j2 = Timecop.freeze(5.minutes.ago) { Delayed::Job.get_and_lock_next_available('w2') }
586
+ j3 = Timecop.freeze(5.seconds.ago) { Delayed::Job.get_and_lock_next_available('w3') }
587
+ [j1, j2, j3].compact.size.should == 3
588
+
589
+ Delayed::Job.running_jobs.should == [j2, j1, j3]
590
+ end
591
+ end
592
+
593
+ describe "future jobs" do
594
+ it "should find future jobs once their run_at rolls by" do
595
+ Timecop.freeze {
596
+ @job = create_job :run_at => 5.minutes.from_now
597
+ expect(Delayed::Job.find_available(5)).not_to include(@job)
598
+ }
599
+ Timecop.freeze(1.hour.from_now) {
600
+ expect(Delayed::Job.find_available(5)).to include(@job)
601
+ Delayed::Job.get_and_lock_next_available('test').should == @job
602
+ }
603
+ end
604
+
605
+ it "should return future jobs sorted by their run_at" do
606
+ @j1 = create_job
607
+ @j2 = create_job :run_at => 1.hour.from_now
608
+ @j3 = create_job :run_at => 30.minutes.from_now
609
+ Delayed::Job.list_jobs(:future, 1).should == [@j3]
610
+ Delayed::Job.list_jobs(:future, 5).should == [@j3, @j2]
611
+ Delayed::Job.list_jobs(:future, 1, 1).should == [@j2]
612
+ end
613
+ end
614
+
615
+ describe "failed jobs" do
616
+ # the sort order of failed_jobs depends on the back-end implementation,
617
+ # so sort order isn't tested here
618
+ it "should return the list of failed jobs" do
619
+ jobs = []
620
+ 3.times { jobs << create_job(:priority => 3) }
621
+ jobs = jobs.sort_by { |j| j.id }
622
+ Delayed::Job.list_jobs(:failed, 1).should == []
623
+ jobs[0].fail!
624
+ jobs[1].fail!
625
+ failed = (Delayed::Job.list_jobs(:failed, 1, 0) + Delayed::Job.list_jobs(:failed, 1, 1)).sort_by { |j| j.id }
626
+ failed.size.should == 2
627
+ failed[0].original_job_id.should == jobs[0].id
628
+ failed[1].original_job_id.should == jobs[1].id
629
+ end
630
+ end
631
+
632
+ describe "bulk_update" do
633
+ shared_examples_for "scope" do
634
+ before do
635
+ @affected_jobs = []
636
+ @ignored_jobs = []
637
+ end
638
+
639
+ it "should hold a scope of jobs" do
640
+ @affected_jobs.all? { |j| j.on_hold? }.should be false
641
+ @ignored_jobs.any? { |j| j.on_hold? }.should be false
642
+ Delayed::Job.bulk_update('hold', :flavor => @flavor, :query => @query).should == @affected_jobs.size
643
+
644
+ @affected_jobs.all? { |j| Delayed::Job.find(j.id).on_hold? }.should be true
645
+ @ignored_jobs.any? { |j| Delayed::Job.find(j.id).on_hold? }.should be false
646
+ end
647
+
648
+ it "should un-hold a scope of jobs" do
649
+ Delayed::Job.bulk_update('unhold', :flavor => @flavor, :query => @query).should == @affected_jobs.size
650
+
651
+ @affected_jobs.any? { |j| Delayed::Job.find(j.id).on_hold? }.should be false
652
+ @ignored_jobs.any? { |j| Delayed::Job.find(j.id).on_hold? }.should be false
653
+ end
654
+
655
+ it "should delete a scope of jobs" do
656
+ Delayed::Job.bulk_update('destroy', :flavor => @flavor, :query => @query).should == @affected_jobs.size
657
+ @affected_jobs.map { |j| Delayed::Job.find(j.id) rescue nil }.compact.should be_blank
658
+ @ignored_jobs.map { |j| Delayed::Job.find(j.id) rescue nil }.compact.size.should == @ignored_jobs.size
659
+ end
660
+ end
661
+
662
+ describe "scope: current" do
663
+ include_examples "scope"
664
+ before do
665
+ @flavor = 'current'
666
+ Timecop.freeze(5.minutes.ago) do
667
+ 3.times { @affected_jobs << create_job }
668
+ @ignored_jobs << create_job(:run_at => 2.hours.from_now)
669
+ @ignored_jobs << create_job(:queue => 'q2')
670
+ end
671
+ end
672
+ end
673
+
674
+ describe "scope: future" do
675
+ include_examples "scope"
676
+ before do
677
+ @flavor = 'future'
678
+ Timecop.freeze(5.minutes.ago) do
679
+ 3.times { @affected_jobs << create_job(:run_at => 2.hours.from_now) }
680
+ @ignored_jobs << create_job
681
+ @ignored_jobs << create_job(:queue => 'q2', :run_at => 2.hours.from_now)
682
+ end
683
+ end
684
+ end
685
+
686
+ describe "scope: strand" do
687
+ include_examples "scope"
688
+ before do
689
+ @flavor = 'strand'
690
+ @query = 's1'
691
+ Timecop.freeze(5.minutes.ago) do
692
+ @affected_jobs << create_job(:strand => 's1')
693
+ @affected_jobs << create_job(:strand => 's1', :run_at => 2.hours.from_now)
694
+ @ignored_jobs << create_job
695
+ @ignored_jobs << create_job(:strand => 's2')
696
+ @ignored_jobs << create_job(:strand => 's2', :run_at => 2.hours.from_now)
697
+ end
698
+ end
699
+ end
700
+
701
+ describe "scope: tag" do
702
+ include_examples "scope"
703
+ before do
704
+ @flavor = 'tag'
705
+ @query = 'String#to_i'
706
+ Timecop.freeze(5.minutes.ago) do
707
+ @affected_jobs << "test".send_later_enqueue_args(:to_i, :no_delay => true)
708
+ @affected_jobs << "test".send_later_enqueue_args(:to_i, :strand => 's1', :no_delay => true)
709
+ @affected_jobs << "test".send_later_enqueue_args(:to_i, :run_at => 2.hours.from_now, :no_delay => true)
710
+ @ignored_jobs << create_job
711
+ @ignored_jobs << create_job(:run_at => 1.hour.from_now)
712
+ end
713
+ end
714
+ end
715
+
716
+ it "should hold and un-hold given job ids" do
717
+ j1 = "test".send_later_enqueue_args(:to_i, :no_delay => true)
718
+ j2 = create_job(:run_at => 2.hours.from_now)
719
+ j3 = "test".send_later_enqueue_args(:to_i, :strand => 's1', :no_delay => true)
720
+ Delayed::Job.bulk_update('hold', :ids => [j1.id, j2.id]).should == 2
721
+ Delayed::Job.find(j1.id).on_hold?.should be true
722
+ Delayed::Job.find(j2.id).on_hold?.should be true
723
+ Delayed::Job.find(j3.id).on_hold?.should be false
724
+
725
+ Delayed::Job.bulk_update('unhold', :ids => [j2.id]).should == 1
726
+ Delayed::Job.find(j1.id).on_hold?.should be true
727
+ Delayed::Job.find(j2.id).on_hold?.should be false
728
+ Delayed::Job.find(j3.id).on_hold?.should be false
729
+ end
730
+
731
+ it "should delete given job ids" do
732
+ jobs = (0..2).map { create_job }
733
+ Delayed::Job.bulk_update('destroy', :ids => jobs[0,2].map(&:id)).should == 2
734
+ jobs.map { |j| Delayed::Job.find(j.id) rescue nil }.compact.should == jobs[2,1]
735
+ end
736
+ end
737
+
738
+ describe "tag_counts" do
739
+ before do
740
+ @cur = []
741
+ 3.times { @cur << "test".send_later_enqueue_args(:to_s, no_delay: true) }
742
+ 5.times { @cur << "test".send_later_enqueue_args(:to_i, no_delay: true) }
743
+ 2.times { @cur << "test".send_later_enqueue_args(:upcase, no_delay: true) }
744
+ ("test".send_later_enqueue_args :downcase, no_delay: true).fail!
745
+ @future = []
746
+ 5.times { @future << "test".send_later_enqueue_args(:downcase, run_at: 3.hours.from_now, no_delay: true) }
747
+ @cur << "test".send_later_enqueue_args(:downcase, no_delay: true)
748
+ end
749
+
750
+ it "should return a sorted list of popular current tags" do
751
+ Delayed::Job.tag_counts(:current, 1).should == [{ :tag => "String#to_i", :count => 5 }]
752
+ Delayed::Job.tag_counts(:current, 1, 1).should == [{ :tag => "String#to_s", :count => 3 }]
753
+ Delayed::Job.tag_counts(:current, 5).should == [{ :tag => "String#to_i", :count => 5 },
754
+ { :tag => "String#to_s", :count => 3 },
755
+ { :tag => "String#upcase", :count => 2 },
756
+ { :tag => "String#downcase", :count => 1 }]
757
+ @cur[0,4].each { |j| j.destroy }
758
+ @future[0].run_at = @future[1].run_at = 1.hour.ago
759
+ @future[0].save!
760
+ @future[1].save!
761
+
762
+ Delayed::Job.tag_counts(:current, 5).should == [{ :tag => "String#to_i", :count => 4 },
763
+ { :tag => "String#downcase", :count => 3 },
764
+ { :tag => "String#upcase", :count => 2 },]
765
+ end
766
+
767
+ it "should return a sorted list of all popular tags" do
768
+ Delayed::Job.tag_counts(:all, 1).should == [{ :tag => "String#downcase", :count => 6 }]
769
+ Delayed::Job.tag_counts(:all, 1, 1).should == [{ :tag => "String#to_i", :count => 5 }]
770
+ Delayed::Job.tag_counts(:all, 5).should == [{ :tag => "String#downcase", :count => 6 },
771
+ { :tag => "String#to_i", :count => 5 },
772
+ { :tag => "String#to_s", :count => 3 },
773
+ { :tag => "String#upcase", :count => 2 },]
774
+
775
+ @cur[0,4].each { |j| j.destroy }
776
+ @future[0].destroy
777
+ @future[1].fail!
778
+ @future[2].fail!
779
+
780
+ Delayed::Job.tag_counts(:all, 5).should == [{ :tag => "String#to_i", :count => 4 },
781
+ { :tag => "String#downcase", :count => 3 },
782
+ { :tag => "String#upcase", :count => 2 },]
783
+ end
784
+ end
785
+
786
+ it "should unlock orphaned jobs" do
787
+ change_setting(Delayed::Settings, :max_attempts, 2) do
788
+ job1 = Delayed::Job.new(:tag => 'tag')
789
+ job2 = Delayed::Job.new(:tag => 'tag')
790
+ job3 = Delayed::Job.new(:tag => 'tag')
791
+ job4 = Delayed::Job.new(:tag => 'tag')
792
+ job1.create_and_lock!("Jobworker:#{Process.pid}")
793
+ `echo ''`
794
+ child_pid = $?.pid
795
+ job2.create_and_lock!("Jobworker:#{child_pid}")
796
+ job3.create_and_lock!("someoneelse:#{Process.pid}")
797
+ job4.create_and_lock!("Jobworker:notanumber")
798
+
799
+ Delayed::Job.unlock_orphaned_jobs(nil, "Jobworker").should == 1
800
+
801
+ Delayed::Job.find(job1.id).locked_by.should_not be_nil
802
+ Delayed::Job.find(job2.id).locked_by.should be_nil
803
+ Delayed::Job.find(job3.id).locked_by.should_not be_nil
804
+ Delayed::Job.find(job4.id).locked_by.should_not be_nil
805
+
806
+ Delayed::Job.unlock_orphaned_jobs(nil, "Jobworker").should == 0
807
+ end
808
+ end
809
+
810
+ it "should unlock orphaned jobs given a pid" do
811
+ change_setting(Delayed::Settings, :max_attempts, 2) do
812
+ job1 = Delayed::Job.new(:tag => 'tag')
813
+ job2 = Delayed::Job.new(:tag => 'tag')
814
+ job3 = Delayed::Job.new(:tag => 'tag')
815
+ job4 = Delayed::Job.new(:tag => 'tag')
816
+ job1.create_and_lock!("Jobworker:#{Process.pid}")
817
+ `echo ''`
818
+ child_pid = $?.pid
819
+ `echo ''`
820
+ child_pid2 = $?.pid
821
+ job2.create_and_lock!("Jobworker:#{child_pid}")
822
+ job3.create_and_lock!("someoneelse:#{Process.pid}")
823
+ job4.create_and_lock!("Jobworker:notanumber")
824
+
825
+ Delayed::Job.unlock_orphaned_jobs(child_pid2, "Jobworker").should == 0
826
+ Delayed::Job.unlock_orphaned_jobs(child_pid, "Jobworker").should == 1
827
+
828
+ Delayed::Job.find(job1.id).locked_by.should_not be_nil
829
+ Delayed::Job.find(job2.id).locked_by.should be_nil
830
+ Delayed::Job.find(job3.id).locked_by.should_not be_nil
831
+ Delayed::Job.find(job4.id).locked_by.should_not be_nil
832
+
833
+ Delayed::Job.unlock_orphaned_jobs(child_pid, "Jobworker").should == 0
834
+ end
835
+ end
836
+ end