delayed-job-ajaycb 2.0.10
Sign up to get free protection for your applications and to get access to all the features.
- data/MIT-LICENSE +20 -0
- data/README.textile +250 -0
- data/contrib/delayed_job.monitrc +14 -0
- data/contrib/delayed_job_multiple.monitrc +23 -0
- data/generators/delayed_job/delayed_job_generator.rb +28 -0
- data/generators/delayed_job/templates/migration.rb +22 -0
- data/generators/delayed_job/templates/migration_queue_name.rb +12 -0
- data/generators/delayed_job/templates/script +5 -0
- data/lib/delayed/backend/active_record.rb +97 -0
- data/lib/delayed/backend/base.rb +126 -0
- data/lib/delayed/backend/data_mapper.rb +125 -0
- data/lib/delayed/backend/mongo_mapper.rb +110 -0
- data/lib/delayed/command.rb +116 -0
- data/lib/delayed/deserialization_error.rb +4 -0
- data/lib/delayed/message_sending.rb +53 -0
- data/lib/delayed/performable_method.rb +62 -0
- data/lib/delayed/railtie.rb +10 -0
- data/lib/delayed/recipes.rb +50 -0
- data/lib/delayed/tasks.rb +15 -0
- data/lib/delayed/worker.rb +190 -0
- data/lib/delayed_job.rb +15 -0
- data/rails/init.rb +5 -0
- data/recipes/delayed_job.rb +1 -0
- data/spec/backend/active_record_job_spec.rb +70 -0
- data/spec/backend/data_mapper_job_spec.rb +16 -0
- data/spec/backend/mongo_mapper_job_spec.rb +94 -0
- data/spec/backend/shared_backend_spec.rb +342 -0
- data/spec/delayed_method_spec.rb +46 -0
- data/spec/message_sending_spec.rb +89 -0
- data/spec/performable_method_spec.rb +53 -0
- data/spec/sample_jobs.rb +26 -0
- data/spec/setup/active_record.rb +34 -0
- data/spec/setup/data_mapper.rb +8 -0
- data/spec/setup/mongo_mapper.rb +17 -0
- data/spec/spec_helper.rb +28 -0
- data/spec/story_spec.rb +17 -0
- data/spec/worker_spec.rb +237 -0
- data/tasks/jobs.rake +1 -0
- metadata +329 -0
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'backend/shared_backend_spec'
|
3
|
+
require 'delayed/backend/data_mapper'
|
4
|
+
|
5
|
+
describe Delayed::Backend::DataMapper::Job do
|
6
|
+
before(:all) do
|
7
|
+
@backend = Delayed::Backend::DataMapper::Job
|
8
|
+
end
|
9
|
+
|
10
|
+
before(:each) do
|
11
|
+
# reset database before each example is run
|
12
|
+
DataMapper.auto_migrate!
|
13
|
+
end
|
14
|
+
|
15
|
+
it_should_behave_like 'a backend'
|
16
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'backend/shared_backend_spec'
|
3
|
+
require 'delayed/backend/mongo_mapper'
|
4
|
+
|
5
|
+
describe Delayed::Backend::MongoMapper::Job do
|
6
|
+
before(:all) do
|
7
|
+
@backend = Delayed::Backend::MongoMapper::Job
|
8
|
+
end
|
9
|
+
|
10
|
+
before(:each) do
|
11
|
+
MongoMapper.database.collections.each(&:remove)
|
12
|
+
end
|
13
|
+
|
14
|
+
it_should_behave_like 'a backend'
|
15
|
+
|
16
|
+
describe "indexes" do
|
17
|
+
it "should have combo index on priority and run_at" do
|
18
|
+
@backend.collection.index_information.detect { |index| index[0] == 'priority_1_run_at_1' }.should_not be_nil
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should have index on locked_by" do
|
22
|
+
@backend.collection.index_information.detect { |index| index[0] == 'locked_by_1' }.should_not be_nil
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "delayed method" do
|
27
|
+
class MongoStoryReader
|
28
|
+
def read(story)
|
29
|
+
"Epilog: #{story.tell}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class MongoStory
|
34
|
+
include ::MongoMapper::Document
|
35
|
+
key :text, String
|
36
|
+
|
37
|
+
def tell
|
38
|
+
text
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should ignore not found errors because they are permanent" do
|
43
|
+
story = MongoStory.create :text => 'Once upon a time...'
|
44
|
+
job = story.delay.tell
|
45
|
+
story.destroy
|
46
|
+
lambda { job.invoke_job }.should_not raise_error
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should store the object as string" do
|
50
|
+
story = MongoStory.create :text => 'Once upon a time...'
|
51
|
+
job = story.delay.tell
|
52
|
+
|
53
|
+
job.payload_object.class.should == Delayed::PerformableMethod
|
54
|
+
job.payload_object.object.should == story
|
55
|
+
job.payload_object.method.should == :tell
|
56
|
+
job.payload_object.args.should == []
|
57
|
+
job.payload_object.perform.should == 'Once upon a time...'
|
58
|
+
end
|
59
|
+
|
60
|
+
it "should store arguments as string" do
|
61
|
+
story = MongoStory.create :text => 'Once upon a time...'
|
62
|
+
job = MongoStoryReader.new.delay.read(story)
|
63
|
+
job.payload_object.class.should == Delayed::PerformableMethod
|
64
|
+
job.payload_object.method.should == :read
|
65
|
+
job.payload_object.args.should == [story]
|
66
|
+
job.payload_object.perform.should == 'Epilog: Once upon a time...'
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
describe "before_fork" do
|
71
|
+
after do
|
72
|
+
MongoMapper.connection.connect
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should disconnect" do
|
76
|
+
lambda do
|
77
|
+
Delayed::Backend::MongoMapper::Job.before_fork
|
78
|
+
end.should change { !!MongoMapper.connection.connected? }.from(true).to(false)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
describe "after_fork" do
|
83
|
+
before do
|
84
|
+
MongoMapper.connection.close
|
85
|
+
end
|
86
|
+
|
87
|
+
it "should call reconnect" do
|
88
|
+
lambda do
|
89
|
+
Delayed::Backend::MongoMapper::Job.after_fork
|
90
|
+
end.should change { !!MongoMapper.connection.connected? }.from(false).to(true)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
@@ -0,0 +1,342 @@
|
|
1
|
+
shared_examples_for 'a backend' do
|
2
|
+
def create_job(opts = {})
|
3
|
+
@backend.create(opts.merge(:payload_object => SimpleJob.new))
|
4
|
+
end
|
5
|
+
|
6
|
+
before do
|
7
|
+
Delayed::Worker.max_priority = nil
|
8
|
+
Delayed::Worker.min_priority = nil
|
9
|
+
Delayed::Worker.default_priority = 99
|
10
|
+
SimpleJob.runs = 0
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should set run_at automatically if not set" do
|
14
|
+
@backend.create(:payload_object => ErrorJob.new ).run_at.should_not be_nil
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should not set run_at automatically if already set" do
|
18
|
+
later = @backend.db_time_now + 5.minutes
|
19
|
+
@backend.create(:payload_object => ErrorJob.new, :run_at => later).run_at.should be_close(later, 1)
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should raise ArgumentError when handler doesn't respond_to :perform" do
|
23
|
+
lambda { @backend.enqueue(Object.new) }.should raise_error(ArgumentError)
|
24
|
+
end
|
25
|
+
|
26
|
+
it "should increase count after enqueuing items" do
|
27
|
+
@backend.enqueue SimpleJob.new
|
28
|
+
@backend.count.should == 1
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should be able to set priority when enqueuing items" do
|
32
|
+
@job = @backend.enqueue SimpleJob.new, 5
|
33
|
+
@job.priority.should == 5
|
34
|
+
end
|
35
|
+
|
36
|
+
it "should use default priority when it is not set" do
|
37
|
+
@job = @backend.enqueue SimpleJob.new
|
38
|
+
@job.priority.should == 99
|
39
|
+
end
|
40
|
+
|
41
|
+
it "should be able to set run_at when enqueuing items" do
|
42
|
+
later = @backend.db_time_now + 5.minutes
|
43
|
+
@job = @backend.enqueue SimpleJob.new, 5, later
|
44
|
+
@job.run_at.should be_close(later, 1)
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should work with jobs in modules" do
|
48
|
+
M::ModuleJob.runs = 0
|
49
|
+
job = @backend.enqueue M::ModuleJob.new
|
50
|
+
lambda { job.invoke_job }.should change { M::ModuleJob.runs }.from(0).to(1)
|
51
|
+
end
|
52
|
+
|
53
|
+
it "should raise an DeserializationError when the job class is totally unknown" do
|
54
|
+
job = @backend.new :handler => "--- !ruby/object:JobThatDoesNotExist {}"
|
55
|
+
lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
|
56
|
+
end
|
57
|
+
|
58
|
+
it "should raise an DeserializationError when the job is badly encoded" do
|
59
|
+
job = @backend.new :handler => "--- !ruby/object:SimpleJob {"
|
60
|
+
lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
|
61
|
+
end
|
62
|
+
|
63
|
+
it "should try to load the class when it is unknown at the time of the deserialization" do
|
64
|
+
job = @backend.new :handler => "--- !ruby/object:JobThatDoesNotExist {}"
|
65
|
+
job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
|
66
|
+
lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
|
67
|
+
end
|
68
|
+
|
69
|
+
it "should try include the namespace when loading unknown objects" do
|
70
|
+
job = @backend.new :handler => "--- !ruby/object:Delayed::JobThatDoesNotExist {}"
|
71
|
+
job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
|
72
|
+
lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should also try to load structs when they are unknown (raises TypeError)" do
|
76
|
+
job = @backend.new :handler => "--- !ruby/struct:JobThatDoesNotExist {}"
|
77
|
+
job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
|
78
|
+
lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should try include the namespace when loading unknown structs" do
|
82
|
+
job = @backend.new :handler => "--- !ruby/struct:Delayed::JobThatDoesNotExist {}"
|
83
|
+
job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
|
84
|
+
lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
|
85
|
+
end
|
86
|
+
|
87
|
+
describe "find_available" do
|
88
|
+
it "should not find failed jobs" do
|
89
|
+
@job = create_job :attempts => 50, :failed_at => @backend.db_time_now
|
90
|
+
@backend.find_available('worker', 5, 1.second).should_not include(@job)
|
91
|
+
end
|
92
|
+
|
93
|
+
it "should not find jobs scheduled for the future" do
|
94
|
+
@job = create_job :run_at => (@backend.db_time_now + 1.minute)
|
95
|
+
@backend.find_available('worker', 5, 4.hours).should_not include(@job)
|
96
|
+
end
|
97
|
+
|
98
|
+
it "should not find jobs locked by another worker" do
|
99
|
+
@job = create_job(:locked_by => 'other_worker', :locked_at => @backend.db_time_now - 1.minute)
|
100
|
+
@backend.find_available('worker', 5, 4.hours).should_not include(@job)
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should find open jobs" do
|
104
|
+
@job = create_job
|
105
|
+
@backend.find_available('worker', 5, 4.hours).should include(@job)
|
106
|
+
end
|
107
|
+
|
108
|
+
it "should find expired jobs" do
|
109
|
+
@job = create_job(:locked_by => 'worker', :locked_at => @backend.db_time_now - 2.minutes)
|
110
|
+
@backend.find_available('worker', 5, 1.minute).should include(@job)
|
111
|
+
end
|
112
|
+
|
113
|
+
it "should find own jobs" do
|
114
|
+
@job = create_job(:locked_by => 'worker', :locked_at => (@backend.db_time_now - 1.minutes))
|
115
|
+
@backend.find_available('worker', 5, 4.hours).should include(@job)
|
116
|
+
end
|
117
|
+
|
118
|
+
it "should find only the right amount of jobs" do
|
119
|
+
10.times { create_job }
|
120
|
+
@backend.find_available('worker', 7, 4.hours).should have(7).jobs
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
context "when another worker is already performing an task, it" do
|
125
|
+
|
126
|
+
before :each do
|
127
|
+
@job = @backend.create :payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => @backend.db_time_now - 5.minutes
|
128
|
+
end
|
129
|
+
|
130
|
+
it "should not allow a second worker to get exclusive access" do
|
131
|
+
@job.lock_exclusively!(4.hours, 'worker2').should == false
|
132
|
+
end
|
133
|
+
|
134
|
+
it "should allow a second worker to get exclusive access if the timeout has passed" do
|
135
|
+
@job.lock_exclusively!(1.minute, 'worker2').should == true
|
136
|
+
end
|
137
|
+
|
138
|
+
it "should be able to get access to the task if it was started more then max_age ago" do
|
139
|
+
@job.locked_at = 5.hours.ago
|
140
|
+
@job.save
|
141
|
+
|
142
|
+
@job.lock_exclusively! 4.hours, 'worker2'
|
143
|
+
@job.reload
|
144
|
+
@job.locked_by.should == 'worker2'
|
145
|
+
@job.locked_at.should > 1.minute.ago
|
146
|
+
end
|
147
|
+
|
148
|
+
it "should not be found by another worker" do
|
149
|
+
@backend.find_available('worker2', 1, 6.minutes).length.should == 0
|
150
|
+
end
|
151
|
+
|
152
|
+
it "should be found by another worker if the time has expired" do
|
153
|
+
@backend.find_available('worker2', 1, 4.minutes).length.should == 1
|
154
|
+
end
|
155
|
+
|
156
|
+
it "should be able to get exclusive access again when the worker name is the same" do
|
157
|
+
@job.lock_exclusively!(5.minutes, 'worker1').should be_true
|
158
|
+
@job.lock_exclusively!(5.minutes, 'worker1').should be_true
|
159
|
+
@job.lock_exclusively!(5.minutes, 'worker1').should be_true
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
context "when another worker has worked on a task since the job was found to be available, it" do
|
164
|
+
|
165
|
+
before :each do
|
166
|
+
@job = @backend.create :payload_object => SimpleJob.new
|
167
|
+
@job_copy_for_worker_2 = @backend.find(@job.id)
|
168
|
+
end
|
169
|
+
|
170
|
+
it "should not allow a second worker to get exclusive access if already successfully processed by worker1" do
|
171
|
+
@job.destroy
|
172
|
+
@job_copy_for_worker_2.lock_exclusively!(4.hours, 'worker2').should == false
|
173
|
+
end
|
174
|
+
|
175
|
+
it "should not allow a second worker to get exclusive access if failed to be processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
|
176
|
+
@job.update_attributes(:attempts => 1, :run_at => 1.day.from_now)
|
177
|
+
@job_copy_for_worker_2.lock_exclusively!(4.hours, 'worker2').should == false
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
|
182
|
+
describe "reserve" do
|
183
|
+
before do
|
184
|
+
Delayed::Worker.max_run_time = 2.minutes
|
185
|
+
@worker = Delayed::Worker.new(:quiet => true)
|
186
|
+
end
|
187
|
+
|
188
|
+
it "should not reserve failed jobs" do
|
189
|
+
create_job :attempts => 50, :failed_at => described_class.db_time_now
|
190
|
+
described_class.reserve(@worker).should be_nil
|
191
|
+
end
|
192
|
+
|
193
|
+
it "should not reserve jobs scheduled for the future" do
|
194
|
+
create_job :run_at => (described_class.db_time_now + 1.minute)
|
195
|
+
described_class.reserve(@worker).should be_nil
|
196
|
+
end
|
197
|
+
|
198
|
+
it "should lock the job so other workers can't reserve it" do
|
199
|
+
job = create_job
|
200
|
+
described_class.reserve(@worker).should == job
|
201
|
+
new_worker = Delayed::Worker.new(:quiet => true)
|
202
|
+
new_worker.name = 'worker2'
|
203
|
+
described_class.reserve(new_worker).should be_nil
|
204
|
+
end
|
205
|
+
|
206
|
+
it "should reserve open jobs" do
|
207
|
+
job = create_job
|
208
|
+
described_class.reserve(@worker).should == job
|
209
|
+
end
|
210
|
+
|
211
|
+
it "should reserve expired jobs" do
|
212
|
+
job = create_job(:locked_by => @worker.name, :locked_at => described_class.db_time_now - 3.minutes)
|
213
|
+
described_class.reserve(@worker).should == job
|
214
|
+
end
|
215
|
+
|
216
|
+
it "should reserve own jobs" do
|
217
|
+
job = create_job(:locked_by => @worker.name, :locked_at => (described_class.db_time_now - 1.minutes))
|
218
|
+
described_class.reserve(@worker).should == job
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
context "#name" do
|
223
|
+
it "should be the class name of the job that was enqueued" do
|
224
|
+
@backend.create(:payload_object => ErrorJob.new ).name.should == 'ErrorJob'
|
225
|
+
end
|
226
|
+
|
227
|
+
it "should be the method that will be called if its a performable method object" do
|
228
|
+
@job = Story.delay.create
|
229
|
+
@job.name.should == "Story.create"
|
230
|
+
end
|
231
|
+
|
232
|
+
it "should be the instance method that will be called if its a performable method object" do
|
233
|
+
@job = Story.create(:text => "...").delay.save
|
234
|
+
@job.name.should == 'Story#save'
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
context "worker prioritization" do
|
239
|
+
before(:each) do
|
240
|
+
Delayed::Worker.max_priority = nil
|
241
|
+
Delayed::Worker.min_priority = nil
|
242
|
+
end
|
243
|
+
|
244
|
+
it "should fetch jobs ordered by priority" do
|
245
|
+
10.times { @backend.enqueue SimpleJob.new, rand(10) }
|
246
|
+
jobs = @backend.find_available('worker', 10)
|
247
|
+
jobs.size.should == 10
|
248
|
+
jobs.each_cons(2) do |a, b|
|
249
|
+
a.priority.should <= b.priority
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
it "should only find jobs greater than or equal to min priority" do
|
254
|
+
min = 5
|
255
|
+
Delayed::Worker.min_priority = min
|
256
|
+
10.times {|i| @backend.enqueue SimpleJob.new, i }
|
257
|
+
jobs = @backend.find_available('worker', 10)
|
258
|
+
jobs.each {|job| job.priority.should >= min}
|
259
|
+
end
|
260
|
+
|
261
|
+
it "should only find jobs less than or equal to max priority" do
|
262
|
+
max = 5
|
263
|
+
Delayed::Worker.max_priority = max
|
264
|
+
10.times {|i| @backend.enqueue SimpleJob.new, i }
|
265
|
+
jobs = @backend.find_available('worker', 10)
|
266
|
+
jobs.each {|job| job.priority.should <= max}
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
context "clear_locks!" do
|
271
|
+
before do
|
272
|
+
@job = create_job(:locked_by => 'worker', :locked_at => @backend.db_time_now)
|
273
|
+
end
|
274
|
+
|
275
|
+
it "should clear locks for the given worker" do
|
276
|
+
@backend.clear_locks!('worker')
|
277
|
+
@backend.find_available('worker2', 5, 1.minute).should include(@job)
|
278
|
+
end
|
279
|
+
|
280
|
+
it "should not clear locks for other workers" do
|
281
|
+
@backend.clear_locks!('worker1')
|
282
|
+
@backend.find_available('worker1', 5, 1.minute).should_not include(@job)
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
286
|
+
context "unlock" do
|
287
|
+
before do
|
288
|
+
@job = create_job(:locked_by => 'worker', :locked_at => @backend.db_time_now)
|
289
|
+
end
|
290
|
+
|
291
|
+
it "should clear locks" do
|
292
|
+
@job.unlock
|
293
|
+
@job.locked_by.should be_nil
|
294
|
+
@job.locked_at.should be_nil
|
295
|
+
end
|
296
|
+
end
|
297
|
+
|
298
|
+
context "large handler" do
|
299
|
+
before do
|
300
|
+
text = "Lorem ipsum dolor sit amet. " * 1000
|
301
|
+
@job = @backend.enqueue Delayed::PerformableMethod.new(text, :length, {})
|
302
|
+
end
|
303
|
+
|
304
|
+
it "should have an id" do
|
305
|
+
@job.id.should_not be_nil
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
context "max_attempts" do
|
310
|
+
before(:each) do
|
311
|
+
@job = described_class.enqueue SimpleJob.new
|
312
|
+
end
|
313
|
+
|
314
|
+
it 'should not be defined' do
|
315
|
+
@job.max_attempts.should be_nil
|
316
|
+
end
|
317
|
+
|
318
|
+
it 'should use the max_retries value on the payload when defined' do
|
319
|
+
@job.payload_object.stub!(:max_attempts).and_return(99)
|
320
|
+
@job.max_attempts.should == 99
|
321
|
+
end
|
322
|
+
end
|
323
|
+
|
324
|
+
describe "worker integration" do
|
325
|
+
before do
|
326
|
+
@worker = Delayed::Worker.new(:max_priority => nil, :min_priority => nil, :quiet => true)
|
327
|
+
end
|
328
|
+
|
329
|
+
describe "running a job" do
|
330
|
+
|
331
|
+
context "when the job raises a deserialization error" do
|
332
|
+
it "should mark the job as failed" do
|
333
|
+
Delayed::Worker.destroy_failed_jobs = false
|
334
|
+
job = described_class.create! :handler => "--- !ruby/object:JobThatDoesNotExist {}"
|
335
|
+
@worker.work_off
|
336
|
+
job.reload
|
337
|
+
job.failed_at.should_not be_nil
|
338
|
+
end
|
339
|
+
end
|
340
|
+
end
|
341
|
+
end
|
342
|
+
end
|