jkreeftmeijer-delayed_job 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,347 @@
1
+
2
+
3
+ class SimpleJob
4
+ cattr_accessor :runs; self.runs = 0
5
+ def perform; @@runs += 1; end
6
+ end
7
+
8
+ class ErrorJob
9
+ cattr_accessor :runs; self.runs = 0
10
+ def perform; raise 'did not work'; end
11
+ end
12
+
13
+ module M
14
+ class ModuleJob
15
+ cattr_accessor :runs; self.runs = 0
16
+ def perform; @@runs += 1; end
17
+ end
18
+
19
+ end
20
+
21
+ describe Delayed::Job do
22
+ before do
23
+ Delayed::Job.max_priority = nil
24
+ Delayed::Job.min_priority = nil
25
+
26
+ Delayed::Job.delete_all
27
+ end
28
+
29
+ before(:each) do
30
+ SimpleJob.runs = 0
31
+ end
32
+
33
+ it "should set run_at automatically if not set" do
34
+ Delayed::Job.create(:payload_object => ErrorJob.new ).run_at.should_not == nil
35
+ end
36
+
37
+ it "should not set run_at automatically if already set" do
38
+ later = 5.minutes.from_now
39
+ Delayed::Job.create(:payload_object => ErrorJob.new, :run_at => later).run_at.should == later
40
+ end
41
+
42
+ it "should raise ArgumentError when handler doesn't respond_to :perform" do
43
+ lambda { Delayed::Job.enqueue(Object.new) }.should raise_error(ArgumentError)
44
+ end
45
+
46
+ it "should increase count after enqueuing items" do
47
+ Delayed::Job.enqueue SimpleJob.new
48
+ Delayed::Job.count.should == 1
49
+ end
50
+
51
+ it "should be able to set priority when enqueuing items" do
52
+ Delayed::Job.enqueue SimpleJob.new, 5
53
+ Delayed::Job.first.priority.should == 5
54
+ end
55
+
56
+ it "should be able to set run_at when enqueuing items" do
57
+ later = (Delayed::Job.db_time_now+5.minutes)
58
+ Delayed::Job.enqueue SimpleJob.new, 5, later
59
+
60
+ # use be close rather than equal to because millisecond values cn be lost in DB round trip
61
+ Delayed::Job.first.run_at.should be_close(later, 1)
62
+ end
63
+
64
+ it "should call perform on jobs when running work_off" do
65
+ SimpleJob.runs.should == 0
66
+
67
+ Delayed::Job.enqueue SimpleJob.new
68
+ Delayed::Job.work_off
69
+
70
+ SimpleJob.runs.should == 1
71
+ end
72
+
73
+
74
+ it "should work with eval jobs" do
75
+ $eval_job_ran = false
76
+
77
+ Delayed::Job.enqueue do <<-JOB
78
+ $eval_job_ran = true
79
+ JOB
80
+ end
81
+
82
+ Delayed::Job.work_off
83
+
84
+ $eval_job_ran.should == true
85
+ end
86
+
87
+ it "should work with jobs in modules" do
88
+ M::ModuleJob.runs.should == 0
89
+
90
+ Delayed::Job.enqueue M::ModuleJob.new
91
+ Delayed::Job.work_off
92
+
93
+ M::ModuleJob.runs.should == 1
94
+ end
95
+
96
+ it "should re-schedule by about 1 second at first and increment this more and more minutes when it fails to execute properly" do
97
+ Delayed::Job.enqueue ErrorJob.new
98
+ Delayed::Job.work_off(1)
99
+
100
+ job = Delayed::Job.find(:first)
101
+
102
+ job.last_error.should =~ /did not work/
103
+ job.last_error.should =~ /job_spec.rb:10:in `perform'/
104
+ job.attempts.should == 1
105
+
106
+ job.run_at.should > Delayed::Job.db_time_now - 10.minutes
107
+ job.run_at.should < Delayed::Job.db_time_now + 10.minutes
108
+ end
109
+
110
+ it "should raise an DeserializationError when the job class is totally unknown" do
111
+
112
+ job = Delayed::Job.new
113
+ job['handler'] = "--- !ruby/object:JobThatDoesNotExist {}"
114
+
115
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
116
+ end
117
+
118
+ it "should try to load the class when it is unknown at the time of the deserialization" do
119
+ job = Delayed::Job.new
120
+ job['handler'] = "--- !ruby/object:JobThatDoesNotExist {}"
121
+
122
+ job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
123
+
124
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
125
+ end
126
+
127
+ it "should try include the namespace when loading unknown objects" do
128
+ job = Delayed::Job.new
129
+ job['handler'] = "--- !ruby/object:Delayed::JobThatDoesNotExist {}"
130
+ job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
131
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
132
+ end
133
+
134
+ it "should also try to load structs when they are unknown (raises TypeError)" do
135
+ job = Delayed::Job.new
136
+ job['handler'] = "--- !ruby/struct:JobThatDoesNotExist {}"
137
+
138
+ job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
139
+
140
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
141
+ end
142
+
143
+ it "should try include the namespace when loading unknown structs" do
144
+ job = Delayed::Job.new
145
+ job['handler'] = "--- !ruby/struct:Delayed::JobThatDoesNotExist {}"
146
+
147
+ job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
148
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
149
+ end
150
+
151
+ it "should be failed if it failed more than MAX_ATTEMPTS times and we don't want to destroy jobs" do
152
+ default = Delayed::Job.destroy_failed_jobs
153
+ Delayed::Job.destroy_failed_jobs = false
154
+
155
+ @job = Delayed::Job.create :payload_object => SimpleJob.new, :attempts => 50
156
+ @job = Delayed::Job.find(@job.id)
157
+ @job.failed_at.should == nil
158
+ @job.reschedule 'FAIL'
159
+ @job = Delayed::Job.find(@job.id)
160
+ @job.failed_at.should_not == nil
161
+
162
+ Delayed::Job.destroy_failed_jobs = default
163
+ end
164
+
165
+ it "should be destroyed if it failed more than MAX_ATTEMPTS times and we want to destroy jobs" do
166
+ default = Delayed::Job.destroy_failed_jobs
167
+ Delayed::Job.destroy_failed_jobs = true
168
+
169
+ @job = Delayed::Job.create :payload_object => SimpleJob.new, :attempts => 50
170
+ @job.should_receive(:destroy)
171
+ @job.reschedule 'FAIL'
172
+
173
+ Delayed::Job.destroy_failed_jobs = default
174
+ end
175
+
176
+ it "should never find failed jobs" do
177
+ @job = Delayed::Job.create :payload_object => SimpleJob.new, :attempts => 50, :failed_at => Delayed::Job.db_time_now
178
+ Delayed::Job.find_available(1).length.should == 0
179
+ end
180
+
181
+ context "when another worker is already performing an task, it" do
182
+
183
+ before :each do
184
+ Delayed::Job.worker_name = 'worker1'
185
+ @job = Delayed::Job.create :payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => Delayed::Job.db_time_now - 5.minutes
186
+ end
187
+
188
+ it "should not allow a second worker to get exclusive access" do
189
+ @job.lock_exclusively!(4.hours, 'worker2').should == false
190
+ end
191
+
192
+ it "should allow a second worker to get exclusive access if the timeout has passed" do
193
+ @job.lock_exclusively!(1.minute, 'worker2').should == true
194
+ end
195
+
196
+ it "should be able to get access to the task if it was started more then max_age ago" do
197
+ @job.locked_at = 5.hours.ago
198
+ @job.save
199
+
200
+ @job.lock_exclusively! 4.hours, 'worker2'
201
+ @job.reload
202
+ @job.locked_by.should == 'worker2'
203
+ @job.locked_at.should > 1.minute.ago
204
+ end
205
+
206
+ it "should not be found by another worker" do
207
+ Delayed::Job.worker_name = 'worker2'
208
+
209
+ Delayed::Job.find_available(1, 6.minutes).length.should == 0
210
+ end
211
+
212
+ it "should be found by another worker if the time has expired" do
213
+ Delayed::Job.worker_name = 'worker2'
214
+
215
+ Delayed::Job.find_available(1, 4.minutes).length.should == 1
216
+ end
217
+
218
+ it "should be able to get exclusive access again when the worker name is the same" do
219
+ @job.lock_exclusively! 5.minutes, 'worker1'
220
+ @job.lock_exclusively! 5.minutes, 'worker1'
221
+ @job.lock_exclusively! 5.minutes, 'worker1'
222
+ end
223
+ end
224
+
225
+ context "#name" do
226
+ it "should be the class name of the job that was enqueued" do
227
+ Delayed::Job.create(:payload_object => ErrorJob.new ).name.should == 'ErrorJob'
228
+ end
229
+
230
+ it "should be the method that will be called if its a performable method object" do
231
+ Delayed::Job.send_later(:clear_locks!)
232
+ Delayed::Job.last.name.should == 'Delayed::Job.clear_locks!'
233
+
234
+ end
235
+ it "should be the instance method that will be called if its a performable method object" do
236
+ story = Story.create :text => "..."
237
+
238
+ story.send_later(:save)
239
+
240
+ Delayed::Job.last.name.should == 'Story#save'
241
+ end
242
+ end
243
+
244
+ context "worker prioritization" do
245
+
246
+ before(:each) do
247
+ Delayed::Job.max_priority = nil
248
+ Delayed::Job.min_priority = nil
249
+ end
250
+
251
+ it "should only work_off jobs that are >= min_priority" do
252
+ Delayed::Job.min_priority = -5
253
+ Delayed::Job.max_priority = 5
254
+ SimpleJob.runs.should == 0
255
+
256
+ Delayed::Job.enqueue SimpleJob.new, -10
257
+ Delayed::Job.enqueue SimpleJob.new, 0
258
+ Delayed::Job.work_off
259
+
260
+ SimpleJob.runs.should == 1
261
+ end
262
+
263
+ it "should only work_off jobs that are <= max_priority" do
264
+ Delayed::Job.min_priority = -5
265
+ Delayed::Job.max_priority = 5
266
+ SimpleJob.runs.should == 0
267
+
268
+ Delayed::Job.enqueue SimpleJob.new, 10
269
+ Delayed::Job.enqueue SimpleJob.new, 0
270
+
271
+ Delayed::Job.work_off
272
+
273
+ SimpleJob.runs.should == 1
274
+ end
275
+
276
+ end
277
+
278
+ context "when pulling jobs off the queue for processing, it" do
279
+ before(:each) do
280
+ @job = Delayed::Job.create(
281
+ :payload_object => SimpleJob.new,
282
+ :locked_by => 'worker1',
283
+ :locked_at => Delayed::Job.db_time_now - 5.minutes)
284
+ end
285
+
286
+ it "should leave the queue in a consistent state and not run the job if locking fails" do
287
+ SimpleJob.runs.should == 0
288
+ @job.stub!(:lock_exclusively!).with(any_args).once.and_return(false)
289
+ Delayed::Job.should_receive(:find_available).once.and_return([@job])
290
+ Delayed::Job.work_off(1)
291
+ SimpleJob.runs.should == 0
292
+ end
293
+
294
+ end
295
+
296
+ context "while running alongside other workers that locked jobs, it" do
297
+ before(:each) do
298
+ Delayed::Job.worker_name = 'worker1'
299
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
300
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker2', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
301
+ Delayed::Job.create(:payload_object => SimpleJob.new)
302
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
303
+ end
304
+
305
+ it "should ingore locked jobs from other workers" do
306
+ Delayed::Job.worker_name = 'worker3'
307
+ SimpleJob.runs.should == 0
308
+ Delayed::Job.work_off
309
+ SimpleJob.runs.should == 1 # runs the one open job
310
+ end
311
+
312
+ it "should find our own jobs regardless of locks" do
313
+ Delayed::Job.worker_name = 'worker1'
314
+ SimpleJob.runs.should == 0
315
+ Delayed::Job.work_off
316
+ SimpleJob.runs.should == 3 # runs open job plus worker1 jobs that were already locked
317
+ end
318
+ end
319
+
320
+ context "while running with locked and expired jobs, it" do
321
+ before(:each) do
322
+ Delayed::Job.worker_name = 'worker1'
323
+ exp_time = Delayed::Job.db_time_now - (1.minutes + Delayed::Job::MAX_RUN_TIME)
324
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => exp_time)
325
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker2', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
326
+ Delayed::Job.create(:payload_object => SimpleJob.new)
327
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
328
+ end
329
+
330
+ it "should only find unlocked and expired jobs" do
331
+ Delayed::Job.worker_name = 'worker3'
332
+ SimpleJob.runs.should == 0
333
+ Delayed::Job.work_off
334
+ SimpleJob.runs.should == 2 # runs the one open job and one expired job
335
+ end
336
+
337
+ it "should ignore locks when finding our own jobs" do
338
+ Delayed::Job.worker_name = 'worker1'
339
+ SimpleJob.runs.should == 0
340
+ Delayed::Job.work_off
341
+ SimpleJob.runs.should == 3 # runs open job plus worker1 jobs
342
+ # This is useful in the case of a crash/restart on worker1, but make sure multiple workers on the same host have unique names!
343
+ end
344
+
345
+ end
346
+
347
+ end
@@ -0,0 +1,42 @@
1
+ $:.unshift(File.dirname(__FILE__) + '/../../lib')
2
+ $:.unshift(File.dirname(__FILE__) + '/../../../rspec/lib')
3
+
4
+ require 'rubygems'
5
+ require 'active_record'
6
+ gem 'sqlite3-ruby'
7
+
8
+ require File.dirname(__FILE__) + '/../../init'
9
+ require 'spec'
10
+
11
+ ActiveRecord::Base.logger = Logger.new('/tmp/dj.log')
12
+ ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => '/tmp/jobs.sqlite')
13
+ ActiveRecord::Migration.verbose = false
14
+ ActiveRecord::Base.default_timezone = :utc if Time.zone.nil?
15
+
16
+ ActiveRecord::Schema.define do
17
+
18
+ create_table :delayed_jobs, :force => true do |table|
19
+ table.integer :priority, :default => 0
20
+ table.integer :attempts, :default => 0
21
+ table.text :handler
22
+ table.string :last_error
23
+ table.datetime :run_at
24
+ table.datetime :locked_at
25
+ table.string :locked_by
26
+ table.datetime :failed_at
27
+ table.timestamps
28
+ end
29
+
30
+ create_table :stories, :force => true do |table|
31
+ table.string :text
32
+ end
33
+
34
+ end
35
+
36
+ class Story < ActiveRecord::Base
37
+
38
+ def tell; text; end
39
+ def whatever(n, _); tell*n; end
40
+
41
+ handle_asynchronously :whatever
42
+ end
@@ -0,0 +1,22 @@
1
+ $:.unshift(File.dirname(__FILE__) + '/../../lib')
2
+ $:.unshift(File.dirname(__FILE__) + '/../../../rspec/lib')
3
+
4
+ require 'rubygems' # not required for Ruby 1.9
5
+ require 'mongo_mapper'
6
+
7
+ require File.dirname(__FILE__) + '/../../init'
8
+
9
+ MongoMapper.connection = Mongo::Connection.new('127.0.0.1', 27017, {
10
+ :logger => Logger.new('/tmp/dj.log')
11
+ })
12
+ MongoMapper.database = 'test'
13
+
14
+ # Purely useful for test cases...
15
+ class Story
16
+ include MongoMapper::Document
17
+
18
+ def tell; text; end
19
+ def whatever(n, _); tell*n; end
20
+
21
+ handle_asynchronously :whatever
22
+ end
@@ -0,0 +1,15 @@
1
+ describe "A story" do
2
+
3
+ before(:all) do
4
+ @story = Story.create :text => "Once upon a time..."
5
+ end
6
+
7
+ it "should be shared" do
8
+ @story.tell.should == 'Once upon a time...'
9
+ end
10
+
11
+ it "should not return its result if it storytelling is delayed" do
12
+ @story.send_later(:tell).should_not == 'Once upon a time...'
13
+ end
14
+
15
+ end
@@ -0,0 +1 @@
1
+ require File.join(File.dirname(__FILE__), 'tasks')
@@ -0,0 +1,15 @@
1
+ # Re-definitions are appended to existing tasks
2
+ task :environment
3
+ task :merb_env
4
+
5
+ namespace :jobs do
6
+ desc "Clear the delayed_job queue."
7
+ task :clear => [:merb_env, :environment] do
8
+ Delayed::Job.delete_all
9
+ end
10
+
11
+ desc "Start a delayed_job worker."
12
+ task :work => [:merb_env, :environment] do
13
+ Delayed::Worker.new(:min_priority => ENV['MIN_PRIORITY'], :max_priority => ENV['MAX_PRIORITY']).start
14
+ end
15
+ end