delayed_job_on_steroids 1.7.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,54 @@
1
+ module Delayed
2
+ class Worker
3
+ SLEEP = 5
4
+
5
+ cattr_accessor :logger
6
+ self.logger = if defined?(Merb::Logger)
7
+ Merb.logger
8
+ elsif defined?(RAILS_DEFAULT_LOGGER)
9
+ RAILS_DEFAULT_LOGGER
10
+ end
11
+
12
+ def initialize(options={})
13
+ @quiet = options[:quiet]
14
+ Delayed::Job.min_priority = options[:min_priority] if options.has_key?(:min_priority)
15
+ Delayed::Job.max_priority = options[:max_priority] if options.has_key?(:max_priority)
16
+ end
17
+
18
+ def start
19
+ say "*** Starting job worker #{Delayed::Job.worker_name}"
20
+
21
+ trap('TERM') { say 'Exiting...'; $exit = true }
22
+ trap('INT') { say 'Exiting...'; $exit = true }
23
+
24
+ loop do
25
+ result = nil
26
+
27
+ realtime = Benchmark.realtime do
28
+ result = Delayed::Job.work_off
29
+ end
30
+
31
+ count = result.sum
32
+
33
+ break if $exit
34
+
35
+ if count.zero?
36
+ sleep(SLEEP)
37
+ else
38
+ say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
39
+ end
40
+
41
+ break if $exit
42
+ end
43
+
44
+ ensure
45
+ Delayed::Job.clear_locks!
46
+ end
47
+
48
+ def say(text)
49
+ puts text unless @quiet
50
+ logger.info text if logger
51
+ end
52
+
53
+ end
54
+ end
@@ -0,0 +1,13 @@
1
+ autoload :ActiveRecord, 'activerecord'
2
+
3
+ require File.dirname(__FILE__) + '/delayed/message_sending'
4
+ require File.dirname(__FILE__) + '/delayed/performable_method'
5
+ require File.dirname(__FILE__) + '/delayed/job'
6
+ require File.dirname(__FILE__) + '/delayed/worker'
7
+
8
+ Object.send(:include, Delayed::MessageSending)
9
+ Module.send(:include, Delayed::MessageSending::ClassMethods)
10
+
11
+ if defined?(Merb::Plugins)
12
+ Merb::Plugins.add_rakefiles File.dirname(__FILE__) / '..' / 'tasks' / 'tasks'
13
+ end
data/spec/database.rb ADDED
@@ -0,0 +1,43 @@
1
+ $:.unshift(File.dirname(__FILE__) + '/../lib')
2
+ $:.unshift(File.dirname(__FILE__) + '/../../rspec/lib')
3
+
4
+ require 'rubygems'
5
+ require 'active_record'
6
+ gem 'sqlite3-ruby'
7
+
8
+ require File.dirname(__FILE__) + '/../init'
9
+ require 'spec'
10
+
11
+ ActiveRecord::Base.logger = Logger.new('/tmp/dj.log')
12
+ ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => '/tmp/jobs.sqlite')
13
+ ActiveRecord::Migration.verbose = false
14
+ ActiveRecord::Base.default_timezone = :utc if Time.zone.nil?
15
+
16
+ ActiveRecord::Schema.define do
17
+
18
+ create_table :delayed_jobs, :force => true do |table|
19
+ table.integer :priority, :default => 0
20
+ table.integer :attempts, :default => 0
21
+ table.text :handler
22
+ table.string :last_error
23
+ table.datetime :run_at
24
+ table.datetime :locked_at
25
+ table.string :locked_by
26
+ table.datetime :failed_at
27
+ table.timestamps
28
+ end
29
+
30
+ create_table :stories, :force => true do |table|
31
+ table.string :text
32
+ end
33
+
34
+ end
35
+
36
+
37
+ # Purely useful for test cases...
38
+ class Story < ActiveRecord::Base
39
+ def tell; text; end
40
+ def whatever(n, _); tell*n; end
41
+
42
+ handle_asynchronously :whatever
43
+ end
@@ -0,0 +1,128 @@
1
+ require File.dirname(__FILE__) + '/database'
2
+
3
+ class SimpleJob
4
+ cattr_accessor :runs; self.runs = 0
5
+ def perform; @@runs += 1; end
6
+ end
7
+
8
+ class RandomRubyObject
9
+ def say_hello
10
+ 'hello'
11
+ end
12
+ end
13
+
14
+ class ErrorObject
15
+
16
+ def throw
17
+ raise ActiveRecord::RecordNotFound, '...'
18
+ false
19
+ end
20
+
21
+ end
22
+
23
+ class StoryReader
24
+
25
+ def read(story)
26
+ "Epilog: #{story.tell}"
27
+ end
28
+
29
+ end
30
+
31
+ class StoryReader
32
+
33
+ def read(story)
34
+ "Epilog: #{story.tell}"
35
+ end
36
+
37
+ end
38
+
39
+ describe 'random ruby objects' do
40
+ before { Delayed::Job.delete_all }
41
+
42
+ it "should respond_to :send_later method" do
43
+
44
+ RandomRubyObject.new.respond_to?(:send_later)
45
+
46
+ end
47
+
48
+ it "should raise a ArgumentError if send_later is called but the target method doesn't exist" do
49
+ lambda { RandomRubyObject.new.send_later(:method_that_deos_not_exist) }.should raise_error(NoMethodError)
50
+ end
51
+
52
+ it "should add a new entry to the job table when send_later is called on it" do
53
+ Delayed::Job.count.should == 0
54
+
55
+ RandomRubyObject.new.send_later(:to_s)
56
+
57
+ Delayed::Job.count.should == 1
58
+ end
59
+
60
+ it "should add a new entry to the job table when send_later is called on the class" do
61
+ Delayed::Job.count.should == 0
62
+
63
+ RandomRubyObject.send_later(:to_s)
64
+
65
+ Delayed::Job.count.should == 1
66
+ end
67
+
68
+ it "should run get the original method executed when the job is performed" do
69
+
70
+ RandomRubyObject.new.send_later(:say_hello)
71
+
72
+ Delayed::Job.count.should == 1
73
+ end
74
+
75
+ it "should ignore ActiveRecord::RecordNotFound errors because they are permanent" do
76
+
77
+ ErrorObject.new.send_later(:throw)
78
+
79
+ Delayed::Job.count.should == 1
80
+
81
+ Delayed::Job.reserve_and_run_one_job
82
+
83
+ Delayed::Job.count.should == 0
84
+
85
+ end
86
+
87
+ it "should store the object as string if its an active record" do
88
+ story = Story.create :text => 'Once upon...'
89
+ story.send_later(:tell)
90
+
91
+ job = Delayed::Job.find(:first)
92
+ job.payload_object.class.should == Delayed::PerformableMethod
93
+ job.payload_object.object.should == "AR:Story:#{story.id}"
94
+ job.payload_object.method.should == :tell
95
+ job.payload_object.args.should == []
96
+ job.payload_object.perform.should == 'Once upon...'
97
+ end
98
+
99
+ it "should store arguments as string if they an active record" do
100
+
101
+ story = Story.create :text => 'Once upon...'
102
+
103
+ reader = StoryReader.new
104
+ reader.send_later(:read, story)
105
+
106
+ job = Delayed::Job.find(:first)
107
+ job.payload_object.class.should == Delayed::PerformableMethod
108
+ job.payload_object.method.should == :read
109
+ job.payload_object.args.should == ["AR:Story:#{story.id}"]
110
+ job.payload_object.perform.should == 'Epilog: Once upon...'
111
+ end
112
+
113
+ it "should call send later on methods which are wrapped with handle_asynchronously" do
114
+ story = Story.create :text => 'Once upon...'
115
+
116
+ Delayed::Job.count.should == 0
117
+
118
+ story.whatever(1, 5)
119
+
120
+ Delayed::Job.count.should == 1
121
+ job = Delayed::Job.find(:first)
122
+ job.payload_object.class.should == Delayed::PerformableMethod
123
+ job.payload_object.method.should == :whatever_without_send_later
124
+ job.payload_object.args.should == [1, 5]
125
+ job.payload_object.perform.should == 'Once upon...'
126
+ end
127
+
128
+ end
data/spec/job_spec.rb ADDED
@@ -0,0 +1,345 @@
1
+ require File.dirname(__FILE__) + '/database'
2
+
3
+ class SimpleJob
4
+ cattr_accessor :runs; self.runs = 0
5
+ def perform; @@runs += 1; end
6
+ end
7
+
8
+ class ErrorJob
9
+ cattr_accessor :runs; self.runs = 0
10
+ def perform; raise 'did not work'; end
11
+ end
12
+
13
+ module M
14
+ class ModuleJob
15
+ cattr_accessor :runs; self.runs = 0
16
+ def perform; @@runs += 1; end
17
+ end
18
+
19
+ end
20
+
21
+ describe Delayed::Job do
22
+ before do
23
+ Delayed::Job.max_priority = nil
24
+ Delayed::Job.min_priority = nil
25
+
26
+ Delayed::Job.delete_all
27
+ end
28
+
29
+ before(:each) do
30
+ SimpleJob.runs = 0
31
+ end
32
+
33
+ it "should set run_at automatically if not set" do
34
+ Delayed::Job.create(:payload_object => ErrorJob.new ).run_at.should_not == nil
35
+ end
36
+
37
+ it "should not set run_at automatically if already set" do
38
+ later = 5.minutes.from_now
39
+ Delayed::Job.create(:payload_object => ErrorJob.new, :run_at => later).run_at.should == later
40
+ end
41
+
42
+ it "should raise ArgumentError when handler doesn't respond_to :perform" do
43
+ lambda { Delayed::Job.enqueue(Object.new) }.should raise_error(ArgumentError)
44
+ end
45
+
46
+ it "should increase count after enqueuing items" do
47
+ Delayed::Job.enqueue SimpleJob.new
48
+ Delayed::Job.count.should == 1
49
+ end
50
+
51
+ it "should be able to set priority when enqueuing items" do
52
+ Delayed::Job.enqueue SimpleJob.new, 5
53
+ Delayed::Job.first.priority.should == 5
54
+ end
55
+
56
+ it "should be able to set run_at when enqueuing items" do
57
+ later = (Delayed::Job.db_time_now+5.minutes)
58
+ Delayed::Job.enqueue SimpleJob.new, 5, later
59
+
60
+ # use be close rather than equal to because millisecond values cn be lost in DB round trip
61
+ Delayed::Job.first.run_at.should be_close(later, 1)
62
+ end
63
+
64
+ it "should call perform on jobs when running work_off" do
65
+ SimpleJob.runs.should == 0
66
+
67
+ Delayed::Job.enqueue SimpleJob.new
68
+ Delayed::Job.work_off
69
+
70
+ SimpleJob.runs.should == 1
71
+ end
72
+
73
+
74
+ it "should work with eval jobs" do
75
+ $eval_job_ran = false
76
+
77
+ Delayed::Job.enqueue do <<-JOB
78
+ $eval_job_ran = true
79
+ JOB
80
+ end
81
+
82
+ Delayed::Job.work_off
83
+
84
+ $eval_job_ran.should == true
85
+ end
86
+
87
+ it "should work with jobs in modules" do
88
+ M::ModuleJob.runs.should == 0
89
+
90
+ Delayed::Job.enqueue M::ModuleJob.new
91
+ Delayed::Job.work_off
92
+
93
+ M::ModuleJob.runs.should == 1
94
+ end
95
+
96
+ it "should re-schedule by about 1 second at first and increment this more and more minutes when it fails to execute properly" do
97
+ Delayed::Job.enqueue ErrorJob.new
98
+ Delayed::Job.work_off(1)
99
+
100
+ job = Delayed::Job.find(:first)
101
+
102
+ job.last_error.should =~ /did not work/
103
+ job.last_error.should =~ /job_spec.rb:10:in `perform'/
104
+ job.attempts.should == 1
105
+
106
+ job.run_at.should > Delayed::Job.db_time_now - 10.minutes
107
+ job.run_at.should < Delayed::Job.db_time_now + 10.minutes
108
+ end
109
+
110
+ it "should raise an DeserializationError when the job class is totally unknown" do
111
+
112
+ job = Delayed::Job.new
113
+ job['handler'] = "--- !ruby/object:JobThatDoesNotExist {}"
114
+
115
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
116
+ end
117
+
118
+ it "should try to load the class when it is unknown at the time of the deserialization" do
119
+ job = Delayed::Job.new
120
+ job['handler'] = "--- !ruby/object:JobThatDoesNotExist {}"
121
+
122
+ job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
123
+
124
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
125
+ end
126
+
127
+ it "should try include the namespace when loading unknown objects" do
128
+ job = Delayed::Job.new
129
+ job['handler'] = "--- !ruby/object:Delayed::JobThatDoesNotExist {}"
130
+ job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
131
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
132
+ end
133
+
134
+ it "should also try to load structs when they are unknown (raises TypeError)" do
135
+ job = Delayed::Job.new
136
+ job['handler'] = "--- !ruby/struct:JobThatDoesNotExist {}"
137
+
138
+ job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
139
+
140
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
141
+ end
142
+
143
+ it "should try include the namespace when loading unknown structs" do
144
+ job = Delayed::Job.new
145
+ job['handler'] = "--- !ruby/struct:Delayed::JobThatDoesNotExist {}"
146
+
147
+ job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
148
+ lambda { job.payload_object.perform }.should raise_error(Delayed::DeserializationError)
149
+ end
150
+
151
+ it "should be failed if it failed more than MAX_ATTEMPTS times and we don't want to destroy jobs" do
152
+ default = Delayed::Job.destroy_failed_jobs
153
+ Delayed::Job.destroy_failed_jobs = false
154
+
155
+ @job = Delayed::Job.create :payload_object => SimpleJob.new, :attempts => 50
156
+ @job.reload.failed_at.should == nil
157
+ @job.reschedule 'FAIL'
158
+ @job.reload.failed_at.should_not == nil
159
+
160
+ Delayed::Job.destroy_failed_jobs = default
161
+ end
162
+
163
+ it "should be destroyed if it failed more than MAX_ATTEMPTS times and we want to destroy jobs" do
164
+ default = Delayed::Job.destroy_failed_jobs
165
+ Delayed::Job.destroy_failed_jobs = true
166
+
167
+ @job = Delayed::Job.create :payload_object => SimpleJob.new, :attempts => 50
168
+ @job.should_receive(:destroy)
169
+ @job.reschedule 'FAIL'
170
+
171
+ Delayed::Job.destroy_failed_jobs = default
172
+ end
173
+
174
+ it "should never find failed jobs" do
175
+ @job = Delayed::Job.create :payload_object => SimpleJob.new, :attempts => 50, :failed_at => Delayed::Job.db_time_now
176
+ Delayed::Job.find_available(1).length.should == 0
177
+ end
178
+
179
+ context "when another worker is already performing an task, it" do
180
+
181
+ before :each do
182
+ Delayed::Job.worker_name = 'worker1'
183
+ @job = Delayed::Job.create :payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => Delayed::Job.db_time_now - 5.minutes
184
+ end
185
+
186
+ it "should not allow a second worker to get exclusive access" do
187
+ @job.lock_exclusively!(4.hours, 'worker2').should == false
188
+ end
189
+
190
+ it "should allow a second worker to get exclusive access if the timeout has passed" do
191
+ @job.lock_exclusively!(1.minute, 'worker2').should == true
192
+ end
193
+
194
+ it "should be able to get access to the task if it was started more then MAX_RUN_TIME ago" do
195
+ @job.locked_at = Delayed::Job.db_time_now - 5.hours
196
+ @job.save
197
+
198
+ @job.lock_exclusively!(4.hours, 'worker2').should == true
199
+ @job.reload
200
+ @job.locked_by.should == 'worker2'
201
+ @job.locked_at.should > 1.minute.ago
202
+ end
203
+
204
+ it "should not be found by another worker" do
205
+ Delayed::Job.worker_name = 'worker2'
206
+
207
+ Delayed::Job.find_available(1, 6.minutes).length.should == 0
208
+ end
209
+
210
+ it "should be found by another worker if the time has expired" do
211
+ Delayed::Job.worker_name = 'worker2'
212
+
213
+ Delayed::Job.find_available(1, 4.minutes).length.should == 1
214
+ end
215
+
216
+ it "should be able to get exclusive access again when the worker name is the same" do
217
+ @job.lock_exclusively! 5.minutes, 'worker1'
218
+ @job.lock_exclusively! 5.minutes, 'worker1'
219
+ @job.lock_exclusively! 5.minutes, 'worker1'
220
+ end
221
+ end
222
+
223
+ context "#name" do
224
+ it "should be the class name of the job that was enqueued" do
225
+ Delayed::Job.create(:payload_object => ErrorJob.new ).name.should == 'ErrorJob'
226
+ end
227
+
228
+ it "should be the method that will be called if its a performable method object" do
229
+ Delayed::Job.send_later(:clear_locks!)
230
+ Delayed::Job.last.name.should == 'Delayed::Job.clear_locks!'
231
+
232
+ end
233
+ it "should be the instance method that will be called if its a performable method object" do
234
+ story = Story.create :text => "..."
235
+
236
+ story.send_later(:save)
237
+
238
+ Delayed::Job.last.name.should == 'Story#save'
239
+ end
240
+ end
241
+
242
+ context "worker prioritization" do
243
+
244
+ before(:each) do
245
+ Delayed::Job.max_priority = nil
246
+ Delayed::Job.min_priority = nil
247
+ end
248
+
249
+ it "should only work_off jobs that are >= min_priority" do
250
+ Delayed::Job.min_priority = -5
251
+ Delayed::Job.max_priority = 5
252
+ SimpleJob.runs.should == 0
253
+
254
+ Delayed::Job.enqueue SimpleJob.new, -10
255
+ Delayed::Job.enqueue SimpleJob.new, 0
256
+ Delayed::Job.work_off
257
+
258
+ SimpleJob.runs.should == 1
259
+ end
260
+
261
+ it "should only work_off jobs that are <= max_priority" do
262
+ Delayed::Job.min_priority = -5
263
+ Delayed::Job.max_priority = 5
264
+ SimpleJob.runs.should == 0
265
+
266
+ Delayed::Job.enqueue SimpleJob.new, 10
267
+ Delayed::Job.enqueue SimpleJob.new, 0
268
+
269
+ Delayed::Job.work_off
270
+
271
+ SimpleJob.runs.should == 1
272
+ end
273
+
274
+ end
275
+
276
+ context "when pulling jobs off the queue for processing, it" do
277
+ before(:each) do
278
+ @job = Delayed::Job.create(
279
+ :payload_object => SimpleJob.new,
280
+ :locked_by => 'worker1',
281
+ :locked_at => Delayed::Job.db_time_now - 5.minutes)
282
+ end
283
+
284
+ it "should leave the queue in a consistent state and not run the job if locking fails" do
285
+ SimpleJob.runs.should == 0
286
+ @job.stub!(:lock_exclusively!).with(any_args).once.and_return(false)
287
+ Delayed::Job.should_receive(:find_available).once.and_return([@job])
288
+ Delayed::Job.work_off(1)
289
+ SimpleJob.runs.should == 0
290
+ end
291
+
292
+ end
293
+
294
+ context "while running alongside other workers that locked jobs, it" do
295
+ before(:each) do
296
+ Delayed::Job.worker_name = 'worker1'
297
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
298
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker2', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
299
+ Delayed::Job.create(:payload_object => SimpleJob.new)
300
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
301
+ end
302
+
303
+ it "should ingore locked jobs from other workers" do
304
+ Delayed::Job.worker_name = 'worker3'
305
+ SimpleJob.runs.should == 0
306
+ Delayed::Job.work_off
307
+ SimpleJob.runs.should == 1 # runs the one open job
308
+ end
309
+
310
+ it "should find our own jobs regardless of locks" do
311
+ Delayed::Job.worker_name = 'worker1'
312
+ SimpleJob.runs.should == 0
313
+ Delayed::Job.work_off
314
+ SimpleJob.runs.should == 3 # runs open job plus worker1 jobs that were already locked
315
+ end
316
+ end
317
+
318
+ context "while running with locked and expired jobs, it" do
319
+ before(:each) do
320
+ Delayed::Job.worker_name = 'worker1'
321
+ exp_time = Delayed::Job.db_time_now - (1.minutes + Delayed::Job::MAX_RUN_TIME)
322
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => exp_time)
323
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker2', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
324
+ Delayed::Job.create(:payload_object => SimpleJob.new)
325
+ Delayed::Job.create(:payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
326
+ end
327
+
328
+ it "should only find unlocked and expired jobs" do
329
+ Delayed::Job.worker_name = 'worker3'
330
+ SimpleJob.runs.should == 0
331
+ Delayed::Job.work_off
332
+ SimpleJob.runs.should == 2 # runs the one open job and one expired job
333
+ end
334
+
335
+ it "should ignore locks when finding our own jobs" do
336
+ Delayed::Job.worker_name = 'worker1'
337
+ SimpleJob.runs.should == 0
338
+ Delayed::Job.work_off
339
+ SimpleJob.runs.should == 3 # runs open job plus worker1 jobs
340
+ # This is useful in the case of a crash/restart on worker1, but make sure multiple workers on the same host have unique names!
341
+ end
342
+
343
+ end
344
+
345
+ end
@@ -0,0 +1,17 @@
1
+ require File.dirname(__FILE__) + '/database'
2
+
3
+ describe "A story" do
4
+
5
+ before(:all) do
6
+ @story = Story.create :text => "Once upon a time..."
7
+ end
8
+
9
+ it "should be shared" do
10
+ @story.tell.should == 'Once upon a time...'
11
+ end
12
+
13
+ it "should not return its result if it storytelling is delayed" do
14
+ @story.send_later(:tell).should_not == 'Once upon a time...'
15
+ end
16
+
17
+ end
data/tasks/jobs.rake ADDED
@@ -0,0 +1 @@
1
+ require File.join(File.dirname(__FILE__), 'tasks')
data/tasks/tasks.rb ADDED
@@ -0,0 +1,15 @@
1
+ # Re-definitions are appended to existing tasks
2
+ task :environment
3
+ task :merb_env
4
+
5
+ namespace :jobs do
6
+ desc "Clear the delayed_job queue."
7
+ task :clear => [:merb_env, :environment] do
8
+ Delayed::Job.delete_all
9
+ end
10
+
11
+ desc "Start a delayed_job worker."
12
+ task :work => [:merb_env, :environment] do
13
+ Delayed::Worker.new(:min_priority => ENV['MIN_PRIORITY'], :max_priority => ENV['MAX_PRIORITY']).start
14
+ end
15
+ end