delayed_job_csi 2.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/MIT-LICENSE +20 -0
- data/README.textile +246 -0
- data/contrib/delayed_job.monitrc +14 -0
- data/contrib/delayed_job_multiple.monitrc +23 -0
- data/generators/delayed_job/delayed_job_generator.rb +22 -0
- data/generators/delayed_job/templates/migration.rb +21 -0
- data/generators/delayed_job/templates/script +5 -0
- data/lib/delayed/backend/active_record.rb +92 -0
- data/lib/delayed/backend/base.rb +129 -0
- data/lib/delayed/backend/data_mapper.rb +125 -0
- data/lib/delayed/backend/mongo_mapper.rb +110 -0
- data/lib/delayed/command.rb +108 -0
- data/lib/delayed/message_sending.rb +53 -0
- data/lib/delayed/performable_method.rb +62 -0
- data/lib/delayed/railtie.rb +10 -0
- data/lib/delayed/recipes.rb +50 -0
- data/lib/delayed/tasks.rb +15 -0
- data/lib/delayed/worker.rb +176 -0
- data/lib/delayed_job.rb +14 -0
- data/rails/init.rb +5 -0
- data/recipes/delayed_job.rb +1 -0
- data/spec/backend/active_record_job_spec.rb +46 -0
- data/spec/backend/data_mapper_job_spec.rb +16 -0
- data/spec/backend/mongo_mapper_job_spec.rb +94 -0
- data/spec/backend/shared_backend_spec.rb +317 -0
- data/spec/delayed_method_spec.rb +46 -0
- data/spec/message_sending_spec.rb +89 -0
- data/spec/performable_method_spec.rb +53 -0
- data/spec/sample_jobs.rb +26 -0
- data/spec/setup/active_record.rb +33 -0
- data/spec/setup/data_mapper.rb +8 -0
- data/spec/setup/mongo_mapper.rb +17 -0
- data/spec/spec_helper.rb +28 -0
- data/spec/story_spec.rb +17 -0
- data/spec/worker_spec.rb +237 -0
- data/tasks/jobs.rake +1 -0
- metadata +327 -0
@@ -0,0 +1,53 @@
|
|
1
|
+
module Delayed
|
2
|
+
class DelayProxy < ActiveSupport::BasicObject
|
3
|
+
def initialize(target, options)
|
4
|
+
@target = target
|
5
|
+
@options = options
|
6
|
+
end
|
7
|
+
|
8
|
+
def method_missing(method, *args)
|
9
|
+
Job.create({
|
10
|
+
:payload_object => PerformableMethod.new(@target, method.to_sym, args),
|
11
|
+
:priority => ::Delayed::Worker.default_priority
|
12
|
+
}.merge(@options))
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
module MessageSending
|
17
|
+
def delay(options = {})
|
18
|
+
DelayProxy.new(self, options)
|
19
|
+
end
|
20
|
+
alias __delay__ delay
|
21
|
+
|
22
|
+
def send_later(method, *args)
|
23
|
+
warn "[DEPRECATION] `object.send_later(:method)` is deprecated. Use `object.delay.method"
|
24
|
+
__delay__.__send__(method, *args)
|
25
|
+
end
|
26
|
+
|
27
|
+
def send_at(time, method, *args)
|
28
|
+
warn "[DEPRECATION] `object.send_at(time, :method)` is deprecated. Use `object.delay(:run_at => time).method"
|
29
|
+
__delay__(:run_at => time).__send__(method, *args)
|
30
|
+
end
|
31
|
+
|
32
|
+
module ClassMethods
|
33
|
+
def handle_asynchronously(method, opts = {})
|
34
|
+
aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
|
35
|
+
with_method, without_method = "#{aliased_method}_with_delay#{punctuation}", "#{aliased_method}_without_delay#{punctuation}"
|
36
|
+
define_method(with_method) do |*args|
|
37
|
+
curr_opts = opts.clone
|
38
|
+
curr_opts.each_key do |key|
|
39
|
+
if (val = curr_opts[key]).is_a?(Proc)
|
40
|
+
curr_opts[key] = if val.arity == 1
|
41
|
+
val.call(self)
|
42
|
+
else
|
43
|
+
val.call
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
delay(curr_opts).__send__(without_method, *args)
|
48
|
+
end
|
49
|
+
alias_method_chain method, :delay
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
class Class
|
2
|
+
def load_for_delayed_job(arg)
|
3
|
+
self
|
4
|
+
end
|
5
|
+
|
6
|
+
def dump_for_delayed_job
|
7
|
+
name
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
module Delayed
|
12
|
+
class PerformableMethod < Struct.new(:object, :method, :args)
|
13
|
+
STRING_FORMAT = /^LOAD\;([A-Z][\w\:]+)(?:\;(\w+))?$/
|
14
|
+
|
15
|
+
class LoadError < StandardError
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize(object, method, args)
|
19
|
+
raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method, true)
|
20
|
+
|
21
|
+
self.object = dump(object)
|
22
|
+
self.args = args.map { |a| dump(a) }
|
23
|
+
self.method = method.to_sym
|
24
|
+
end
|
25
|
+
|
26
|
+
def display_name
|
27
|
+
if STRING_FORMAT === object
|
28
|
+
"#{$1}#{$2 ? '#' : '.'}#{method}"
|
29
|
+
else
|
30
|
+
"#{object.class}##{method}"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def perform
|
35
|
+
load(object).send(method, *args.map{|a| load(a)})
|
36
|
+
rescue PerformableMethod::LoadError
|
37
|
+
# We cannot do anything about objects that can't be loaded
|
38
|
+
true
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def load(obj)
|
44
|
+
if STRING_FORMAT === obj
|
45
|
+
$1.constantize.load_for_delayed_job($2)
|
46
|
+
else
|
47
|
+
obj
|
48
|
+
end
|
49
|
+
rescue => e
|
50
|
+
Delayed::Worker.logger.warn "Could not load object for job: #{e.message}"
|
51
|
+
raise PerformableMethod::LoadError
|
52
|
+
end
|
53
|
+
|
54
|
+
def dump(obj)
|
55
|
+
if obj.respond_to?(:dump_for_delayed_job)
|
56
|
+
"LOAD;#{obj.dump_for_delayed_job}"
|
57
|
+
else
|
58
|
+
obj
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# Capistrano Recipes for managing delayed_job
|
2
|
+
#
|
3
|
+
# Add these callbacks to have the delayed_job process restart when the server
|
4
|
+
# is restarted:
|
5
|
+
#
|
6
|
+
# after "deploy:stop", "delayed_job:stop"
|
7
|
+
# after "deploy:start", "delayed_job:start"
|
8
|
+
# after "deploy:restart", "delayed_job:restart"
|
9
|
+
#
|
10
|
+
# If you want to use command line options, for example to start multiple workers,
|
11
|
+
# define a Capistrano variable delayed_job_args:
|
12
|
+
#
|
13
|
+
# set :delayed_jobs_args, "-n 2"
|
14
|
+
#
|
15
|
+
# If you've got delayed_job workers running on a servers, you can also specify
|
16
|
+
# which servers have delayed_job running and should be restarted after deploy.
|
17
|
+
#
|
18
|
+
# set :delayed_job_server_role, :worker
|
19
|
+
#
|
20
|
+
|
21
|
+
Capistrano::Configuration.instance.load do
|
22
|
+
namespace :delayed_job do
|
23
|
+
def rails_env
|
24
|
+
fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : ''
|
25
|
+
end
|
26
|
+
|
27
|
+
def args
|
28
|
+
fetch(:delayed_job_args, "")
|
29
|
+
end
|
30
|
+
|
31
|
+
def roles
|
32
|
+
fetch(:delayed_job_server_role, :app)
|
33
|
+
end
|
34
|
+
|
35
|
+
desc "Stop the delayed_job process"
|
36
|
+
task :stop, :roles => lambda { roles } do
|
37
|
+
run "cd #{current_path};#{rails_env} script/delayed_job stop"
|
38
|
+
end
|
39
|
+
|
40
|
+
desc "Start the delayed_job process"
|
41
|
+
task :start, :roles => lambda { roles } do
|
42
|
+
run "cd #{current_path};#{rails_env} script/delayed_job start #{args}"
|
43
|
+
end
|
44
|
+
|
45
|
+
desc "Restart the delayed_job process"
|
46
|
+
task :restart, :roles => lambda { roles } do
|
47
|
+
run "cd #{current_path};#{rails_env} script/delayed_job restart #{args}"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# Re-definitions are appended to existing tasks
|
2
|
+
task :environment
|
3
|
+
task :merb_env
|
4
|
+
|
5
|
+
namespace :jobs do
|
6
|
+
desc "Clear the delayed_job queue."
|
7
|
+
task :clear => [:merb_env, :environment] do
|
8
|
+
Delayed::Job.delete_all
|
9
|
+
end
|
10
|
+
|
11
|
+
desc "Start a delayed_job worker."
|
12
|
+
task :work => [:merb_env, :environment] do
|
13
|
+
Delayed::Worker.new(:min_priority => ENV['MIN_PRIORITY'], :max_priority => ENV['MAX_PRIORITY']).start
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,176 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
require 'active_support/core_ext/numeric/time'
|
3
|
+
|
4
|
+
module Delayed
|
5
|
+
class Worker
|
6
|
+
cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time, :default_priority, :sleep_delay, :logger
|
7
|
+
self.sleep_delay = 5
|
8
|
+
self.max_attempts = 25
|
9
|
+
self.max_run_time = 4.hours
|
10
|
+
self.default_priority = 0
|
11
|
+
|
12
|
+
# By default failed jobs are destroyed after too many attempts. If you want to keep them around
|
13
|
+
# (perhaps to inspect the reason for the failure), set this to false.
|
14
|
+
cattr_accessor :destroy_failed_jobs
|
15
|
+
self.destroy_failed_jobs = true
|
16
|
+
|
17
|
+
self.logger = if defined?(Merb::Logger)
|
18
|
+
Merb.logger
|
19
|
+
elsif defined?(RAILS_DEFAULT_LOGGER)
|
20
|
+
RAILS_DEFAULT_LOGGER
|
21
|
+
end
|
22
|
+
|
23
|
+
# name_prefix is ignored if name is set directly
|
24
|
+
attr_accessor :name_prefix
|
25
|
+
|
26
|
+
cattr_reader :backend
|
27
|
+
|
28
|
+
def self.backend=(backend)
|
29
|
+
if backend.is_a? Symbol
|
30
|
+
require "delayed/backend/#{backend}"
|
31
|
+
backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize
|
32
|
+
end
|
33
|
+
@@backend = backend
|
34
|
+
silence_warnings { ::Delayed.const_set(:Job, backend) }
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.guess_backend
|
38
|
+
self.backend ||= if defined?(ActiveRecord)
|
39
|
+
:active_record
|
40
|
+
elsif defined?(MongoMapper)
|
41
|
+
:mongo_mapper
|
42
|
+
else
|
43
|
+
logger.warn "Could not decide on a backend, defaulting to active_record"
|
44
|
+
:active_record
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def initialize(options={})
|
49
|
+
@quiet = options[:quiet]
|
50
|
+
self.class.min_priority = options[:min_priority] if options.has_key?(:min_priority)
|
51
|
+
self.class.max_priority = options[:max_priority] if options.has_key?(:max_priority)
|
52
|
+
self.class.sleep_delay = options[:sleep_delay] if options.has_key?(:sleep_delay)
|
53
|
+
end
|
54
|
+
|
55
|
+
# Every worker has a unique name which by default is the pid of the process. There are some
|
56
|
+
# advantages to overriding this with something which survives worker retarts: Workers can#
|
57
|
+
# safely resume working on tasks which are locked by themselves. The worker will assume that
|
58
|
+
# it crashed before.
|
59
|
+
def name
|
60
|
+
return @name unless @name.nil?
|
61
|
+
"#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}"
|
62
|
+
end
|
63
|
+
|
64
|
+
# Sets the name of the worker.
|
65
|
+
# Setting the name to nil will reset the default worker name
|
66
|
+
def name=(val)
|
67
|
+
@name = val
|
68
|
+
end
|
69
|
+
|
70
|
+
def start
|
71
|
+
say "Starting job worker"
|
72
|
+
|
73
|
+
trap('TERM') { say 'Exiting...'; $exit = true }
|
74
|
+
trap('INT') { say 'Exiting...'; $exit = true }
|
75
|
+
|
76
|
+
loop do
|
77
|
+
result = nil
|
78
|
+
|
79
|
+
realtime = Benchmark.realtime do
|
80
|
+
result = work_off
|
81
|
+
end
|
82
|
+
|
83
|
+
count = result.sum
|
84
|
+
|
85
|
+
break if $exit
|
86
|
+
|
87
|
+
if count.zero?
|
88
|
+
sleep(self.class.sleep_delay)
|
89
|
+
else
|
90
|
+
say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
|
91
|
+
end
|
92
|
+
|
93
|
+
break if $exit
|
94
|
+
end
|
95
|
+
|
96
|
+
ensure
|
97
|
+
Delayed::Job.clear_locks!(name)
|
98
|
+
end
|
99
|
+
|
100
|
+
# Do num jobs and return stats on success/failure.
|
101
|
+
# Exit early if interrupted.
|
102
|
+
def work_off(num = 100)
|
103
|
+
success, failure = 0, 0
|
104
|
+
|
105
|
+
num.times do
|
106
|
+
case reserve_and_run_one_job
|
107
|
+
when true
|
108
|
+
success += 1
|
109
|
+
when false
|
110
|
+
failure += 1
|
111
|
+
else
|
112
|
+
break # leave if no work could be done
|
113
|
+
end
|
114
|
+
break if $exit # leave if we're exiting
|
115
|
+
end
|
116
|
+
|
117
|
+
return [success, failure]
|
118
|
+
end
|
119
|
+
|
120
|
+
def run(job)
|
121
|
+
runtime = Benchmark.realtime do
|
122
|
+
Timeout.timeout(self.class.max_run_time.to_i) { job.invoke_job }
|
123
|
+
job.destroy
|
124
|
+
end
|
125
|
+
say "#{job.name} completed after %.4f" % runtime
|
126
|
+
return true # did work
|
127
|
+
rescue Exception => e
|
128
|
+
handle_failed_job(job, e)
|
129
|
+
return false # work failed
|
130
|
+
end
|
131
|
+
|
132
|
+
# Reschedule the job in the future (when a job fails).
|
133
|
+
# Uses an exponential scale depending on the number of failed attempts.
|
134
|
+
def reschedule(job, time = nil)
|
135
|
+
if (job.attempts += 1) < max_attempts(job)
|
136
|
+
job.run_at = time || job.reschedule_at
|
137
|
+
job.unlock
|
138
|
+
job.save!
|
139
|
+
else
|
140
|
+
say "PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.", Logger::INFO
|
141
|
+
|
142
|
+
if job.payload_object.respond_to? :on_permanent_failure
|
143
|
+
say "Running on_permanent_failure hook"
|
144
|
+
job.payload_object.on_permanent_failure
|
145
|
+
end
|
146
|
+
|
147
|
+
self.class.destroy_failed_jobs ? job.destroy : job.update_attributes(:failed_at => Delayed::Job.db_time_now)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def say(text, level = Logger::INFO)
|
152
|
+
text = "[Worker(#{name})] #{text}"
|
153
|
+
puts text unless @quiet
|
154
|
+
logger.add level, "#{Time.now.strftime('%FT%T%z')}: #{text}" if logger
|
155
|
+
end
|
156
|
+
|
157
|
+
def max_attempts(job)
|
158
|
+
job.max_attempts || self.class.max_attempts
|
159
|
+
end
|
160
|
+
|
161
|
+
protected
|
162
|
+
|
163
|
+
def handle_failed_job(job, error)
|
164
|
+
job.last_error = error.message + "\n" + error.backtrace.join("\n")
|
165
|
+
say "#{job.name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts", Logger::ERROR
|
166
|
+
reschedule(job)
|
167
|
+
end
|
168
|
+
|
169
|
+
# Run the next job we can get an exclusive lock on.
|
170
|
+
# If no jobs are left we return nil
|
171
|
+
def reserve_and_run_one_job
|
172
|
+
job = Delayed::Job.reserve(self)
|
173
|
+
run(job) if job
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
data/lib/delayed_job.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'active_support'
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/delayed/message_sending'
|
4
|
+
require File.dirname(__FILE__) + '/delayed/performable_method'
|
5
|
+
require File.dirname(__FILE__) + '/delayed/backend/base'
|
6
|
+
require File.dirname(__FILE__) + '/delayed/worker'
|
7
|
+
require File.dirname(__FILE__) + '/delayed/railtie' if defined?(::Rails::Railtie)
|
8
|
+
|
9
|
+
Object.send(:include, Delayed::MessageSending)
|
10
|
+
Module.send(:include, Delayed::MessageSending::ClassMethods)
|
11
|
+
|
12
|
+
if defined?(Merb::Plugins)
|
13
|
+
Merb::Plugins.add_rakefiles File.dirname(__FILE__) / 'delayed' / 'tasks'
|
14
|
+
end
|
data/rails/init.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'delayed', 'recipes'))
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'backend/shared_backend_spec'
|
3
|
+
require 'delayed/backend/active_record'
|
4
|
+
|
5
|
+
describe Delayed::Backend::ActiveRecord::Job do
|
6
|
+
before(:all) do
|
7
|
+
@backend = Delayed::Backend::ActiveRecord::Job
|
8
|
+
end
|
9
|
+
|
10
|
+
before(:each) do
|
11
|
+
Delayed::Backend::ActiveRecord::Job.delete_all
|
12
|
+
SimpleJob.runs = 0
|
13
|
+
end
|
14
|
+
|
15
|
+
after do
|
16
|
+
Time.zone = nil
|
17
|
+
end
|
18
|
+
|
19
|
+
it_should_behave_like 'a backend'
|
20
|
+
|
21
|
+
context "db_time_now" do
|
22
|
+
it "should return time in current time zone if set" do
|
23
|
+
Time.zone = 'Eastern Time (US & Canada)'
|
24
|
+
%w(EST EDT).should include(Delayed::Job.db_time_now.zone)
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should return UTC time if that is the AR default" do
|
28
|
+
Time.zone = nil
|
29
|
+
ActiveRecord::Base.default_timezone = :utc
|
30
|
+
Delayed::Backend::ActiveRecord::Job.db_time_now.zone.should == 'UTC'
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should return local time if that is the AR default" do
|
34
|
+
Time.zone = 'Central Time (US & Canada)'
|
35
|
+
ActiveRecord::Base.default_timezone = :local
|
36
|
+
%w(CST CDT).should include(Delayed::Backend::ActiveRecord::Job.db_time_now.zone)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
describe "after_fork" do
|
41
|
+
it "should call reconnect on the connection" do
|
42
|
+
ActiveRecord::Base.connection.should_receive(:reconnect!)
|
43
|
+
Delayed::Backend::ActiveRecord::Job.after_fork
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|