delayed_job_unique_key 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/MIT-LICENSE +20 -0
- data/README.textile +246 -0
- data/contrib/delayed_job.monitrc +14 -0
- data/contrib/delayed_job_multiple.monitrc +23 -0
- data/lib/delayed/backend/base.rb +152 -0
- data/lib/delayed/backend/shared_spec.rb +566 -0
- data/lib/delayed/command.rb +101 -0
- data/lib/delayed/deserialization_error.rb +4 -0
- data/lib/delayed/lifecycle.rb +84 -0
- data/lib/delayed/message_sending.rb +54 -0
- data/lib/delayed/performable_mailer.rb +21 -0
- data/lib/delayed/performable_method.rb +33 -0
- data/lib/delayed/plugin.rb +15 -0
- data/lib/delayed/plugins/clear_locks.rb +15 -0
- data/lib/delayed/psych_ext.rb +75 -0
- data/lib/delayed/railtie.rb +16 -0
- data/lib/delayed/recipes.rb +50 -0
- data/lib/delayed/serialization/active_record.rb +19 -0
- data/lib/delayed/syck_ext.rb +34 -0
- data/lib/delayed/tasks.rb +11 -0
- data/lib/delayed/worker.rb +222 -0
- data/lib/delayed/yaml_ext.rb +10 -0
- data/lib/delayed_job.rb +22 -0
- data/lib/generators/delayed_job/delayed_job_generator.rb +11 -0
- data/lib/generators/delayed_job/templates/script +5 -0
- data/recipes/delayed_job.rb +1 -0
- data/spec/autoloaded/clazz.rb +7 -0
- data/spec/autoloaded/instance_clazz.rb +6 -0
- data/spec/autoloaded/instance_struct.rb +6 -0
- data/spec/autoloaded/struct.rb +7 -0
- data/spec/delayed/backend/test.rb +113 -0
- data/spec/delayed/serialization/test.rb +0 -0
- data/spec/fixtures/bad_alias.yml +1 -0
- data/spec/lifecycle_spec.rb +107 -0
- data/spec/message_sending_spec.rb +116 -0
- data/spec/performable_mailer_spec.rb +46 -0
- data/spec/performable_method_spec.rb +89 -0
- data/spec/sample_jobs.rb +75 -0
- data/spec/spec_helper.rb +45 -0
- data/spec/test_backend_spec.rb +13 -0
- data/spec/worker_spec.rb +19 -0
- data/spec/yaml_ext_spec.rb +41 -0
- metadata +197 -0
@@ -0,0 +1,84 @@
|
|
1
|
+
module Delayed
|
2
|
+
class InvalidCallback < Exception; end
|
3
|
+
|
4
|
+
class Lifecycle
|
5
|
+
EVENTS = {
|
6
|
+
:enqueue => [:job],
|
7
|
+
:execute => [:worker],
|
8
|
+
:loop => [:worker],
|
9
|
+
:perform => [:worker, :job],
|
10
|
+
:error => [:worker, :job],
|
11
|
+
:failure => [:worker, :job],
|
12
|
+
:invoke_job => [:job]
|
13
|
+
}
|
14
|
+
|
15
|
+
def initialize
|
16
|
+
@callbacks = EVENTS.keys.inject({}) { |hash, e| hash[e] = Callback.new; hash }
|
17
|
+
end
|
18
|
+
|
19
|
+
def before(event, &block)
|
20
|
+
add(:before, event, &block)
|
21
|
+
end
|
22
|
+
|
23
|
+
def after(event, &block)
|
24
|
+
add(:after, event, &block)
|
25
|
+
end
|
26
|
+
|
27
|
+
def around(event, &block)
|
28
|
+
add(:around, event, &block)
|
29
|
+
end
|
30
|
+
|
31
|
+
def run_callbacks(event, *args, &block)
|
32
|
+
missing_callback(event) unless @callbacks.has_key?(event)
|
33
|
+
|
34
|
+
unless EVENTS[event].size == args.size
|
35
|
+
raise ArgumentError, "Callback #{event} expects #{EVENTS[event].size} parameter(s): #{EVENTS[event].join(', ')}"
|
36
|
+
end
|
37
|
+
|
38
|
+
@callbacks[event].execute(*args, &block)
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def add(type, event, &block)
|
44
|
+
missing_callback(event) unless @callbacks.has_key?(event)
|
45
|
+
|
46
|
+
@callbacks[event].add(type, &block)
|
47
|
+
end
|
48
|
+
|
49
|
+
def missing_callback(event)
|
50
|
+
raise InvalidCallback, "Unknown callback event: #{event}"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
class Callback
|
55
|
+
def initialize
|
56
|
+
@before = []
|
57
|
+
@after = []
|
58
|
+
|
59
|
+
# Identity proc. Avoids special cases when there is no existing around chain.
|
60
|
+
@around = lambda { |*args, &block| block.call(*args) }
|
61
|
+
end
|
62
|
+
|
63
|
+
def execute(*args, &block)
|
64
|
+
@before.each { |c| c.call(*args) }
|
65
|
+
result = @around.call(*args, &block)
|
66
|
+
@after.each { |c| c.call(*args) }
|
67
|
+
result
|
68
|
+
end
|
69
|
+
|
70
|
+
def add(type, &callback)
|
71
|
+
case type
|
72
|
+
when :before
|
73
|
+
@before << callback
|
74
|
+
when :after
|
75
|
+
@after << callback
|
76
|
+
when :around
|
77
|
+
chain = @around # use a local variable so that the current chain is closed over in the following lambda
|
78
|
+
@around = lambda { |*a, &block| chain.call(*a) { |*b| callback.call(*b, &block) } }
|
79
|
+
else
|
80
|
+
raise InvalidCallback, "Invalid callback type: #{type}"
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'active_support/basic_object'
|
2
|
+
require 'active_support/core_ext/module/aliasing'
|
3
|
+
|
4
|
+
module Delayed
|
5
|
+
class DelayProxy < ActiveSupport::BasicObject
|
6
|
+
def initialize(payload_class, target, options)
|
7
|
+
@payload_class = payload_class
|
8
|
+
@target = target
|
9
|
+
@options = options
|
10
|
+
end
|
11
|
+
|
12
|
+
def method_missing(method, *args)
|
13
|
+
Job.enqueue({:payload_object => @payload_class.new(@target, method.to_sym, args)}.merge(@options))
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
module MessageSending
|
18
|
+
def delay(options = {})
|
19
|
+
DelayProxy.new(PerformableMethod, self, options)
|
20
|
+
end
|
21
|
+
alias __delay__ delay
|
22
|
+
|
23
|
+
def send_later(method, *args)
|
24
|
+
warn "[DEPRECATION] `object.send_later(:method)` is deprecated. Use `object.delay.method"
|
25
|
+
__delay__.__send__(method, *args)
|
26
|
+
end
|
27
|
+
|
28
|
+
def send_at(time, method, *args)
|
29
|
+
warn "[DEPRECATION] `object.send_at(time, :method)` is deprecated. Use `object.delay(:run_at => time).method"
|
30
|
+
__delay__(:run_at => time).__send__(method, *args)
|
31
|
+
end
|
32
|
+
|
33
|
+
module ClassMethods
|
34
|
+
def handle_asynchronously(method, opts = {})
|
35
|
+
aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
|
36
|
+
with_method, without_method = "#{aliased_method}_with_delay#{punctuation}", "#{aliased_method}_without_delay#{punctuation}"
|
37
|
+
define_method(with_method) do |*args|
|
38
|
+
curr_opts = opts.clone
|
39
|
+
curr_opts.each_key do |key|
|
40
|
+
if (val = curr_opts[key]).is_a?(Proc)
|
41
|
+
curr_opts[key] = if val.arity == 1
|
42
|
+
val.call(self)
|
43
|
+
else
|
44
|
+
val.call
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
delay(curr_opts).__send__(without_method, *args)
|
49
|
+
end
|
50
|
+
alias_method_chain method, :delay
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'mail'
|
2
|
+
|
3
|
+
module Delayed
|
4
|
+
class PerformableMailer < PerformableMethod
|
5
|
+
def perform
|
6
|
+
object.send(method_name, *args).deliver
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
module DelayMail
|
11
|
+
def delay(options = {})
|
12
|
+
DelayProxy.new(PerformableMailer, self, options)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
Mail::Message.class_eval do
|
18
|
+
def delay(*args)
|
19
|
+
raise RuntimeError, "Use MyMailer.delay.mailer_action(args) to delay sending of emails."
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'active_support/core_ext/module/delegation'
|
2
|
+
|
3
|
+
module Delayed
|
4
|
+
class PerformableMethod
|
5
|
+
attr_accessor :object, :method_name, :args
|
6
|
+
|
7
|
+
delegate :method, :to => :object
|
8
|
+
|
9
|
+
def initialize(object, method_name, args)
|
10
|
+
raise NoMethodError, "undefined method `#{method_name}' for #{object.inspect}" unless object.respond_to?(method_name, true)
|
11
|
+
|
12
|
+
self.object = object
|
13
|
+
self.args = args
|
14
|
+
self.method_name = method_name.to_sym
|
15
|
+
end
|
16
|
+
|
17
|
+
def display_name
|
18
|
+
"#{object.class}##{method_name}"
|
19
|
+
end
|
20
|
+
|
21
|
+
def perform
|
22
|
+
object.send(method_name, *args) if object
|
23
|
+
end
|
24
|
+
|
25
|
+
def method_missing(symbol, *args)
|
26
|
+
object.send(symbol, *args)
|
27
|
+
end
|
28
|
+
|
29
|
+
def respond_to?(symbol, include_private=false)
|
30
|
+
super || object.respond_to?(symbol, include_private)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'active_support/core_ext/class/attribute'
|
2
|
+
|
3
|
+
module Delayed
|
4
|
+
class Plugin
|
5
|
+
class_attribute :callback_block
|
6
|
+
|
7
|
+
def self.callbacks(&block)
|
8
|
+
self.callback_block = block
|
9
|
+
end
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
self.class.callback_block.call(Delayed::Worker.lifecycle) if self.class.callback_block
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Delayed
|
2
|
+
module Plugins
|
3
|
+
class ClearLocks < Plugin
|
4
|
+
callbacks do |lifecycle|
|
5
|
+
lifecycle.around(:execute) do |worker, &block|
|
6
|
+
begin
|
7
|
+
block.call(worker)
|
8
|
+
ensure
|
9
|
+
Delayed::Job.clear_locks!(worker.name)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
if defined?(ActiveRecord)
|
2
|
+
class ActiveRecord::Base
|
3
|
+
# serialize to YAML
|
4
|
+
def encode_with(coder)
|
5
|
+
coder["attributes"] = @attributes
|
6
|
+
coder.tag = ['!ruby/ActiveRecord', self.class.name].join(':')
|
7
|
+
end
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
class Delayed::PerformableMethod
|
12
|
+
# serialize to YAML
|
13
|
+
def encode_with(coder)
|
14
|
+
coder.map = {
|
15
|
+
"object" => object,
|
16
|
+
"method_name" => method_name,
|
17
|
+
"args" => args
|
18
|
+
}
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
module Psych
|
23
|
+
module Visitors
|
24
|
+
class YAMLTree
|
25
|
+
def visit_Class(klass)
|
26
|
+
tag = ['!ruby/class', klass.name].join(':')
|
27
|
+
register(klass, @emitter.start_mapping(nil, tag, false, Nodes::Mapping::BLOCK))
|
28
|
+
@emitter.end_mapping
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class ToRuby
|
33
|
+
def visit_Psych_Nodes_Mapping_with_class(object)
|
34
|
+
return revive(Psych.load_tags[object.tag], object) if Psych.load_tags[object.tag]
|
35
|
+
|
36
|
+
case object.tag
|
37
|
+
when /^!ruby\/class:?(.*)?$/
|
38
|
+
resolve_class $1
|
39
|
+
when /^!ruby\/ActiveRecord:(.+)$/
|
40
|
+
klass = resolve_class($1)
|
41
|
+
payload = Hash[*object.children.map { |c| accept c }]
|
42
|
+
id = payload["attributes"][klass.primary_key]
|
43
|
+
begin
|
44
|
+
if ActiveRecord::VERSION::MAJOR == 3
|
45
|
+
klass.unscoped.find(id)
|
46
|
+
else # Rails 2
|
47
|
+
klass.with_exclusive_scope { klass.find(id) }
|
48
|
+
end
|
49
|
+
rescue ActiveRecord::RecordNotFound
|
50
|
+
raise Delayed::DeserializationError
|
51
|
+
end
|
52
|
+
when /^!ruby\/Mongoid:(.+)$/
|
53
|
+
klass = resolve_class($1)
|
54
|
+
payload = Hash[*object.children.map { |c| accept c }]
|
55
|
+
begin
|
56
|
+
klass.find(payload["attributes"]["_id"])
|
57
|
+
rescue Mongoid::Errors::DocumentNotFound
|
58
|
+
raise Delayed::DeserializationError
|
59
|
+
end
|
60
|
+
else
|
61
|
+
visit_Psych_Nodes_Mapping_without_class(object)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
alias_method_chain :visit_Psych_Nodes_Mapping, :class
|
65
|
+
|
66
|
+
def resolve_class_with_constantize(klass_name)
|
67
|
+
klass_name.constantize
|
68
|
+
rescue
|
69
|
+
resolve_class_without_constantize(klass_name)
|
70
|
+
end
|
71
|
+
alias_method_chain :resolve_class, :constantize
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'delayed_job'
|
2
|
+
require 'rails'
|
3
|
+
|
4
|
+
module Delayed
|
5
|
+
class Railtie < Rails::Railtie
|
6
|
+
initializer :after_initialize do
|
7
|
+
ActiveSupport.on_load(:action_mailer) do
|
8
|
+
ActionMailer::Base.send(:extend, Delayed::DelayMail)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
rake_tasks do
|
13
|
+
load 'delayed/tasks.rb'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# Capistrano Recipes for managing delayed_job
|
2
|
+
#
|
3
|
+
# Add these callbacks to have the delayed_job process restart when the server
|
4
|
+
# is restarted:
|
5
|
+
#
|
6
|
+
# after "deploy:stop", "delayed_job:stop"
|
7
|
+
# after "deploy:start", "delayed_job:start"
|
8
|
+
# after "deploy:restart", "delayed_job:restart"
|
9
|
+
#
|
10
|
+
# If you want to use command line options, for example to start multiple workers,
|
11
|
+
# define a Capistrano variable delayed_job_args:
|
12
|
+
#
|
13
|
+
# set :delayed_job_args, "-n 2"
|
14
|
+
#
|
15
|
+
# If you've got delayed_job workers running on a servers, you can also specify
|
16
|
+
# which servers have delayed_job running and should be restarted after deploy.
|
17
|
+
#
|
18
|
+
# set :delayed_job_server_role, :worker
|
19
|
+
#
|
20
|
+
|
21
|
+
Capistrano::Configuration.instance.load do
|
22
|
+
namespace :delayed_job do
|
23
|
+
def rails_env
|
24
|
+
fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : ''
|
25
|
+
end
|
26
|
+
|
27
|
+
def args
|
28
|
+
fetch(:delayed_job_args, "")
|
29
|
+
end
|
30
|
+
|
31
|
+
def roles
|
32
|
+
fetch(:delayed_job_server_role, :app)
|
33
|
+
end
|
34
|
+
|
35
|
+
desc "Stop the delayed_job process"
|
36
|
+
task :stop, :roles => lambda { roles } do
|
37
|
+
run "cd #{current_path};#{rails_env} script/delayed_job stop"
|
38
|
+
end
|
39
|
+
|
40
|
+
desc "Start the delayed_job process"
|
41
|
+
task :start, :roles => lambda { roles } do
|
42
|
+
run "cd #{current_path};#{rails_env} script/delayed_job start #{args}"
|
43
|
+
end
|
44
|
+
|
45
|
+
desc "Restart the delayed_job process"
|
46
|
+
task :restart, :roles => lambda { roles } do
|
47
|
+
run "cd #{current_path};#{rails_env} script/delayed_job restart #{args}"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
if defined?(ActiveRecord)
|
2
|
+
class ActiveRecord::Base
|
3
|
+
yaml_as "tag:ruby.yaml.org,2002:ActiveRecord"
|
4
|
+
|
5
|
+
def self.yaml_new(klass, tag, val)
|
6
|
+
if ActiveRecord::VERSION::MAJOR == 3
|
7
|
+
klass.unscoped.find(val['attributes'][klass.primary_key])
|
8
|
+
else # Rails 2
|
9
|
+
klass.with_exclusive_scope { klass.find(val['attributes'][klass.primary_key]) }
|
10
|
+
end
|
11
|
+
rescue ActiveRecord::RecordNotFound
|
12
|
+
raise Delayed::DeserializationError
|
13
|
+
end
|
14
|
+
|
15
|
+
def to_yaml_properties
|
16
|
+
['@attributes']
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class Module
|
2
|
+
yaml_as "tag:ruby.yaml.org,2002:module"
|
3
|
+
|
4
|
+
def self.yaml_new(klass, tag, val)
|
5
|
+
klass
|
6
|
+
end
|
7
|
+
|
8
|
+
def to_yaml(options = {})
|
9
|
+
YAML.quick_emit(nil, options) do |out|
|
10
|
+
out.scalar(taguri, name, :plain)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def yaml_tag_read_class(name)
|
15
|
+
# Constantize the object so that ActiveSupport can attempt
|
16
|
+
# its auto loading magic. Will raise LoadError if not successful.
|
17
|
+
name.constantize
|
18
|
+
name
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
class Class
|
23
|
+
yaml_as "tag:ruby.yaml.org,2002:class"
|
24
|
+
remove_method :to_yaml if respond_to?(:to_yaml) && method(:to_yaml).owner == Class # use Module's to_yaml
|
25
|
+
end
|
26
|
+
|
27
|
+
class Struct
|
28
|
+
def self.yaml_tag_read_class(name)
|
29
|
+
# Constantize the object so that ActiveSupport can attempt
|
30
|
+
# its auto loading magic. Will raise LoadError if not successful.
|
31
|
+
name.constantize
|
32
|
+
"Struct::#{ name }"
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
namespace :jobs do
|
2
|
+
desc "Clear the delayed_job queue."
|
3
|
+
task :clear => :environment do
|
4
|
+
Delayed::Job.delete_all
|
5
|
+
end
|
6
|
+
|
7
|
+
desc "Start a delayed_job worker."
|
8
|
+
task :work => :environment do
|
9
|
+
Delayed::Worker.new(:min_priority => ENV['MIN_PRIORITY'], :max_priority => ENV['MAX_PRIORITY'], :queues => (ENV['QUEUES'] || ENV['QUEUE'] || '').split(','), :quiet => false).start
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,222 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
require 'active_support/core_ext/numeric/time'
|
3
|
+
require 'active_support/core_ext/class/attribute_accessors'
|
4
|
+
require 'active_support/core_ext/kernel'
|
5
|
+
require 'active_support/core_ext/enumerable'
|
6
|
+
require 'logger'
|
7
|
+
|
8
|
+
module Delayed
|
9
|
+
class Worker
|
10
|
+
cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time, :default_priority, :sleep_delay, :logger, :delay_jobs, :queues
|
11
|
+
self.sleep_delay = 5
|
12
|
+
self.max_attempts = 25
|
13
|
+
self.max_run_time = 4.hours
|
14
|
+
self.default_priority = 0
|
15
|
+
self.delay_jobs = true
|
16
|
+
self.queues = []
|
17
|
+
|
18
|
+
# Add or remove plugins in this list before the worker is instantiated
|
19
|
+
cattr_accessor :plugins
|
20
|
+
self.plugins = [Delayed::Plugins::ClearLocks]
|
21
|
+
|
22
|
+
# By default failed jobs are destroyed after too many attempts. If you want to keep them around
|
23
|
+
# (perhaps to inspect the reason for the failure), set this to false.
|
24
|
+
cattr_accessor :destroy_failed_jobs
|
25
|
+
self.destroy_failed_jobs = true
|
26
|
+
|
27
|
+
self.logger = if defined?(Rails)
|
28
|
+
Rails.logger
|
29
|
+
elsif defined?(RAILS_DEFAULT_LOGGER)
|
30
|
+
RAILS_DEFAULT_LOGGER
|
31
|
+
end
|
32
|
+
|
33
|
+
# name_prefix is ignored if name is set directly
|
34
|
+
attr_accessor :name_prefix
|
35
|
+
|
36
|
+
cattr_reader :backend
|
37
|
+
|
38
|
+
def self.backend=(backend)
|
39
|
+
if backend.is_a? Symbol
|
40
|
+
require "delayed/serialization/#{backend}" if YAML.parser.class.name =~ /syck/i
|
41
|
+
require "delayed/backend/#{backend}"
|
42
|
+
backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize
|
43
|
+
end
|
44
|
+
@@backend = backend
|
45
|
+
silence_warnings { ::Delayed.const_set(:Job, backend) }
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.guess_backend
|
49
|
+
warn "[DEPRECATION] guess_backend is deprecated. Please remove it from your code."
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.before_fork
|
53
|
+
unless @files_to_reopen
|
54
|
+
@files_to_reopen = []
|
55
|
+
ObjectSpace.each_object(File) do |file|
|
56
|
+
@files_to_reopen << file unless file.closed?
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
backend.before_fork
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.after_fork
|
64
|
+
# Re-open file handles
|
65
|
+
@files_to_reopen.each do |file|
|
66
|
+
begin
|
67
|
+
file.reopen file.path, "a+"
|
68
|
+
file.sync = true
|
69
|
+
rescue ::Exception
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
backend.after_fork
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.lifecycle
|
77
|
+
@lifecycle ||= Delayed::Lifecycle.new
|
78
|
+
end
|
79
|
+
|
80
|
+
def initialize(options={})
|
81
|
+
@quiet = options.has_key?(:quiet) ? options[:quiet] : true
|
82
|
+
self.class.min_priority = options[:min_priority] if options.has_key?(:min_priority)
|
83
|
+
self.class.max_priority = options[:max_priority] if options.has_key?(:max_priority)
|
84
|
+
self.class.sleep_delay = options[:sleep_delay] if options.has_key?(:sleep_delay)
|
85
|
+
self.class.queues = options[:queues] if options.has_key?(:queues)
|
86
|
+
|
87
|
+
self.plugins.each { |klass| klass.new }
|
88
|
+
end
|
89
|
+
|
90
|
+
# Every worker has a unique name which by default is the pid of the process. There are some
|
91
|
+
# advantages to overriding this with something which survives worker retarts: Workers can#
|
92
|
+
# safely resume working on tasks which are locked by themselves. The worker will assume that
|
93
|
+
# it crashed before.
|
94
|
+
def name
|
95
|
+
return @name unless @name.nil?
|
96
|
+
"#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}"
|
97
|
+
end
|
98
|
+
|
99
|
+
# Sets the name of the worker.
|
100
|
+
# Setting the name to nil will reset the default worker name
|
101
|
+
def name=(val)
|
102
|
+
@name = val
|
103
|
+
end
|
104
|
+
|
105
|
+
def start
|
106
|
+
trap('TERM') { say 'Exiting...'; stop }
|
107
|
+
trap('INT') { say 'Exiting...'; stop }
|
108
|
+
|
109
|
+
say "Starting job worker"
|
110
|
+
|
111
|
+
self.class.lifecycle.run_callbacks(:execute, self) do
|
112
|
+
loop do
|
113
|
+
self.class.lifecycle.run_callbacks(:loop, self) do
|
114
|
+
result = nil
|
115
|
+
|
116
|
+
realtime = Benchmark.realtime do
|
117
|
+
result = work_off
|
118
|
+
end
|
119
|
+
|
120
|
+
count = result.sum
|
121
|
+
|
122
|
+
break if @exit
|
123
|
+
|
124
|
+
if count.zero?
|
125
|
+
sleep(self.class.sleep_delay)
|
126
|
+
else
|
127
|
+
say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
break if @exit
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
def stop
|
137
|
+
@exit = true
|
138
|
+
end
|
139
|
+
|
140
|
+
# Do num jobs and return stats on success/failure.
|
141
|
+
# Exit early if interrupted.
|
142
|
+
def work_off(num = 100)
|
143
|
+
success, failure = 0, 0
|
144
|
+
|
145
|
+
num.times do
|
146
|
+
case reserve_and_run_one_job
|
147
|
+
when true
|
148
|
+
success += 1
|
149
|
+
when false
|
150
|
+
failure += 1
|
151
|
+
else
|
152
|
+
break # leave if no work could be done
|
153
|
+
end
|
154
|
+
break if $exit # leave if we're exiting
|
155
|
+
end
|
156
|
+
|
157
|
+
return [success, failure]
|
158
|
+
end
|
159
|
+
|
160
|
+
def run(job)
|
161
|
+
runtime = Benchmark.realtime do
|
162
|
+
Timeout.timeout(self.class.max_run_time.to_i) { job.invoke_job }
|
163
|
+
job.destroy
|
164
|
+
end
|
165
|
+
say "#{job.name} completed after %.4f" % runtime
|
166
|
+
return true # did work
|
167
|
+
rescue DeserializationError => error
|
168
|
+
job.last_error = "{#{error.message}\n#{error.backtrace.join("\n")}"
|
169
|
+
failed(job)
|
170
|
+
rescue Exception => error
|
171
|
+
self.class.lifecycle.run_callbacks(:error, self, job){ handle_failed_job(job, error) }
|
172
|
+
return false # work failed
|
173
|
+
end
|
174
|
+
|
175
|
+
# Reschedule the job in the future (when a job fails).
|
176
|
+
# Uses an exponential scale depending on the number of failed attempts.
|
177
|
+
def reschedule(job, time = nil)
|
178
|
+
if (job.attempts += 1) < max_attempts(job)
|
179
|
+
time ||= job.reschedule_at
|
180
|
+
job.run_at = time
|
181
|
+
job.unlock
|
182
|
+
job.save!
|
183
|
+
else
|
184
|
+
say "PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.", Logger::INFO
|
185
|
+
failed(job)
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
def failed(job)
|
190
|
+
self.class.lifecycle.run_callbacks(:failure, self, job) do
|
191
|
+
job.hook(:failure)
|
192
|
+
self.class.destroy_failed_jobs ? job.destroy : job.fail!
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def say(text, level = Logger::INFO)
|
197
|
+
text = "[Worker(#{name})] #{text}"
|
198
|
+
puts text unless @quiet
|
199
|
+
logger.add level, "#{Time.now.strftime('%FT%T%z')}: #{text}" if logger
|
200
|
+
end
|
201
|
+
|
202
|
+
def max_attempts(job)
|
203
|
+
job.max_attempts || self.class.max_attempts
|
204
|
+
end
|
205
|
+
|
206
|
+
protected
|
207
|
+
|
208
|
+
def handle_failed_job(job, error)
|
209
|
+
job.last_error = "{#{error.message}\n#{error.backtrace.join("\n")}"
|
210
|
+
say "#{job.name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts", Logger::ERROR
|
211
|
+
reschedule(job)
|
212
|
+
end
|
213
|
+
|
214
|
+
# Run the next job we can get an exclusive lock on.
|
215
|
+
# If no jobs are left we return nil
|
216
|
+
def reserve_and_run_one_job
|
217
|
+
job = Delayed::Job.reserve(self)
|
218
|
+
self.class.lifecycle.run_callbacks(:perform, self, job){ result = run(job) } if job
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# These extensions allow properly serializing and autoloading of
|
2
|
+
# Classes, Modules and Structs
|
3
|
+
|
4
|
+
require 'yaml'
|
5
|
+
if YAML.parser.class.name =~ /syck/i
|
6
|
+
require File.expand_path('../syck_ext', __FILE__)
|
7
|
+
require File.expand_path('../serialization/active_record', __FILE__)
|
8
|
+
else
|
9
|
+
require File.expand_path('../psych_ext', __FILE__)
|
10
|
+
end
|
data/lib/delayed_job.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'active_support'
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/delayed/message_sending'
|
4
|
+
require File.dirname(__FILE__) + '/delayed/performable_method'
|
5
|
+
|
6
|
+
# PerformableMailer is compatible with ActionMailer 3 (and possibly 3.1)
|
7
|
+
if defined?(ActionMailer)
|
8
|
+
require 'action_mailer/version'
|
9
|
+
require File.dirname(__FILE__) + '/delayed/performable_mailer' if 3 == ActionMailer::VERSION::MAJOR
|
10
|
+
end
|
11
|
+
|
12
|
+
require File.dirname(__FILE__) + '/delayed/yaml_ext'
|
13
|
+
require File.dirname(__FILE__) + '/delayed/lifecycle'
|
14
|
+
require File.dirname(__FILE__) + '/delayed/plugin'
|
15
|
+
require File.dirname(__FILE__) + '/delayed/plugins/clear_locks'
|
16
|
+
require File.dirname(__FILE__) + '/delayed/backend/base'
|
17
|
+
require File.dirname(__FILE__) + '/delayed/worker'
|
18
|
+
require File.dirname(__FILE__) + '/delayed/deserialization_error'
|
19
|
+
require File.dirname(__FILE__) + '/delayed/railtie' if defined?(Rails::Railtie)
|
20
|
+
|
21
|
+
Object.send(:include, Delayed::MessageSending)
|
22
|
+
Module.send(:include, Delayed::MessageSending::ClassMethods)
|