delayed 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +20 -0
- data/README.md +560 -0
- data/Rakefile +35 -0
- data/lib/delayed.rb +72 -0
- data/lib/delayed/active_job_adapter.rb +65 -0
- data/lib/delayed/backend/base.rb +166 -0
- data/lib/delayed/backend/job_preparer.rb +43 -0
- data/lib/delayed/exceptions.rb +14 -0
- data/lib/delayed/job.rb +250 -0
- data/lib/delayed/lifecycle.rb +85 -0
- data/lib/delayed/message_sending.rb +65 -0
- data/lib/delayed/monitor.rb +134 -0
- data/lib/delayed/performable_mailer.rb +22 -0
- data/lib/delayed/performable_method.rb +47 -0
- data/lib/delayed/plugin.rb +15 -0
- data/lib/delayed/plugins/connection.rb +13 -0
- data/lib/delayed/plugins/instrumentation.rb +39 -0
- data/lib/delayed/priority.rb +164 -0
- data/lib/delayed/psych_ext.rb +135 -0
- data/lib/delayed/railtie.rb +7 -0
- data/lib/delayed/runnable.rb +46 -0
- data/lib/delayed/serialization/active_record.rb +18 -0
- data/lib/delayed/syck_ext.rb +42 -0
- data/lib/delayed/tasks.rb +40 -0
- data/lib/delayed/worker.rb +233 -0
- data/lib/delayed/yaml_ext.rb +10 -0
- data/lib/delayed_job.rb +1 -0
- data/lib/delayed_job_active_record.rb +1 -0
- data/lib/generators/delayed/generator.rb +7 -0
- data/lib/generators/delayed/migration_generator.rb +28 -0
- data/lib/generators/delayed/next_migration_version.rb +14 -0
- data/lib/generators/delayed/templates/migration.rb +22 -0
- data/spec/autoloaded/clazz.rb +6 -0
- data/spec/autoloaded/instance_clazz.rb +5 -0
- data/spec/autoloaded/instance_struct.rb +6 -0
- data/spec/autoloaded/struct.rb +7 -0
- data/spec/database.yml +25 -0
- data/spec/delayed/active_job_adapter_spec.rb +267 -0
- data/spec/delayed/job_spec.rb +953 -0
- data/spec/delayed/monitor_spec.rb +276 -0
- data/spec/delayed/plugins/instrumentation_spec.rb +49 -0
- data/spec/delayed/priority_spec.rb +154 -0
- data/spec/delayed/serialization/active_record_spec.rb +15 -0
- data/spec/delayed/tasks_spec.rb +116 -0
- data/spec/helper.rb +196 -0
- data/spec/lifecycle_spec.rb +77 -0
- data/spec/message_sending_spec.rb +149 -0
- data/spec/performable_mailer_spec.rb +68 -0
- data/spec/performable_method_spec.rb +123 -0
- data/spec/psych_ext_spec.rb +94 -0
- data/spec/sample_jobs.rb +117 -0
- data/spec/worker_spec.rb +235 -0
- data/spec/yaml_ext_spec.rb +48 -0
- metadata +326 -0
@@ -0,0 +1,135 @@
|
|
1
|
+
module Delayed
|
2
|
+
class PerformableMethod
|
3
|
+
# serialize to YAML
|
4
|
+
def encode_with(coder)
|
5
|
+
coder.map = {
|
6
|
+
'object' => object,
|
7
|
+
'method_name' => method_name,
|
8
|
+
'args' => args,
|
9
|
+
}
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
module Psych
|
15
|
+
def self.load_dj(yaml)
|
16
|
+
result = parse(yaml)
|
17
|
+
result ? Delayed::PsychExt::ToRuby.create.accept(result) : result
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.dump_dj(object)
|
21
|
+
visitor = Delayed::PsychExt::YAMLTree.create
|
22
|
+
visitor << object
|
23
|
+
visitor.tree.yaml
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
module Delayed
|
28
|
+
module PsychExt
|
29
|
+
class YAMLTree < Psych::Visitors::YAMLTree
|
30
|
+
def accept(target)
|
31
|
+
if defined?(ActiveRecord::Base) && target.is_a?(ActiveRecord::Base)
|
32
|
+
tag = ['!ruby/ActiveRecord', target.class.name].compact.join(':')
|
33
|
+
map = @emitter.start_mapping(nil, tag, false, Psych::Nodes::Mapping::BLOCK)
|
34
|
+
register(target, map)
|
35
|
+
@emitter.scalar('attributes', nil, nil, true, false, Psych::Nodes::Mapping::ANY)
|
36
|
+
accept target.attributes.slice(target.class.primary_key)
|
37
|
+
|
38
|
+
@emitter.end_mapping
|
39
|
+
else
|
40
|
+
super
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
class ToRuby < Psych::Visitors::ToRuby
|
46
|
+
unless respond_to?(:create)
|
47
|
+
def self.create
|
48
|
+
new
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def accept(target)
|
53
|
+
super.tap do |value|
|
54
|
+
register(target, value) if value.class.include?(Singleton)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def visit_Psych_Nodes_Mapping(object) # rubocop:disable Metrics/CyclomaticComplexity, Naming/MethodName, Metrics/PerceivedComplexity
|
59
|
+
klass = Psych.load_tags[object.tag]
|
60
|
+
if klass
|
61
|
+
# Implementation changed here https://github.com/ruby/psych/commit/2c644e184192975b261a81f486a04defa3172b3f
|
62
|
+
# load_tags used to have class values, now the values are strings
|
63
|
+
klass = resolve_class(klass) if klass.is_a?(String)
|
64
|
+
return revive(klass, object)
|
65
|
+
end
|
66
|
+
|
67
|
+
case object.tag
|
68
|
+
when %r{^!ruby/object}
|
69
|
+
result = super
|
70
|
+
if jruby_is_seriously_borked && result.is_a?(ActiveRecord::Base)
|
71
|
+
klass = result.class
|
72
|
+
id = result[klass.primary_key]
|
73
|
+
begin
|
74
|
+
klass.unscoped.find(id)
|
75
|
+
rescue ActiveRecord::RecordNotFound => e
|
76
|
+
raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass}, primary key: #{id} (#{e.message})"
|
77
|
+
end
|
78
|
+
else
|
79
|
+
result
|
80
|
+
end
|
81
|
+
when %r{^!ruby/ActiveRecord:(.+)$}
|
82
|
+
klass = resolve_class(Regexp.last_match[1])
|
83
|
+
payload = Hash[*object.children.map { |c| accept c }]
|
84
|
+
id = payload['attributes'][klass.primary_key]
|
85
|
+
id = id.value if defined?(ActiveRecord::Attribute) && id.is_a?(ActiveRecord::Attribute)
|
86
|
+
begin
|
87
|
+
klass.unscoped.find(id)
|
88
|
+
rescue ActiveRecord::RecordNotFound => e
|
89
|
+
raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass}, primary key: #{id} (#{e.message})"
|
90
|
+
end
|
91
|
+
when %r{^!ruby/Mongoid:(.+)$}
|
92
|
+
klass = resolve_class(Regexp.last_match[1])
|
93
|
+
payload = Hash[*object.children.map { |c| accept c }]
|
94
|
+
id = payload['attributes']['_id']
|
95
|
+
begin
|
96
|
+
klass.find(id)
|
97
|
+
rescue Mongoid::Errors::DocumentNotFound => e
|
98
|
+
raise Delayed::DeserializationError, "Mongoid::Errors::DocumentNotFound, class: #{klass}, primary key: #{id} (#{e.message})"
|
99
|
+
end
|
100
|
+
when %r{^!ruby/DataMapper:(.+)$}
|
101
|
+
klass = resolve_class(Regexp.last_match[1])
|
102
|
+
payload = Hash[*object.children.map { |c| accept c }]
|
103
|
+
begin
|
104
|
+
primary_keys = klass.properties.select(&:key?)
|
105
|
+
key_names = primary_keys.map { |p| p.name.to_s }
|
106
|
+
klass.get!(*key_names.map { |k| payload['attributes'][k] })
|
107
|
+
rescue DataMapper::ObjectNotFoundError => e
|
108
|
+
raise Delayed::DeserializationError, "DataMapper::ObjectNotFoundError, class: #{klass} (#{e.message})"
|
109
|
+
end
|
110
|
+
else
|
111
|
+
super
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
# defined? is triggering something really messed up in
|
116
|
+
# jruby causing both the if AND else clauses to execute,
|
117
|
+
# however if the check is run here, everything is fine
|
118
|
+
def jruby_is_seriously_borked
|
119
|
+
defined?(ActiveRecord::Base)
|
120
|
+
end
|
121
|
+
|
122
|
+
def resolve_class(klass_name)
|
123
|
+
return nil if klass_name.blank?
|
124
|
+
|
125
|
+
klass_name.constantize
|
126
|
+
rescue StandardError
|
127
|
+
super
|
128
|
+
end
|
129
|
+
|
130
|
+
def revive(klass, node)
|
131
|
+
klass.include?(Singleton) ? klass.instance : super
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module Delayed
|
2
|
+
module Runnable
|
3
|
+
def start
|
4
|
+
trap('TERM') { quit! }
|
5
|
+
trap('INT') { quit! }
|
6
|
+
|
7
|
+
say "Starting #{self.class.name}"
|
8
|
+
|
9
|
+
Delayed.lifecycle.run_callbacks(:execute, nil) do
|
10
|
+
loop do
|
11
|
+
run!
|
12
|
+
break if stop?
|
13
|
+
end
|
14
|
+
end
|
15
|
+
ensure
|
16
|
+
on_exit!
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def on_exit!; end
|
22
|
+
|
23
|
+
def interruptable_sleep(seconds)
|
24
|
+
IO.select([pipe[0]], nil, nil, seconds)
|
25
|
+
end
|
26
|
+
|
27
|
+
def stop
|
28
|
+
pipe[1].close
|
29
|
+
end
|
30
|
+
|
31
|
+
def stop?
|
32
|
+
pipe[1].closed?
|
33
|
+
end
|
34
|
+
|
35
|
+
def quit!
|
36
|
+
Thread.new { say 'Exiting...' }.tap do |t|
|
37
|
+
stop
|
38
|
+
t.join
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def pipe
|
43
|
+
@pipe ||= IO.pipe
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
if defined?(ActiveRecord)
|
2
|
+
module ActiveRecord
|
3
|
+
class Base
|
4
|
+
yaml_tag 'tag:ruby.yaml.org,2002:ActiveRecord'
|
5
|
+
|
6
|
+
def self.yaml_new(klass, _tag, val)
|
7
|
+
klass.unscoped.find(val['attributes'][klass.primary_key])
|
8
|
+
rescue ActiveRecord::RecordNotFound
|
9
|
+
raise Delayed::DeserializationError,
|
10
|
+
"ActiveRecord::RecordNotFound, class: #{klass} , primary key: #{val['attributes'][klass.primary_key]}"
|
11
|
+
end
|
12
|
+
|
13
|
+
def to_yaml_properties
|
14
|
+
['@attributes']
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
class Module
|
2
|
+
yaml_tag 'tag:ruby.yaml.org,2002:module'
|
3
|
+
|
4
|
+
def self.yaml_new(_klass, _tag, val)
|
5
|
+
val.constantize
|
6
|
+
end
|
7
|
+
|
8
|
+
def to_yaml(options = {})
|
9
|
+
YAML.quick_emit(nil, options) do |out|
|
10
|
+
out.scalar(taguri, name, :plain)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def yaml_tag_read_class(name)
|
15
|
+
# Constantize the object so that ActiveSupport can attempt
|
16
|
+
# its auto loading magic. Will raise LoadError if not successful.
|
17
|
+
name.constantize
|
18
|
+
name
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
class Class
|
23
|
+
yaml_tag 'tag:ruby.yaml.org,2002:class'
|
24
|
+
remove_method :to_yaml if respond_to?(:to_yaml) && method(:to_yaml).owner == Class # use Module's to_yaml
|
25
|
+
end
|
26
|
+
|
27
|
+
class Struct
|
28
|
+
def self.yaml_tag_read_class(name)
|
29
|
+
# Constantize the object so that ActiveSupport can attempt
|
30
|
+
# its auto loading magic. Will raise LoadError if not successful.
|
31
|
+
name.constantize
|
32
|
+
"Struct::#{name}"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
module YAML
|
37
|
+
def load_dj(yaml)
|
38
|
+
# See https://github.com/dtao/safe_yaml
|
39
|
+
# When the method is there, we need to load our YAML like this...
|
40
|
+
respond_to?(:unsafe_load) ? load(yaml, safe: false) : load(yaml)
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
namespace :delayed do
|
2
|
+
task delayed_environment: :environment do
|
3
|
+
Delayed::Worker.min_priority = ENV['MIN_PRIORITY'].to_i if ENV.key?('MIN_PRIORITY')
|
4
|
+
Delayed::Worker.max_priority = ENV['MAX_PRIORITY'].to_i if ENV.key?('MAX_PRIORITY')
|
5
|
+
Delayed::Worker.queues = [ENV['QUEUE']] if ENV.key?('QUEUE')
|
6
|
+
Delayed::Worker.queues = ENV['QUEUES'].split(',') if ENV.key?('QUEUES')
|
7
|
+
Delayed::Worker.sleep_delay = ENV['SLEEP_DELAY'].to_i if ENV.key?('SLEEP_DELAY')
|
8
|
+
Delayed::Worker.read_ahead = ENV['READ_AHEAD'].to_i if ENV.key?('READ_AHEAD')
|
9
|
+
Delayed::Worker.max_claims = ENV['MAX_CLAIMS'].to_i if ENV.key?('MAX_CLAIMS')
|
10
|
+
|
11
|
+
next unless defined?(Rails.application.config)
|
12
|
+
|
13
|
+
# By default, Rails < 6.1 overrides eager_load to 'false' inside of rake tasks, which is not ideal in production environments.
|
14
|
+
# Additionally, the classic Rails autoloader is not threadsafe, so we do not want any autoloading after we start the worker.
|
15
|
+
# While the zeitwork autoloader technically does not need this workaround, we will still eager load for consistency's sake.
|
16
|
+
# We will use the cache_classes config as a proxy for determining if we should eager load before booting workers.
|
17
|
+
if !Rails.application.config.respond_to?(:rake_eager_load) && Rails.application.config.cache_classes
|
18
|
+
Rails.application.config.eager_load = true
|
19
|
+
Rails::Application::Finisher.initializers
|
20
|
+
.find { |i| i.name == :eager_load! }
|
21
|
+
.bind(Rails.application)
|
22
|
+
.run
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
desc 'start a delayed worker'
|
27
|
+
task work: :delayed_environment do
|
28
|
+
Delayed::Worker.new.start
|
29
|
+
end
|
30
|
+
|
31
|
+
desc 'monitor job queue and emit metrics at an interval'
|
32
|
+
task monitor: :delayed_environment do
|
33
|
+
Delayed::Monitor.new.start
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# For backwards compatibility:
|
38
|
+
namespace :jobs do
|
39
|
+
task work: %i(delayed:work)
|
40
|
+
end
|
@@ -0,0 +1,233 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
require 'active_support/dependencies'
|
3
|
+
require 'active_support/core_ext/numeric/time'
|
4
|
+
require 'active_support/core_ext/class/attribute_accessors'
|
5
|
+
require 'active_support/hash_with_indifferent_access'
|
6
|
+
require 'active_support/core_ext/hash/indifferent_access'
|
7
|
+
require 'benchmark'
|
8
|
+
require 'concurrent'
|
9
|
+
|
10
|
+
module Delayed
|
11
|
+
class Worker
|
12
|
+
include Runnable
|
13
|
+
|
14
|
+
cattr_accessor :sleep_delay, instance_writer: false, default: 5
|
15
|
+
cattr_accessor :max_attempts, instance_writer: false, default: 25
|
16
|
+
cattr_accessor :max_claims, instance_writer: false, default: 5
|
17
|
+
cattr_accessor :max_run_time, instance_writer: false, default: 20.minutes
|
18
|
+
cattr_accessor :default_priority, instance_writer: false, default: 10
|
19
|
+
cattr_accessor :delay_jobs, instance_writer: false, default: true
|
20
|
+
cattr_accessor :queues, instance_writer: false, default: [].freeze
|
21
|
+
cattr_accessor :read_ahead, instance_writer: false, default: 5
|
22
|
+
cattr_accessor :destroy_failed_jobs, instance_writer: false, default: false
|
23
|
+
|
24
|
+
cattr_accessor :min_priority, :max_priority, instance_writer: false
|
25
|
+
|
26
|
+
# TODO: Remove this and rely on ActiveJob.queue_name when no queue is specified
|
27
|
+
cattr_accessor :default_queue_name, instance_writer: false, default: 'default'
|
28
|
+
|
29
|
+
# name_prefix is ignored if name is set directly
|
30
|
+
attr_accessor :name_prefix
|
31
|
+
|
32
|
+
class << self
|
33
|
+
delegate :lifecycle, :plugins, :plugins=, :logger, :logger=,
|
34
|
+
:default_log_level, :default_log_level=, to: Delayed
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.delay_job?(job)
|
38
|
+
if delay_jobs.is_a?(Proc)
|
39
|
+
delay_jobs.arity == 1 ? delay_jobs.call(job) : delay_jobs.call
|
40
|
+
else
|
41
|
+
delay_jobs
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def initialize
|
46
|
+
@failed_reserve_count = 0
|
47
|
+
|
48
|
+
# Reset lifecycle on the offhand chance that something lazily
|
49
|
+
# triggered its creation before all plugins had been registered.
|
50
|
+
Delayed.setup_lifecycle
|
51
|
+
end
|
52
|
+
|
53
|
+
# Every worker has a unique name which by default is the pid of the process. There are some
|
54
|
+
# advantages to overriding this with something which survives worker restarts: Workers can
|
55
|
+
# safely resume working on tasks which are locked by themselves. The worker will assume that
|
56
|
+
# it crashed before.
|
57
|
+
def name
|
58
|
+
return @name unless @name.nil?
|
59
|
+
|
60
|
+
begin
|
61
|
+
"#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}"
|
62
|
+
rescue StandardError
|
63
|
+
"#{@name_prefix}pid:#{Process.pid}"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
# Sets the name of the worker.
|
68
|
+
# Setting the name to nil will reset the default worker name
|
69
|
+
attr_writer :name
|
70
|
+
|
71
|
+
def run!
|
72
|
+
@realtime = Benchmark.realtime do
|
73
|
+
@result = work_off
|
74
|
+
end
|
75
|
+
|
76
|
+
count = @result[0] + @result[1]
|
77
|
+
|
78
|
+
say format("#{count} jobs processed at %.4f j/s, %d failed", count / @realtime, @result.last) if count.positive?
|
79
|
+
interruptable_sleep(self.class.sleep_delay) if count < max_claims
|
80
|
+
|
81
|
+
reload! unless stop?
|
82
|
+
end
|
83
|
+
|
84
|
+
def on_exit!
|
85
|
+
Delayed::Job.clear_locks!(name)
|
86
|
+
end
|
87
|
+
|
88
|
+
# Do num jobs and return stats on success/failure.
|
89
|
+
# Exit early if interrupted.
|
90
|
+
def work_off(num = 100)
|
91
|
+
success = Concurrent::AtomicFixnum.new(0)
|
92
|
+
failure = Concurrent::AtomicFixnum.new(0)
|
93
|
+
|
94
|
+
num.times do
|
95
|
+
jobs = reserve_jobs
|
96
|
+
break if jobs.empty?
|
97
|
+
|
98
|
+
pool = Concurrent::FixedThreadPool.new(jobs.length)
|
99
|
+
jobs.each do |job|
|
100
|
+
pool.post do
|
101
|
+
run_thread_callbacks(job) do
|
102
|
+
if run_job(job)
|
103
|
+
success.increment
|
104
|
+
else
|
105
|
+
failure.increment
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
pool.shutdown
|
112
|
+
pool.wait_for_termination
|
113
|
+
|
114
|
+
break if stop? # leave if we're exiting
|
115
|
+
end
|
116
|
+
|
117
|
+
[success, failure].map(&:value)
|
118
|
+
end
|
119
|
+
|
120
|
+
def run_thread_callbacks(job, &block)
|
121
|
+
self.class.lifecycle.run_callbacks(:thread, self, job, &block)
|
122
|
+
end
|
123
|
+
|
124
|
+
def run(job)
|
125
|
+
metadata = {
|
126
|
+
status: 'RUNNING',
|
127
|
+
name: job.name,
|
128
|
+
run_at: job.run_at,
|
129
|
+
created_at: job.created_at,
|
130
|
+
priority: job.priority,
|
131
|
+
queue: job.queue,
|
132
|
+
attempts: job.attempts,
|
133
|
+
enqueued_for: (Time.current - job.created_at).round,
|
134
|
+
}
|
135
|
+
job_say job, metadata.to_json
|
136
|
+
run_time = Benchmark.realtime do
|
137
|
+
Timeout.timeout(max_run_time(job).to_i, WorkerTimeout) do
|
138
|
+
job.invoke_job
|
139
|
+
end
|
140
|
+
job.destroy
|
141
|
+
end
|
142
|
+
job_say job, format('COMPLETED after %.4f seconds', run_time)
|
143
|
+
true # did work
|
144
|
+
rescue DeserializationError => e
|
145
|
+
job_say job, "FAILED permanently with #{e.class.name}: #{e.message}", 'error'
|
146
|
+
|
147
|
+
job.error = e
|
148
|
+
failed(job)
|
149
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
150
|
+
self.class.lifecycle.run_callbacks(:error, self, job) { handle_failed_job(job, e) }
|
151
|
+
false # work failed
|
152
|
+
end
|
153
|
+
|
154
|
+
# Reschedule the job in the future (when a job fails).
|
155
|
+
# Uses an exponential scale depending on the number of failed attempts.
|
156
|
+
def reschedule(job, time = nil)
|
157
|
+
if (job.attempts += 1) < max_attempts(job)
|
158
|
+
time ||= job.reschedule_at
|
159
|
+
job.run_at = time
|
160
|
+
job.unlock
|
161
|
+
job.save!
|
162
|
+
else
|
163
|
+
job_say job, "FAILED permanently because of #{job.attempts} consecutive failures", 'error'
|
164
|
+
failed(job)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def failed(job)
|
169
|
+
self.class.lifecycle.run_callbacks(:failure, self, job) do
|
170
|
+
job.hook(:failure)
|
171
|
+
rescue StandardError => e
|
172
|
+
say "Error when running failure callback: #{e}", 'error'
|
173
|
+
say e.backtrace.join("\n"), 'error'
|
174
|
+
ensure
|
175
|
+
job.destroy_failed_jobs? ? job.destroy : job.fail!
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
def job_say(job, text, level = Delayed.default_log_level)
|
180
|
+
text = "Job #{job.name} (id=#{job.id})#{say_queue(job.queue)} #{text}"
|
181
|
+
say text, level
|
182
|
+
end
|
183
|
+
|
184
|
+
def say(text, level = Delayed.default_log_level)
|
185
|
+
text = "[Worker(#{name})] #{text}"
|
186
|
+
Delayed.say("#{Time.now.strftime('%FT%T%z')}: #{text}", level)
|
187
|
+
end
|
188
|
+
|
189
|
+
def max_attempts(job)
|
190
|
+
job.max_attempts || self.class.max_attempts
|
191
|
+
end
|
192
|
+
|
193
|
+
def max_run_time(job)
|
194
|
+
job.max_run_time || self.class.max_run_time
|
195
|
+
end
|
196
|
+
|
197
|
+
protected
|
198
|
+
|
199
|
+
def say_queue(queue)
|
200
|
+
" (queue=#{queue})" if queue
|
201
|
+
end
|
202
|
+
|
203
|
+
def handle_failed_job(job, error)
|
204
|
+
job.error = error
|
205
|
+
job_say job, "FAILED (#{job.attempts} prior attempts) with #{error.class.name}: #{error.message}", 'error'
|
206
|
+
reschedule(job)
|
207
|
+
end
|
208
|
+
|
209
|
+
def run_job(job)
|
210
|
+
self.class.lifecycle.run_callbacks(:perform, self, job) { run(job) }
|
211
|
+
end
|
212
|
+
|
213
|
+
# The backend adapter may return either a list or a single job
|
214
|
+
# In some backends, this can be controlled with the `max_claims` config
|
215
|
+
# Either way, we map this to an array of job instances
|
216
|
+
def reserve_jobs
|
217
|
+
jobs = [Delayed::Job.reserve(self)].compact.flatten(1)
|
218
|
+
@failed_reserve_count = 0
|
219
|
+
jobs
|
220
|
+
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
221
|
+
say "Error while reserving job(s): #{e}"
|
222
|
+
Delayed::Job.recover_from(e)
|
223
|
+
@failed_reserve_count += 1
|
224
|
+
raise FatalBackendError if @failed_reserve_count >= 10
|
225
|
+
|
226
|
+
[]
|
227
|
+
end
|
228
|
+
|
229
|
+
def reload!
|
230
|
+
Rails.application.reloader.reload! if defined?(Rails.application.reloader) && Rails.application.reloader.check!
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|