canvas-jobs 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/db/migrate/20101216224513_create_delayed_jobs.rb +40 -0
- data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +14 -0
- data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +13 -0
- data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +14 -0
- data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +26 -0
- data/db/migrate/20110610213249_optimize_delayed_jobs.rb +40 -0
- data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +52 -0
- data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +31 -0
- data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +15 -0
- data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +80 -0
- data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +15 -0
- data/db/migrate/20120608191051_add_jobs_run_at_index.rb +15 -0
- data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +13 -0
- data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +13 -0
- data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +13 -0
- data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +13 -0
- data/db/migrate/20140512213941_add_source_to_jobs.rb +15 -0
- data/lib/canvas-jobs.rb +1 -0
- data/lib/delayed/backend/active_record.rb +297 -0
- data/lib/delayed/backend/base.rb +317 -0
- data/lib/delayed/backend/redis/bulk_update.lua +40 -0
- data/lib/delayed/backend/redis/destroy_job.lua +2 -0
- data/lib/delayed/backend/redis/enqueue.lua +29 -0
- data/lib/delayed/backend/redis/fail_job.lua +5 -0
- data/lib/delayed/backend/redis/find_available.lua +3 -0
- data/lib/delayed/backend/redis/functions.rb +57 -0
- data/lib/delayed/backend/redis/get_and_lock_next_available.lua +17 -0
- data/lib/delayed/backend/redis/includes/jobs_common.lua +203 -0
- data/lib/delayed/backend/redis/job.rb +481 -0
- data/lib/delayed/backend/redis/set_running.lua +5 -0
- data/lib/delayed/backend/redis/tickle_strand.lua +2 -0
- data/lib/delayed/batch.rb +56 -0
- data/lib/delayed/engine.rb +4 -0
- data/lib/delayed/job_tracking.rb +31 -0
- data/lib/delayed/lifecycle.rb +83 -0
- data/lib/delayed/message_sending.rb +130 -0
- data/lib/delayed/performable_method.rb +42 -0
- data/lib/delayed/periodic.rb +81 -0
- data/lib/delayed/pool.rb +335 -0
- data/lib/delayed/settings.rb +32 -0
- data/lib/delayed/version.rb +3 -0
- data/lib/delayed/worker.rb +213 -0
- data/lib/delayed/yaml_extensions.rb +63 -0
- data/lib/delayed_job.rb +40 -0
- data/spec/active_record_job_spec.rb +61 -0
- data/spec/gemfiles/32.gemfile +6 -0
- data/spec/gemfiles/40.gemfile +6 -0
- data/spec/gemfiles/41.gemfile +6 -0
- data/spec/gemfiles/42.gemfile +6 -0
- data/spec/migrate/20140924140513_add_story_table.rb +7 -0
- data/spec/redis_job_spec.rb +77 -0
- data/spec/sample_jobs.rb +26 -0
- data/spec/shared/delayed_batch.rb +85 -0
- data/spec/shared/delayed_method.rb +419 -0
- data/spec/shared/performable_method.rb +52 -0
- data/spec/shared/shared_backend.rb +836 -0
- data/spec/shared/worker.rb +291 -0
- data/spec/shared_jobs_specs.rb +13 -0
- data/spec/spec_helper.rb +91 -0
- metadata +329 -0
@@ -0,0 +1,56 @@
|
|
1
|
+
module Delayed
|
2
|
+
module Batch
|
3
|
+
class PerformableBatch < Struct.new(:mode, :items)
|
4
|
+
def initialize(mode, items)
|
5
|
+
raise "unsupported mode" unless mode == :serial
|
6
|
+
self.mode = mode
|
7
|
+
self.items = items
|
8
|
+
end
|
9
|
+
|
10
|
+
def display_name
|
11
|
+
"Delayed::Batch.#{mode}"
|
12
|
+
end
|
13
|
+
alias_method :tag, :display_name
|
14
|
+
alias_method :full_name, :display_name
|
15
|
+
|
16
|
+
def perform
|
17
|
+
raise "can't perform a batch directly"
|
18
|
+
end
|
19
|
+
|
20
|
+
def jobs
|
21
|
+
items.map { |opts| Delayed::Job.new(opts) }
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class << self
|
26
|
+
def serial_batch(opts = {})
|
27
|
+
prepare_batches(:serial, opts){ yield }
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
def prepare_batches(mode, opts)
|
32
|
+
raise "nested batching is not supported" if Delayed::Job.batches
|
33
|
+
Delayed::Job.batches = Hash.new { |h,k| h[k] = [] }
|
34
|
+
batch_enqueue_args = [:queue]
|
35
|
+
batch_enqueue_args << :priority unless opts[:priority]
|
36
|
+
Delayed::Job.batch_enqueue_args = batch_enqueue_args
|
37
|
+
yield
|
38
|
+
ensure
|
39
|
+
batches = Delayed::Job.batches
|
40
|
+
Delayed::Job.batches = nil
|
41
|
+
batch_args = opts.slice(:priority)
|
42
|
+
batches.each do |enqueue_args, batch|
|
43
|
+
if batch.size == 0
|
44
|
+
next
|
45
|
+
elsif batch.size == 1
|
46
|
+
args = batch.first.merge(batch_args)
|
47
|
+
payload_object = args.delete(:payload_object)
|
48
|
+
Delayed::Job.enqueue(payload_object, args)
|
49
|
+
else
|
50
|
+
Delayed::Job.enqueue(Delayed::Batch::PerformableBatch.new(mode, batch), enqueue_args.merge(batch_args))
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Delayed
|
2
|
+
# Used when a block of code wants to track what jobs are created,
|
3
|
+
# for instance in tests.
|
4
|
+
# Delayed::Job.track_jobs { ...block... } returns a JobTracking object
|
5
|
+
# Right now this just tracks created jobs, it could be expanded to track a
|
6
|
+
# lot more about what's going on in Delayed Jobs as it's needed.
|
7
|
+
class JobTracking < Struct.new(:created)
|
8
|
+
def self.track
|
9
|
+
@current_tracking = self.new
|
10
|
+
yield
|
11
|
+
tracking = @current_tracking
|
12
|
+
@current_tracking = nil
|
13
|
+
tracking
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.job_created(job)
|
17
|
+
@current_tracking.try(:job_created, job)
|
18
|
+
end
|
19
|
+
|
20
|
+
def job_created(job)
|
21
|
+
return unless job
|
22
|
+
@lock.synchronize { self.created << job }
|
23
|
+
end
|
24
|
+
|
25
|
+
def initialize
|
26
|
+
super
|
27
|
+
self.created = []
|
28
|
+
@lock = Mutex.new
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module Delayed
|
2
|
+
class InvalidCallback < Exception; end
|
3
|
+
|
4
|
+
class Lifecycle
|
5
|
+
EVENTS = {
|
6
|
+
:perform => [:worker, :job],
|
7
|
+
:pop => [:worker],
|
8
|
+
}
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
reset!
|
12
|
+
end
|
13
|
+
|
14
|
+
def reset!
|
15
|
+
@callbacks = EVENTS.keys.inject({}) { |hash, e| hash[e] = Callback.new; hash }
|
16
|
+
end
|
17
|
+
|
18
|
+
def before(event, &block)
|
19
|
+
add(:before, event, &block)
|
20
|
+
end
|
21
|
+
|
22
|
+
def after(event, &block)
|
23
|
+
add(:after, event, &block)
|
24
|
+
end
|
25
|
+
|
26
|
+
def around(event, &block)
|
27
|
+
add(:around, event, &block)
|
28
|
+
end
|
29
|
+
|
30
|
+
def run_callbacks(event, *args, &block)
|
31
|
+
missing_callback(event) unless @callbacks.has_key?(event)
|
32
|
+
|
33
|
+
unless EVENTS[event].size == args.size
|
34
|
+
raise ArgumentError, "Callback #{event} expects #{EVENTS[event].size} parameter(s): #{EVENTS[event].join(', ')}"
|
35
|
+
end
|
36
|
+
|
37
|
+
@callbacks[event].execute(*args, &block)
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def add(type, event, &block)
|
43
|
+
missing_callback(event) unless @callbacks.has_key?(event)
|
44
|
+
|
45
|
+
@callbacks[event].add(type, &block)
|
46
|
+
end
|
47
|
+
|
48
|
+
def missing_callback(event)
|
49
|
+
raise InvalidCallback, "Unknown callback event: #{event}"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
class Callback
|
54
|
+
def initialize
|
55
|
+
@before = []
|
56
|
+
@after = []
|
57
|
+
|
58
|
+
# Identity proc. Avoids special cases when there is no existing around chain.
|
59
|
+
@around = lambda { |*args, &block| block.call(*args) }
|
60
|
+
end
|
61
|
+
|
62
|
+
def execute(*args, &block)
|
63
|
+
@before.each { |c| c.call(*args) }
|
64
|
+
result = @around.call(*args, &block)
|
65
|
+
@after.each { |c| c.call(*args) }
|
66
|
+
result
|
67
|
+
end
|
68
|
+
|
69
|
+
def add(type, &callback)
|
70
|
+
case type
|
71
|
+
when :before
|
72
|
+
@before << callback
|
73
|
+
when :after
|
74
|
+
@after << callback
|
75
|
+
when :around
|
76
|
+
chain = @around # use a local variable so that the current chain is closed over in the following lambda
|
77
|
+
@around = lambda { |*a, &block| chain.call(*a) { |*b| callback.call(*b, &block) } }
|
78
|
+
else
|
79
|
+
raise InvalidCallback, "Invalid callback type: #{type}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,130 @@
|
|
1
|
+
module Delayed
|
2
|
+
module MessageSending
|
3
|
+
def send_later(method, *args)
|
4
|
+
send_later_enqueue_args(method, {}, *args)
|
5
|
+
end
|
6
|
+
|
7
|
+
def send_later_enqueue_args(method, enqueue_args = {}, *args)
|
8
|
+
enqueue_args = enqueue_args.dup
|
9
|
+
# support procs/methods as enqueue arguments
|
10
|
+
enqueue_args.each do |k,v|
|
11
|
+
if v.respond_to?(:call)
|
12
|
+
enqueue_args[k] = v.call(self)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
no_delay = enqueue_args.delete(:no_delay)
|
17
|
+
if !no_delay
|
18
|
+
# delay queuing up the job in another database until the results of the current
|
19
|
+
# transaction are visible
|
20
|
+
connection = self.class.connection if self.class.respond_to?(:connection)
|
21
|
+
connection ||= self.connection if respond_to?(:connection)
|
22
|
+
connection ||= ActiveRecord::Base.connection
|
23
|
+
|
24
|
+
if (Delayed::Job != Delayed::Backend::ActiveRecord::Job || connection != Delayed::Job.connection)
|
25
|
+
connection.after_transaction_commit do
|
26
|
+
Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args), enqueue_args)
|
27
|
+
end
|
28
|
+
return nil
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
result = Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args), enqueue_args)
|
33
|
+
result = nil unless no_delay
|
34
|
+
result
|
35
|
+
end
|
36
|
+
|
37
|
+
def send_later_with_queue(method, queue, *args)
|
38
|
+
send_later_enqueue_args(method, { :queue => queue }, *args)
|
39
|
+
end
|
40
|
+
|
41
|
+
def send_at(time, method, *args)
|
42
|
+
send_later_enqueue_args(method,
|
43
|
+
{ :run_at => time }, *args)
|
44
|
+
end
|
45
|
+
|
46
|
+
def send_at_with_queue(time, method, queue, *args)
|
47
|
+
send_later_enqueue_args(method,
|
48
|
+
{ :run_at => time, :queue => queue },
|
49
|
+
*args)
|
50
|
+
end
|
51
|
+
|
52
|
+
def send_later_unless_in_job(method, *args)
|
53
|
+
if Delayed::Job.in_delayed_job?
|
54
|
+
send(method, *args)
|
55
|
+
else
|
56
|
+
send_later(method, *args)
|
57
|
+
end
|
58
|
+
nil # can't rely on the type of return value, so return nothing
|
59
|
+
end
|
60
|
+
|
61
|
+
def send_later_if_production(*args)
|
62
|
+
if Rails.env.production?
|
63
|
+
send_later(*args)
|
64
|
+
else
|
65
|
+
send(*args)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def send_later_if_production_enqueue_args(method, enqueue_args, *args)
|
70
|
+
if Rails.env.production?
|
71
|
+
send_later_enqueue_args(method, enqueue_args, *args)
|
72
|
+
else
|
73
|
+
send(method, *args)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def send_now_or_later(_when, *args)
|
78
|
+
if _when == :now
|
79
|
+
send(*args)
|
80
|
+
else
|
81
|
+
send_later(*args)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def send_now_or_later_if_production(_when, *args)
|
86
|
+
if _when == :now
|
87
|
+
send(*args)
|
88
|
+
else
|
89
|
+
send_later_if_production(*args)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
module ClassMethods
|
94
|
+
def add_send_later_methods(method, enqueue_args={}, default_async=false)
|
95
|
+
aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
|
96
|
+
|
97
|
+
with_method, without_method = "#{aliased_method}_with_send_later#{punctuation}", "#{aliased_method}_without_send_later#{punctuation}"
|
98
|
+
|
99
|
+
define_method(with_method) do |*args|
|
100
|
+
send_later_enqueue_args(without_method, enqueue_args, *args)
|
101
|
+
end
|
102
|
+
alias_method without_method, method
|
103
|
+
|
104
|
+
if default_async
|
105
|
+
alias_method method, with_method
|
106
|
+
case
|
107
|
+
when public_method_defined?(without_method)
|
108
|
+
public method
|
109
|
+
when protected_method_defined?(without_method)
|
110
|
+
protected method
|
111
|
+
when private_method_defined?(without_method)
|
112
|
+
private method
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def handle_asynchronously(method, enqueue_args={})
|
118
|
+
add_send_later_methods(method, enqueue_args, true)
|
119
|
+
end
|
120
|
+
|
121
|
+
def handle_asynchronously_with_queue(method, queue)
|
122
|
+
add_send_later_methods(method, {:queue => queue}, true)
|
123
|
+
end
|
124
|
+
|
125
|
+
def handle_asynchronously_if_production(method, enqueue_args={})
|
126
|
+
add_send_later_methods(method, enqueue_args, Rails.env.production?)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Delayed
|
2
|
+
class PerformableMethod < Struct.new(:object, :method, :args)
|
3
|
+
def initialize(object, method, args = [])
|
4
|
+
raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method)
|
5
|
+
|
6
|
+
self.object = object
|
7
|
+
self.args = args
|
8
|
+
self.method = method.to_sym
|
9
|
+
end
|
10
|
+
|
11
|
+
def display_name
|
12
|
+
if object.is_a?(Module)
|
13
|
+
"#{object}.#{method}"
|
14
|
+
else
|
15
|
+
"#{object.class}##{method}"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
alias_method :tag, :display_name
|
19
|
+
|
20
|
+
def perform
|
21
|
+
object.send(method, *args)
|
22
|
+
end
|
23
|
+
|
24
|
+
def deep_de_ar_ize(arg)
|
25
|
+
case arg
|
26
|
+
when Hash
|
27
|
+
"{#{arg.map { |k, v| "#{deep_de_ar_ize(k)} => #{deep_de_ar_ize(v)}" }.join(', ')}}"
|
28
|
+
when Array
|
29
|
+
"[#{arg.map { |a| deep_de_ar_ize(a) }.join(', ')}]"
|
30
|
+
when ActiveRecord::Base
|
31
|
+
"#{arg.class}.find(#{arg.id})"
|
32
|
+
else
|
33
|
+
arg.inspect
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def full_name
|
38
|
+
obj_name = object.is_a?(ActiveRecord::Base) ? "#{object.class}.find(#{object.id}).#{method}" : display_name
|
39
|
+
"#{obj_name}(#{args.map { |a| deep_de_ar_ize(a) }.join(', ')})"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'rufus-scheduler'
|
2
|
+
|
3
|
+
module Delayed
|
4
|
+
class Periodic
|
5
|
+
attr_reader :name, :cron
|
6
|
+
|
7
|
+
yaml_as "tag:ruby.yaml.org,2002:Delayed::Periodic"
|
8
|
+
|
9
|
+
def to_yaml(opts = {})
|
10
|
+
YAML.quick_emit(self.object_id, opts) { |out| out.scalar(taguri, @name) }
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.yaml_new(klass, tag, val)
|
14
|
+
self.scheduled[val] || raise(NameError, "job #{val} is no longer scheduled")
|
15
|
+
end
|
16
|
+
|
17
|
+
cattr_accessor :scheduled, :overrides
|
18
|
+
self.scheduled = {}
|
19
|
+
self.overrides = {}
|
20
|
+
|
21
|
+
def self.add_overrides(overrides)
|
22
|
+
overrides.each do |name, cron_line|
|
23
|
+
# throws error if the line is malformed
|
24
|
+
Rufus::CronLine.new(cron_line)
|
25
|
+
end
|
26
|
+
self.overrides.merge!(overrides)
|
27
|
+
end
|
28
|
+
|
29
|
+
STRAND = 'periodic scheduling'
|
30
|
+
|
31
|
+
def self.cron(job_name, cron_line, job_args = {}, &block)
|
32
|
+
raise ArgumentError, "job #{job_name} already scheduled!" if self.scheduled[job_name]
|
33
|
+
cron_line = overrides[job_name] || cron_line
|
34
|
+
self.scheduled[job_name] = self.new(job_name, cron_line, job_args, block)
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.audit_queue
|
38
|
+
# we used to queue up a job in a strand here, and perform the audit inside that job
|
39
|
+
# however, now that we're using singletons for scheduling periodic jobs,
|
40
|
+
# it's fine to just do the audit in-line here without risk of creating duplicates
|
41
|
+
perform_audit!
|
42
|
+
end
|
43
|
+
|
44
|
+
# make sure all periodic jobs are scheduled for their next run in the job queue
|
45
|
+
# this auditing should run on the strand
|
46
|
+
def self.perform_audit!
|
47
|
+
self.scheduled.each { |name, periodic| periodic.enqueue }
|
48
|
+
end
|
49
|
+
|
50
|
+
def initialize(name, cron_line, job_args, block)
|
51
|
+
@name = name
|
52
|
+
@cron = Rufus::CronLine.new(cron_line)
|
53
|
+
@job_args = { :priority => Delayed::LOW_PRIORITY }.merge(job_args.symbolize_keys)
|
54
|
+
@block = block
|
55
|
+
end
|
56
|
+
|
57
|
+
def enqueue
|
58
|
+
Delayed::Job.enqueue(self, @job_args.merge(:max_attempts => 1, :run_at => @cron.next_time(Delayed::Periodic.now), :singleton => tag))
|
59
|
+
end
|
60
|
+
|
61
|
+
def perform
|
62
|
+
@block.call()
|
63
|
+
ensure
|
64
|
+
begin
|
65
|
+
enqueue
|
66
|
+
rescue
|
67
|
+
# double fail! the auditor will have to catch this.
|
68
|
+
Rails.logger.error "Failure enqueueing periodic job! #{@name} #{$!.inspect}"
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def tag
|
73
|
+
"periodic: #{@name}"
|
74
|
+
end
|
75
|
+
alias_method :display_name, :tag
|
76
|
+
|
77
|
+
def self.now
|
78
|
+
Time.now
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|