canvas-jobs 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/db/migrate/20101216224513_create_delayed_jobs.rb +40 -0
  3. data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +14 -0
  4. data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +13 -0
  5. data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +14 -0
  6. data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +26 -0
  7. data/db/migrate/20110610213249_optimize_delayed_jobs.rb +40 -0
  8. data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +52 -0
  9. data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +31 -0
  10. data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +15 -0
  11. data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +80 -0
  12. data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +15 -0
  13. data/db/migrate/20120608191051_add_jobs_run_at_index.rb +15 -0
  14. data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +13 -0
  15. data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +13 -0
  16. data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +13 -0
  17. data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +13 -0
  18. data/db/migrate/20140512213941_add_source_to_jobs.rb +15 -0
  19. data/lib/canvas-jobs.rb +1 -0
  20. data/lib/delayed/backend/active_record.rb +297 -0
  21. data/lib/delayed/backend/base.rb +317 -0
  22. data/lib/delayed/backend/redis/bulk_update.lua +40 -0
  23. data/lib/delayed/backend/redis/destroy_job.lua +2 -0
  24. data/lib/delayed/backend/redis/enqueue.lua +29 -0
  25. data/lib/delayed/backend/redis/fail_job.lua +5 -0
  26. data/lib/delayed/backend/redis/find_available.lua +3 -0
  27. data/lib/delayed/backend/redis/functions.rb +57 -0
  28. data/lib/delayed/backend/redis/get_and_lock_next_available.lua +17 -0
  29. data/lib/delayed/backend/redis/includes/jobs_common.lua +203 -0
  30. data/lib/delayed/backend/redis/job.rb +481 -0
  31. data/lib/delayed/backend/redis/set_running.lua +5 -0
  32. data/lib/delayed/backend/redis/tickle_strand.lua +2 -0
  33. data/lib/delayed/batch.rb +56 -0
  34. data/lib/delayed/engine.rb +4 -0
  35. data/lib/delayed/job_tracking.rb +31 -0
  36. data/lib/delayed/lifecycle.rb +83 -0
  37. data/lib/delayed/message_sending.rb +130 -0
  38. data/lib/delayed/performable_method.rb +42 -0
  39. data/lib/delayed/periodic.rb +81 -0
  40. data/lib/delayed/pool.rb +335 -0
  41. data/lib/delayed/settings.rb +32 -0
  42. data/lib/delayed/version.rb +3 -0
  43. data/lib/delayed/worker.rb +213 -0
  44. data/lib/delayed/yaml_extensions.rb +63 -0
  45. data/lib/delayed_job.rb +40 -0
  46. data/spec/active_record_job_spec.rb +61 -0
  47. data/spec/gemfiles/32.gemfile +6 -0
  48. data/spec/gemfiles/40.gemfile +6 -0
  49. data/spec/gemfiles/41.gemfile +6 -0
  50. data/spec/gemfiles/42.gemfile +6 -0
  51. data/spec/migrate/20140924140513_add_story_table.rb +7 -0
  52. data/spec/redis_job_spec.rb +77 -0
  53. data/spec/sample_jobs.rb +26 -0
  54. data/spec/shared/delayed_batch.rb +85 -0
  55. data/spec/shared/delayed_method.rb +419 -0
  56. data/spec/shared/performable_method.rb +52 -0
  57. data/spec/shared/shared_backend.rb +836 -0
  58. data/spec/shared/worker.rb +291 -0
  59. data/spec/shared_jobs_specs.rb +13 -0
  60. data/spec/spec_helper.rb +91 -0
  61. metadata +329 -0
@@ -0,0 +1,5 @@
1
+ local job_id = unpack(ARGV)
2
+ local locked_at, queue, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'locked_at', 'queue', 'strand'))
3
+
4
+ remove_from_queues(job_id, queue, strand)
5
+ redis.call('ZADD', Keys.running_jobs(), locked_at, job_id)
@@ -0,0 +1,2 @@
1
+ local job_id, strand, now = unpack(ARGV)
2
+ tickle_strand(job_id, strand, now)
@@ -0,0 +1,56 @@
1
+ module Delayed
2
+ module Batch
3
+ class PerformableBatch < Struct.new(:mode, :items)
4
+ def initialize(mode, items)
5
+ raise "unsupported mode" unless mode == :serial
6
+ self.mode = mode
7
+ self.items = items
8
+ end
9
+
10
+ def display_name
11
+ "Delayed::Batch.#{mode}"
12
+ end
13
+ alias_method :tag, :display_name
14
+ alias_method :full_name, :display_name
15
+
16
+ def perform
17
+ raise "can't perform a batch directly"
18
+ end
19
+
20
+ def jobs
21
+ items.map { |opts| Delayed::Job.new(opts) }
22
+ end
23
+ end
24
+
25
+ class << self
26
+ def serial_batch(opts = {})
27
+ prepare_batches(:serial, opts){ yield }
28
+ end
29
+
30
+ private
31
+ def prepare_batches(mode, opts)
32
+ raise "nested batching is not supported" if Delayed::Job.batches
33
+ Delayed::Job.batches = Hash.new { |h,k| h[k] = [] }
34
+ batch_enqueue_args = [:queue]
35
+ batch_enqueue_args << :priority unless opts[:priority]
36
+ Delayed::Job.batch_enqueue_args = batch_enqueue_args
37
+ yield
38
+ ensure
39
+ batches = Delayed::Job.batches
40
+ Delayed::Job.batches = nil
41
+ batch_args = opts.slice(:priority)
42
+ batches.each do |enqueue_args, batch|
43
+ if batch.size == 0
44
+ next
45
+ elsif batch.size == 1
46
+ args = batch.first.merge(batch_args)
47
+ payload_object = args.delete(:payload_object)
48
+ Delayed::Job.enqueue(payload_object, args)
49
+ else
50
+ Delayed::Job.enqueue(Delayed::Batch::PerformableBatch.new(mode, batch), enqueue_args.merge(batch_args))
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,4 @@
1
+ module Delayed
2
+ class Engine < ::Rails::Engine
3
+ end
4
+ end
@@ -0,0 +1,31 @@
1
+ module Delayed
2
+ # Used when a block of code wants to track what jobs are created,
3
+ # for instance in tests.
4
+ # Delayed::Job.track_jobs { ...block... } returns a JobTracking object
5
+ # Right now this just tracks created jobs, it could be expanded to track a
6
+ # lot more about what's going on in Delayed Jobs as it's needed.
7
+ class JobTracking < Struct.new(:created)
8
+ def self.track
9
+ @current_tracking = self.new
10
+ yield
11
+ tracking = @current_tracking
12
+ @current_tracking = nil
13
+ tracking
14
+ end
15
+
16
+ def self.job_created(job)
17
+ @current_tracking.try(:job_created, job)
18
+ end
19
+
20
+ def job_created(job)
21
+ return unless job
22
+ @lock.synchronize { self.created << job }
23
+ end
24
+
25
+ def initialize
26
+ super
27
+ self.created = []
28
+ @lock = Mutex.new
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,83 @@
1
+ module Delayed
2
+ class InvalidCallback < Exception; end
3
+
4
+ class Lifecycle
5
+ EVENTS = {
6
+ :perform => [:worker, :job],
7
+ :pop => [:worker],
8
+ }
9
+
10
+ def initialize
11
+ reset!
12
+ end
13
+
14
+ def reset!
15
+ @callbacks = EVENTS.keys.inject({}) { |hash, e| hash[e] = Callback.new; hash }
16
+ end
17
+
18
+ def before(event, &block)
19
+ add(:before, event, &block)
20
+ end
21
+
22
+ def after(event, &block)
23
+ add(:after, event, &block)
24
+ end
25
+
26
+ def around(event, &block)
27
+ add(:around, event, &block)
28
+ end
29
+
30
+ def run_callbacks(event, *args, &block)
31
+ missing_callback(event) unless @callbacks.has_key?(event)
32
+
33
+ unless EVENTS[event].size == args.size
34
+ raise ArgumentError, "Callback #{event} expects #{EVENTS[event].size} parameter(s): #{EVENTS[event].join(', ')}"
35
+ end
36
+
37
+ @callbacks[event].execute(*args, &block)
38
+ end
39
+
40
+ private
41
+
42
+ def add(type, event, &block)
43
+ missing_callback(event) unless @callbacks.has_key?(event)
44
+
45
+ @callbacks[event].add(type, &block)
46
+ end
47
+
48
+ def missing_callback(event)
49
+ raise InvalidCallback, "Unknown callback event: #{event}"
50
+ end
51
+ end
52
+
53
+ class Callback
54
+ def initialize
55
+ @before = []
56
+ @after = []
57
+
58
+ # Identity proc. Avoids special cases when there is no existing around chain.
59
+ @around = lambda { |*args, &block| block.call(*args) }
60
+ end
61
+
62
+ def execute(*args, &block)
63
+ @before.each { |c| c.call(*args) }
64
+ result = @around.call(*args, &block)
65
+ @after.each { |c| c.call(*args) }
66
+ result
67
+ end
68
+
69
+ def add(type, &callback)
70
+ case type
71
+ when :before
72
+ @before << callback
73
+ when :after
74
+ @after << callback
75
+ when :around
76
+ chain = @around # use a local variable so that the current chain is closed over in the following lambda
77
+ @around = lambda { |*a, &block| chain.call(*a) { |*b| callback.call(*b, &block) } }
78
+ else
79
+ raise InvalidCallback, "Invalid callback type: #{type}"
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,130 @@
1
+ module Delayed
2
+ module MessageSending
3
+ def send_later(method, *args)
4
+ send_later_enqueue_args(method, {}, *args)
5
+ end
6
+
7
+ def send_later_enqueue_args(method, enqueue_args = {}, *args)
8
+ enqueue_args = enqueue_args.dup
9
+ # support procs/methods as enqueue arguments
10
+ enqueue_args.each do |k,v|
11
+ if v.respond_to?(:call)
12
+ enqueue_args[k] = v.call(self)
13
+ end
14
+ end
15
+
16
+ no_delay = enqueue_args.delete(:no_delay)
17
+ if !no_delay
18
+ # delay queuing up the job in another database until the results of the current
19
+ # transaction are visible
20
+ connection = self.class.connection if self.class.respond_to?(:connection)
21
+ connection ||= self.connection if respond_to?(:connection)
22
+ connection ||= ActiveRecord::Base.connection
23
+
24
+ if (Delayed::Job != Delayed::Backend::ActiveRecord::Job || connection != Delayed::Job.connection)
25
+ connection.after_transaction_commit do
26
+ Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args), enqueue_args)
27
+ end
28
+ return nil
29
+ end
30
+ end
31
+
32
+ result = Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args), enqueue_args)
33
+ result = nil unless no_delay
34
+ result
35
+ end
36
+
37
+ def send_later_with_queue(method, queue, *args)
38
+ send_later_enqueue_args(method, { :queue => queue }, *args)
39
+ end
40
+
41
+ def send_at(time, method, *args)
42
+ send_later_enqueue_args(method,
43
+ { :run_at => time }, *args)
44
+ end
45
+
46
+ def send_at_with_queue(time, method, queue, *args)
47
+ send_later_enqueue_args(method,
48
+ { :run_at => time, :queue => queue },
49
+ *args)
50
+ end
51
+
52
+ def send_later_unless_in_job(method, *args)
53
+ if Delayed::Job.in_delayed_job?
54
+ send(method, *args)
55
+ else
56
+ send_later(method, *args)
57
+ end
58
+ nil # can't rely on the type of return value, so return nothing
59
+ end
60
+
61
+ def send_later_if_production(*args)
62
+ if Rails.env.production?
63
+ send_later(*args)
64
+ else
65
+ send(*args)
66
+ end
67
+ end
68
+
69
+ def send_later_if_production_enqueue_args(method, enqueue_args, *args)
70
+ if Rails.env.production?
71
+ send_later_enqueue_args(method, enqueue_args, *args)
72
+ else
73
+ send(method, *args)
74
+ end
75
+ end
76
+
77
+ def send_now_or_later(_when, *args)
78
+ if _when == :now
79
+ send(*args)
80
+ else
81
+ send_later(*args)
82
+ end
83
+ end
84
+
85
+ def send_now_or_later_if_production(_when, *args)
86
+ if _when == :now
87
+ send(*args)
88
+ else
89
+ send_later_if_production(*args)
90
+ end
91
+ end
92
+
93
+ module ClassMethods
94
+ def add_send_later_methods(method, enqueue_args={}, default_async=false)
95
+ aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
96
+
97
+ with_method, without_method = "#{aliased_method}_with_send_later#{punctuation}", "#{aliased_method}_without_send_later#{punctuation}"
98
+
99
+ define_method(with_method) do |*args|
100
+ send_later_enqueue_args(without_method, enqueue_args, *args)
101
+ end
102
+ alias_method without_method, method
103
+
104
+ if default_async
105
+ alias_method method, with_method
106
+ case
107
+ when public_method_defined?(without_method)
108
+ public method
109
+ when protected_method_defined?(without_method)
110
+ protected method
111
+ when private_method_defined?(without_method)
112
+ private method
113
+ end
114
+ end
115
+ end
116
+
117
+ def handle_asynchronously(method, enqueue_args={})
118
+ add_send_later_methods(method, enqueue_args, true)
119
+ end
120
+
121
+ def handle_asynchronously_with_queue(method, queue)
122
+ add_send_later_methods(method, {:queue => queue}, true)
123
+ end
124
+
125
+ def handle_asynchronously_if_production(method, enqueue_args={})
126
+ add_send_later_methods(method, enqueue_args, Rails.env.production?)
127
+ end
128
+ end
129
+ end
130
+ end
@@ -0,0 +1,42 @@
1
+ module Delayed
2
+ class PerformableMethod < Struct.new(:object, :method, :args)
3
+ def initialize(object, method, args = [])
4
+ raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method)
5
+
6
+ self.object = object
7
+ self.args = args
8
+ self.method = method.to_sym
9
+ end
10
+
11
+ def display_name
12
+ if object.is_a?(Module)
13
+ "#{object}.#{method}"
14
+ else
15
+ "#{object.class}##{method}"
16
+ end
17
+ end
18
+ alias_method :tag, :display_name
19
+
20
+ def perform
21
+ object.send(method, *args)
22
+ end
23
+
24
+ def deep_de_ar_ize(arg)
25
+ case arg
26
+ when Hash
27
+ "{#{arg.map { |k, v| "#{deep_de_ar_ize(k)} => #{deep_de_ar_ize(v)}" }.join(', ')}}"
28
+ when Array
29
+ "[#{arg.map { |a| deep_de_ar_ize(a) }.join(', ')}]"
30
+ when ActiveRecord::Base
31
+ "#{arg.class}.find(#{arg.id})"
32
+ else
33
+ arg.inspect
34
+ end
35
+ end
36
+
37
+ def full_name
38
+ obj_name = object.is_a?(ActiveRecord::Base) ? "#{object.class}.find(#{object.id}).#{method}" : display_name
39
+ "#{obj_name}(#{args.map { |a| deep_de_ar_ize(a) }.join(', ')})"
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,81 @@
1
+ require 'rufus-scheduler'
2
+
3
+ module Delayed
4
+ class Periodic
5
+ attr_reader :name, :cron
6
+
7
+ yaml_as "tag:ruby.yaml.org,2002:Delayed::Periodic"
8
+
9
+ def to_yaml(opts = {})
10
+ YAML.quick_emit(self.object_id, opts) { |out| out.scalar(taguri, @name) }
11
+ end
12
+
13
+ def self.yaml_new(klass, tag, val)
14
+ self.scheduled[val] || raise(NameError, "job #{val} is no longer scheduled")
15
+ end
16
+
17
+ cattr_accessor :scheduled, :overrides
18
+ self.scheduled = {}
19
+ self.overrides = {}
20
+
21
+ def self.add_overrides(overrides)
22
+ overrides.each do |name, cron_line|
23
+ # throws error if the line is malformed
24
+ Rufus::CronLine.new(cron_line)
25
+ end
26
+ self.overrides.merge!(overrides)
27
+ end
28
+
29
+ STRAND = 'periodic scheduling'
30
+
31
+ def self.cron(job_name, cron_line, job_args = {}, &block)
32
+ raise ArgumentError, "job #{job_name} already scheduled!" if self.scheduled[job_name]
33
+ cron_line = overrides[job_name] || cron_line
34
+ self.scheduled[job_name] = self.new(job_name, cron_line, job_args, block)
35
+ end
36
+
37
+ def self.audit_queue
38
+ # we used to queue up a job in a strand here, and perform the audit inside that job
39
+ # however, now that we're using singletons for scheduling periodic jobs,
40
+ # it's fine to just do the audit in-line here without risk of creating duplicates
41
+ perform_audit!
42
+ end
43
+
44
+ # make sure all periodic jobs are scheduled for their next run in the job queue
45
+ # this auditing should run on the strand
46
+ def self.perform_audit!
47
+ self.scheduled.each { |name, periodic| periodic.enqueue }
48
+ end
49
+
50
+ def initialize(name, cron_line, job_args, block)
51
+ @name = name
52
+ @cron = Rufus::CronLine.new(cron_line)
53
+ @job_args = { :priority => Delayed::LOW_PRIORITY }.merge(job_args.symbolize_keys)
54
+ @block = block
55
+ end
56
+
57
+ def enqueue
58
+ Delayed::Job.enqueue(self, @job_args.merge(:max_attempts => 1, :run_at => @cron.next_time(Delayed::Periodic.now), :singleton => tag))
59
+ end
60
+
61
+ def perform
62
+ @block.call()
63
+ ensure
64
+ begin
65
+ enqueue
66
+ rescue
67
+ # double fail! the auditor will have to catch this.
68
+ Rails.logger.error "Failure enqueueing periodic job! #{@name} #{$!.inspect}"
69
+ end
70
+ end
71
+
72
+ def tag
73
+ "periodic: #{@name}"
74
+ end
75
+ alias_method :display_name, :tag
76
+
77
+ def self.now
78
+ Time.now
79
+ end
80
+ end
81
+ end