canvas-jobs 0.9.0

data/lib/delayed/backend/redis/set_running.lua

@@ -0,0 +1,5 @@
+local job_id = unpack(ARGV)
+local locked_at, queue, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'locked_at', 'queue', 'strand'))
+
+remove_from_queues(job_id, queue, strand)
+redis.call('ZADD', Keys.running_jobs(), locked_at, job_id)

data/lib/delayed/backend/redis/tickle_strand.lua

@@ -0,0 +1,2 @@
+local job_id, strand, now = unpack(ARGV)
+tickle_strand(job_id, strand, now)

data/lib/delayed/batch.rb

@@ -0,0 +1,56 @@
+module Delayed
+  module Batch
+    class PerformableBatch < Struct.new(:mode, :items)
+      def initialize(mode, items)
+        raise "unsupported mode" unless mode == :serial
+        self.mode   = mode
+        self.items  = items
+      end
+
+      def display_name
+        "Delayed::Batch.#{mode}"
+      end
+      alias_method :tag, :display_name
+      alias_method :full_name, :display_name
+
+      def perform
+        raise "can't perform a batch directly"
+      end
+
+      def jobs
+        items.map { |opts| Delayed::Job.new(opts) }
+      end
+    end
+
+    class << self
+      def serial_batch(opts = {})
+        prepare_batches(:serial, opts){ yield }
+      end
+
+      private
+      def prepare_batches(mode, opts)
+        raise "nested batching is not supported" if Delayed::Job.batches
+        Delayed::Job.batches = Hash.new { |h,k| h[k] = [] }
+        batch_enqueue_args = [:queue]
+        batch_enqueue_args << :priority unless opts[:priority]
+        Delayed::Job.batch_enqueue_args = batch_enqueue_args
+        yield
+      ensure
+        batches = Delayed::Job.batches
+        Delayed::Job.batches = nil
+        batch_args = opts.slice(:priority)
+        batches.each do |enqueue_args, batch|
+          if batch.size == 0
+            next
+          elsif batch.size == 1
+            args = batch.first.merge(batch_args)
+            payload_object = args.delete(:payload_object)
+            Delayed::Job.enqueue(payload_object, args)
+          else
+            Delayed::Job.enqueue(Delayed::Batch::PerformableBatch.new(mode, batch), enqueue_args.merge(batch_args))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/engine.rb

@@ -0,0 +1,4 @@
+module Delayed
+  class Engine < ::Rails::Engine
+  end
+end

data/lib/delayed/job_tracking.rb

@@ -0,0 +1,31 @@
+module Delayed
+# Used when a block of code wants to track what jobs are created,
+# for instance in tests.
+# Delayed::Job.track_jobs { ...block... } returns a JobTracking object
+# Right now this just tracks created jobs, it could be expanded to track a
+# lot more about what's going on in Delayed Jobs as it's needed.
+class JobTracking < Struct.new(:created)
+  def self.track
+    @current_tracking = self.new
+    yield
+    tracking = @current_tracking
+    @current_tracking = nil
+    tracking
+  end
+
+  def self.job_created(job)
+    @current_tracking.try(:job_created, job)
+  end
+
+  def job_created(job)
+    return unless job
+    @lock.synchronize { self.created << job }
+  end
+
+  def initialize
+    super
+    self.created = []
+    @lock = Mutex.new
+  end
+end
+end

data/lib/delayed/lifecycle.rb

@@ -0,0 +1,83 @@
+module Delayed
+  class InvalidCallback < Exception; end
+
+  class Lifecycle
+    EVENTS = {
+      :perform    => [:worker, :job],
+      :pop        => [:worker],
+    }
+
+    def initialize
+      reset!
+    end
+
+    def reset!
+      @callbacks = EVENTS.keys.inject({}) { |hash, e| hash[e] = Callback.new; hash }
+    end
+
+    def before(event, &block)
+      add(:before, event, &block)
+    end
+
+    def after(event, &block)
+      add(:after, event, &block)
+    end
+
+    def around(event, &block)
+      add(:around, event, &block)
+    end
+
+    def run_callbacks(event, *args, &block)
+      missing_callback(event) unless @callbacks.has_key?(event)
+
+      unless EVENTS[event].size == args.size
+        raise ArgumentError, "Callback #{event} expects #{EVENTS[event].size} parameter(s): #{EVENTS[event].join(', ')}"
+      end
+
+      @callbacks[event].execute(*args, &block)
+    end
+
+    private
+
+      def add(type, event, &block)
+        missing_callback(event) unless @callbacks.has_key?(event)
+
+        @callbacks[event].add(type, &block)
+      end
+
+      def missing_callback(event)
+        raise InvalidCallback, "Unknown callback event: #{event}"
+      end
+  end
+
+  class Callback
+    def initialize
+      @before = []
+      @after = []
+
+      # Identity proc. Avoids special cases when there is no existing around chain.
+      @around = lambda { |*args, &block| block.call(*args) }
+    end
+
+    def execute(*args, &block)
+      @before.each { |c| c.call(*args) }
+      result = @around.call(*args, &block)
+      @after.each { |c| c.call(*args) }
+      result
+    end
+
+    def add(type, &callback)
+      case type
+      when :before
+        @before << callback
+      when :after
+        @after << callback
+      when :around
+        chain = @around # use a local variable so that the current chain is closed over in the following lambda
+        @around = lambda { |*a, &block| chain.call(*a) { |*b| callback.call(*b, &block) } }
+      else
+        raise InvalidCallback, "Invalid callback type: #{type}"
+      end
+    end
+  end
+end

data/lib/delayed/message_sending.rb

@@ -0,0 +1,130 @@
+module Delayed
+  module MessageSending
+    def send_later(method, *args)
+      send_later_enqueue_args(method, {}, *args)
+    end
+
+    def send_later_enqueue_args(method, enqueue_args = {}, *args)
+      enqueue_args = enqueue_args.dup
+      # support procs/methods as enqueue arguments
+      enqueue_args.each do |k,v|
+        if v.respond_to?(:call)
+          enqueue_args[k] = v.call(self)
+        end
+      end
+
+      no_delay = enqueue_args.delete(:no_delay)
+      if !no_delay
+        # delay queuing up the job in another database until the results of the current
+        # transaction are visible
+        connection = self.class.connection if self.class.respond_to?(:connection)
+        connection ||= self.connection if respond_to?(:connection)
+        connection ||= ActiveRecord::Base.connection
+
+        if (Delayed::Job != Delayed::Backend::ActiveRecord::Job || connection != Delayed::Job.connection)
+          connection.after_transaction_commit do
+            Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args), enqueue_args)
+          end
+          return nil
+        end
+      end
+
+      result = Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args), enqueue_args)
+      result = nil unless no_delay
+      result
+    end
+
+    def send_later_with_queue(method, queue, *args)
+      send_later_enqueue_args(method, { :queue => queue }, *args)
+    end
+
+    def send_at(time, method, *args)
+      send_later_enqueue_args(method,
+                          { :run_at => time }, *args)
+    end
+
+    def send_at_with_queue(time, method, queue, *args)
+      send_later_enqueue_args(method,
+                          { :run_at => time, :queue => queue },
+                          *args)
+    end
+
+    def send_later_unless_in_job(method, *args)
+      if Delayed::Job.in_delayed_job?
+        send(method, *args)
+      else
+        send_later(method, *args)
+      end
+      nil # can't rely on the type of return value, so return nothing
+    end
+
+    def send_later_if_production(*args)
+      if Rails.env.production?
+        send_later(*args)
+      else
+        send(*args)
+      end
+    end
+
+    def send_later_if_production_enqueue_args(method, enqueue_args, *args)
+      if Rails.env.production?
+        send_later_enqueue_args(method, enqueue_args, *args)
+      else
+        send(method, *args)
+      end
+    end
+
+    def send_now_or_later(_when, *args)
+      if _when == :now
+        send(*args)
+      else
+        send_later(*args)
+      end
+    end
+
+    def send_now_or_later_if_production(_when, *args)
+      if _when == :now
+        send(*args)
+      else
+        send_later_if_production(*args)
+      end
+    end
+
+    module ClassMethods
+      def add_send_later_methods(method, enqueue_args={}, default_async=false)
+        aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
+
+        with_method, without_method = "#{aliased_method}_with_send_later#{punctuation}", "#{aliased_method}_without_send_later#{punctuation}"
+
+        define_method(with_method) do |*args|
+          send_later_enqueue_args(without_method, enqueue_args, *args)
+        end
+        alias_method without_method, method
+
+        if default_async
+          alias_method method, with_method
+          case
+            when public_method_defined?(without_method)
+              public method
+            when protected_method_defined?(without_method)
+              protected method
+            when private_method_defined?(without_method)
+              private method
+          end
+        end
+      end
+
+      def handle_asynchronously(method, enqueue_args={})
+        add_send_later_methods(method, enqueue_args, true)
+      end
+
+      def handle_asynchronously_with_queue(method, queue)
+        add_send_later_methods(method, {:queue => queue}, true)
+      end
+
+      def handle_asynchronously_if_production(method, enqueue_args={})
+        add_send_later_methods(method, enqueue_args, Rails.env.production?)
+      end
+    end
+  end
+end

data/lib/delayed/performable_method.rb

@@ -0,0 +1,42 @@
+module Delayed
+  class PerformableMethod < Struct.new(:object, :method, :args)
+    def initialize(object, method, args = [])
+      raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method)
+
+      self.object = object
+      self.args   = args
+      self.method = method.to_sym
+    end
+
+    def display_name
+      if object.is_a?(Module)
+        "#{object}.#{method}"
+      else
+        "#{object.class}##{method}"
+      end
+    end
+    alias_method :tag, :display_name
+
+    def perform
+      object.send(method, *args)
+    end
+
+    def deep_de_ar_ize(arg)
+      case arg
+      when Hash
+        "{#{arg.map { |k, v| "#{deep_de_ar_ize(k)} => #{deep_de_ar_ize(v)}" }.join(', ')}}"
+      when Array
+        "[#{arg.map { |a| deep_de_ar_ize(a) }.join(', ')}]"
+      when ActiveRecord::Base
+        "#{arg.class}.find(#{arg.id})"
+      else
+        arg.inspect
+      end
+    end
+
+    def full_name
+      obj_name = object.is_a?(ActiveRecord::Base) ? "#{object.class}.find(#{object.id}).#{method}" : display_name
+      "#{obj_name}(#{args.map { |a| deep_de_ar_ize(a) }.join(', ')})"
+    end
+  end
+end

data/lib/delayed/periodic.rb

@@ -0,0 +1,81 @@
+require 'rufus-scheduler'
+
+module Delayed
+class Periodic
+  attr_reader :name, :cron
+
+  yaml_as "tag:ruby.yaml.org,2002:Delayed::Periodic"
+
+  def to_yaml(opts = {})
+    YAML.quick_emit(self.object_id, opts) { |out| out.scalar(taguri, @name) }
+  end
+
+  def self.yaml_new(klass, tag, val)
+    self.scheduled[val] || raise(NameError, "job #{val} is no longer scheduled")
+  end
+
+  cattr_accessor :scheduled, :overrides
+  self.scheduled = {}
+  self.overrides = {}
+
+  def self.add_overrides(overrides)
+    overrides.each do |name, cron_line|
+      # throws error if the line is malformed
+      Rufus::CronLine.new(cron_line)
+    end
+    self.overrides.merge!(overrides)
+  end
+
+  STRAND = 'periodic scheduling'
+
+  def self.cron(job_name, cron_line, job_args = {}, &block)
+    raise ArgumentError, "job #{job_name} already scheduled!" if self.scheduled[job_name]
+    cron_line = overrides[job_name] || cron_line
+    self.scheduled[job_name] = self.new(job_name, cron_line, job_args, block)
+  end
+
+  def self.audit_queue
+    # we used to queue up a job in a strand here, and perform the audit inside that job
+    # however, now that we're using singletons for scheduling periodic jobs,
+    # it's fine to just do the audit in-line here without risk of creating duplicates
+    perform_audit!
+  end
+
+  # make sure all periodic jobs are scheduled for their next run in the job queue
+  # this auditing should run on the strand
+  def self.perform_audit!
+    self.scheduled.each { |name, periodic| periodic.enqueue }
+  end
+
+  def initialize(name, cron_line, job_args, block)
+    @name = name
+    @cron = Rufus::CronLine.new(cron_line)
+    @job_args = { :priority => Delayed::LOW_PRIORITY }.merge(job_args.symbolize_keys)
+    @block = block
+  end
+
+  def enqueue
+    Delayed::Job.enqueue(self, @job_args.merge(:max_attempts => 1, :run_at => @cron.next_time(Delayed::Periodic.now), :singleton => tag))
+  end
+
+  def perform
+    @block.call()
+  ensure
+    begin
+      enqueue
+    rescue
+      # double fail! the auditor will have to catch this.
+      Rails.logger.error "Failure enqueueing periodic job! #{@name} #{$!.inspect}"
+    end
+  end
+
+  def tag
+    "periodic: #{@name}"
+  end
+  alias_method :display_name, :tag
+
+  def self.now
+    Time.now
+  end
+end
+end