inst-jobs 0.11.0

data/lib/delayed/daemon.rb

@@ -0,0 +1,103 @@
+require 'fileutils'
+
+module Delayed
+# Daemon controls the parent proces that runs the Pool and monitors the Worker processes.
+class Daemon
+  attr_reader :pid_folder
+
+  def initialize(pid_folder)
+    @pid_folder = pid_folder
+  end
+
+  def status(print: true, pid: self.pid)
+    alive = pid && (Process.kill(0, pid) rescue false) && :running
+    alive ||= :draining if pid && Process.kill(0, -pid) rescue false
+    if alive
+      puts "Delayed jobs #{alive}, pool PID: #{pid}" if print
+    else
+      puts "No delayed jobs pool running" if print && print != :alive
+    end
+    alive
+  end
+
+  def daemonize!
+    FileUtils.mkdir_p(pid_folder)
+    puts "Daemonizing..."
+
+    exit if fork
+    Process.setsid
+    exit if fork
+    Process.setpgrp
+
+    @daemon = true
+    lock_file = File.open(pid_file, 'wb')
+    # someone else is already running; just exit
+    unless lock_file.flock(File::LOCK_EX | File::LOCK_NB)
+      exit
+    end
+    at_exit { lock_file.flock(File::LOCK_UN) }
+    lock_file.puts(Process.pid.to_s)
+    lock_file.flush
+
+    # if we blow up so badly that we can't syslog the error, try to send
+    # it somewhere useful
+    last_ditch_logfile = Settings.last_ditch_logfile || "log/delayed_job.log"
+    if last_ditch_logfile[0] != '|'
+      last_ditch_logfile = Settings.expand_rails_path(last_ditch_logfile)
+    end
+    STDIN.reopen("/dev/null")
+    STDOUT.reopen(open(last_ditch_logfile, 'a'))
+    STDERR.reopen(STDOUT)
+    STDOUT.sync = STDERR.sync = true
+  end
+
+  # stop the currently running daemon (not this current process, the one in the pid_file)
+  def stop(kill: false, pid: self.pid)
+    alive = status(pid: pid, print: false)
+    if alive == :running || (kill && alive == :draining)
+      puts "Stopping pool #{pid}..."
+      signal = 'INT'
+      if kill
+        pid = -pid # send to the whole group
+        if kill == 9
+          signal = 'KILL'
+        else
+          signal = 'TERM'
+        end
+      end
+      begin
+        Process.kill(signal, pid)
+      rescue Errno::ESRCH
+        # ignore if the pid no longer exists
+      end
+      wait(kill)
+    else
+      status
+    end
+  end
+
+  def wait(kill)
+    if kill
+      sleep(0.5) while status(pid: pid, print: false)
+    else
+      sleep(0.5) while status(pid: pid, print: false) == :running
+    end
+  end
+
+  def pid_file
+    File.join(pid_folder, 'delayed_jobs_pool.pid')
+  end
+
+  def pid
+    if File.file?(pid_file)
+      pid = File.read(pid_file).to_i
+      pid = nil unless pid > 0
+    end
+    pid
+  end
+
+  def daemonized?
+    !!@daemon
+  end
+end
+end

data/lib/delayed/engine.rb

@@ -0,0 +1,4 @@
+module Delayed
+  class Engine < ::Rails::Engine
+  end
+end

data/lib/delayed/job_tracking.rb

@@ -0,0 +1,31 @@
+module Delayed
+# Used when a block of code wants to track what jobs are created,
+# for instance in tests.
+# Delayed::Job.track_jobs { ...block... } returns a JobTracking object
+# Right now this just tracks created jobs, it could be expanded to track a
+# lot more about what's going on in Delayed Jobs as it's needed.
+class JobTracking < Struct.new(:created)
+  def self.track
+    @current_tracking = self.new
+    yield
+    tracking = @current_tracking
+    @current_tracking = nil
+    tracking
+  end
+
+  def self.job_created(job)
+    @current_tracking.try(:job_created, job)
+  end
+
+  def job_created(job)
+    return unless job
+    @lock.synchronize { self.created << job }
+  end
+
+  def initialize
+    super
+    self.created = []
+    @lock = Mutex.new
+  end
+end
+end

data/lib/delayed/lifecycle.rb

@@ -0,0 +1,90 @@
+module Delayed
+  class InvalidCallback < Exception; end
+
+  class Lifecycle
+    EVENTS = {
+      :error            => [:worker, :job, :exception],
+      :exceptional_exit => [:worker, :exception],
+      :execute          => [:worker],
+      :invoke_job       => [:job],
+      :loop             => [:worker],
+      :perform          => [:worker, :job],
+      :pop              => [:worker],
+      :work_queue_pop   => [:work_queue],
+    }
+
+    def initialize
+      reset!
+    end
+
+    def reset!
+      @callbacks = EVENTS.keys.inject({}) { |hash, e| hash[e] = Callback.new; hash }
+      Delayed::Worker.plugins.each { |plugin| plugin.reset! }
+    end
+
+    def before(event, &block)
+      add(:before, event, &block)
+    end
+
+    def after(event, &block)
+      add(:after, event, &block)
+    end
+
+    def around(event, &block)
+      add(:around, event, &block)
+    end
+
+    def run_callbacks(event, *args, &block)
+      missing_callback(event) unless @callbacks.has_key?(event)
+
+      unless EVENTS[event].size == args.size
+        raise ArgumentError, "Callback #{event} expects #{EVENTS[event].size} parameter(s): #{EVENTS[event].join(', ')}"
+      end
+
+      @callbacks[event].execute(*args, &block)
+    end
+
+    private
+
+      def add(type, event, &block)
+        missing_callback(event) unless @callbacks.has_key?(event)
+
+        @callbacks[event].add(type, &block)
+      end
+
+      def missing_callback(event)
+        raise InvalidCallback, "Unknown callback event: #{event}"
+      end
+  end
+
+  class Callback
+    def initialize
+      @before = []
+      @after = []
+
+      # Identity proc. Avoids special cases when there is no existing around chain.
+      @around = lambda { |*args, &block| block.call(*args) }
+    end
+
+    def execute(*args, &block)
+      @before.each { |c| c.call(*args) }
+      result = @around.call(*args, &block)
+      @after.each { |c| c.call(*args) }
+      result
+    end
+
+    def add(type, &callback)
+      case type
+      when :before
+        @before << callback
+      when :after
+        @after << callback
+      when :around
+        chain = @around # use a local variable so that the current chain is closed over in the following lambda
+        @around = lambda { |*a, &block| chain.call(*a) { |*b| callback.call(*b, &block) } }
+      else
+        raise InvalidCallback, "Invalid callback type: #{type}"
+      end
+    end
+  end
+end

data/lib/delayed/log_tailer.rb

@@ -0,0 +1,22 @@
+module Delayed
+class LogTailer
+  def run
+    if Rails.logger.respond_to?(:log_path)
+      log_path = Rails.logger.log_path
+    elsif Rails.logger.instance_variable_get('@logdev').try(:instance_variable_get, '@dev').try(:path)
+      log_path = Rails.logger.instance_variable_get('@logdev').instance_variable_get('@dev').path
+    else
+      return
+    end
+    Rails.logger.auto_flushing = true if Rails.logger.respond_to?(:auto_flushing=)
+    Thread.new do
+      f = File.open(log_path, 'r')
+      f.seek(0, IO::SEEK_END)
+      loop do
+        content = f.read
+        content.present? ? STDOUT.print(content) : sleep(0.5)
+      end
+    end
+  end
+end
+end

data/lib/delayed/message_sending.rb

@@ -0,0 +1,134 @@
+module Delayed
+  module MessageSending
+    def send_later(method, *args)
+      send_later_enqueue_args(method, {}, *args)
+    end
+
+    def send_later_enqueue_args(method, enqueue_args = {}, *args)
+      enqueue_args = enqueue_args.dup
+      # support procs/methods as enqueue arguments
+      enqueue_args.each do |k,v|
+        if v.respond_to?(:call)
+          enqueue_args[k] = v.call(self)
+        end
+      end
+
+      no_delay = enqueue_args.delete(:no_delay)
+      on_failure = enqueue_args.delete(:on_failure)
+      on_permanent_failure = enqueue_args.delete(:on_permanent_failure)
+      if !no_delay
+        # delay queuing up the job in another database until the results of the current
+        # transaction are visible
+        connection = self.class.connection if self.class.respond_to?(:connection)
+        connection ||= self.connection if respond_to?(:connection)
+        connection ||= ActiveRecord::Base.connection
+
+        if (Delayed::Job != Delayed::Backend::ActiveRecord::Job || connection != Delayed::Job.connection)
+          connection.after_transaction_commit do
+            Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args,
+                                                                on_failure, on_permanent_failure), enqueue_args)
+          end
+          return nil
+        end
+      end
+
+      result = Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args,
+                                                                   on_failure, on_permanent_failure), enqueue_args)
+      result = nil unless no_delay
+      result
+    end
+
+    def send_later_with_queue(method, queue, *args)
+      send_later_enqueue_args(method, { :queue => queue }, *args)
+    end
+
+    def send_at(time, method, *args)
+      send_later_enqueue_args(method,
+                          { :run_at => time }, *args)
+    end
+
+    def send_at_with_queue(time, method, queue, *args)
+      send_later_enqueue_args(method,
+                          { :run_at => time, :queue => queue },
+                          *args)
+    end
+
+    def send_later_unless_in_job(method, *args)
+      if Delayed::Job.in_delayed_job?
+        send(method, *args)
+      else
+        send_later(method, *args)
+      end
+      nil # can't rely on the type of return value, so return nothing
+    end
+
+    def send_later_if_production(*args)
+      if Rails.env.production?
+        send_later(*args)
+      else
+        send(*args)
+      end
+    end
+
+    def send_later_if_production_enqueue_args(method, enqueue_args, *args)
+      if Rails.env.production?
+        send_later_enqueue_args(method, enqueue_args, *args)
+      else
+        send(method, *args)
+      end
+    end
+
+    def send_now_or_later(_when, *args)
+      if _when == :now
+        send(*args)
+      else
+        send_later(*args)
+      end
+    end
+
+    def send_now_or_later_if_production(_when, *args)
+      if _when == :now
+        send(*args)
+      else
+        send_later_if_production(*args)
+      end
+    end
+
+    module ClassMethods
+      def add_send_later_methods(method, enqueue_args={}, default_async=false)
+        aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
+
+        with_method, without_method = "#{aliased_method}_with_send_later#{punctuation}", "#{aliased_method}_without_send_later#{punctuation}"
+
+        define_method(with_method) do |*args|
+          send_later_enqueue_args(without_method, enqueue_args, *args)
+        end
+        alias_method without_method, method
+
+        if default_async
+          alias_method method, with_method
+          case
+            when public_method_defined?(without_method)
+              public method
+            when protected_method_defined?(without_method)
+              protected method
+            when private_method_defined?(without_method)
+              private method
+          end
+        end
+      end
+
+      def handle_asynchronously(method, enqueue_args={})
+        add_send_later_methods(method, enqueue_args, true)
+      end
+
+      def handle_asynchronously_with_queue(method, queue)
+        add_send_later_methods(method, {:queue => queue}, true)
+      end
+
+      def handle_asynchronously_if_production(method, enqueue_args={})
+        add_send_later_methods(method, enqueue_args, Rails.env.production?)
+      end
+    end
+  end
+end

data/lib/delayed/performable_method.rb

@@ -0,0 +1,52 @@
+module Delayed
+  class PerformableMethod < Struct.new(:object, :method, :args, :fail_cb, :permanent_fail_cb)
+    def initialize(object, method, args = [], fail_cb = nil, permanent_fail_cb = nil)
+      raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method, true)
+
+      self.object = object
+      self.args   = args
+      self.method = method.to_sym
+      self.fail_cb           = fail_cb
+      self.permanent_fail_cb = permanent_fail_cb
+    end
+
+    def display_name
+      if object.is_a?(Module)
+        "#{object}.#{method}"
+      else
+        "#{object.class}##{method}"
+      end
+    end
+    alias_method :tag, :display_name
+
+    def perform
+      object.send(method, *args)
+    end
+
+    def on_failure(error)
+      object.send(fail_cb, error) if fail_cb
+    end
+
+    def on_permanent_failure(error)
+      object.send(permanent_fail_cb, error) if permanent_fail_cb
+    end
+
+    def deep_de_ar_ize(arg)
+      case arg
+      when Hash
+        "{#{arg.map { |k, v| "#{deep_de_ar_ize(k)} => #{deep_de_ar_ize(v)}" }.join(', ')}}"
+      when Array
+        "[#{arg.map { |a| deep_de_ar_ize(a) }.join(', ')}]"
+      when ActiveRecord::Base
+        "#{arg.class}.find(#{arg.id})"
+      else
+        arg.inspect
+      end
+    end
+
+    def full_name
+      obj_name = object.is_a?(ActiveRecord::Base) ? "#{object.class}.find(#{object.id}).#{method}" : display_name
+      "#{obj_name}(#{args.map { |a| deep_de_ar_ize(a) }.join(', ')})"
+    end
+  end
+end

data/lib/delayed/periodic.rb

@@ -0,0 +1,85 @@
+require 'rufus/scheduler'
+
+module Delayed
+class Periodic
+  attr_reader :name, :cron
+
+  yaml_as "tag:ruby.yaml.org,2002:Delayed::Periodic"
+
+  def to_yaml(opts = {})
+    YAML.quick_emit(self.object_id, opts) { |out| out.scalar(taguri, @name) }
+  end
+
+  def encode_with(coder)
+    coder.scalar("!ruby/Delayed::Periodic", @name)
+  end
+
+  def self.yaml_new(klass, tag, val)
+    self.scheduled[val] || raise(NameError, "job #{val} is no longer scheduled")
+  end
+
+  cattr_accessor :scheduled, :overrides
+  self.scheduled = {}
+  self.overrides = {}
+
+  def self.add_overrides(overrides)
+    overrides.each do |name, cron_line|
+      # throws error if the line is malformed
+      Rufus::Scheduler::CronLine.new(cron_line)
+    end
+    self.overrides.merge!(overrides)
+  end
+
+  STRAND = 'periodic scheduling'
+
+  def self.cron(job_name, cron_line, job_args = {}, &block)
+    raise ArgumentError, "job #{job_name} already scheduled!" if self.scheduled[job_name]
+    cron_line = overrides[job_name] || cron_line
+    self.scheduled[job_name] = self.new(job_name, cron_line, job_args, block)
+  end
+
+  def self.audit_queue
+    # we used to queue up a job in a strand here, and perform the audit inside that job
+    # however, now that we're using singletons for scheduling periodic jobs,
+    # it's fine to just do the audit in-line here without risk of creating duplicates
+    perform_audit!
+  end
+
+  # make sure all periodic jobs are scheduled for their next run in the job queue
+  # this auditing should run on the strand
+  def self.perform_audit!
+    self.scheduled.each { |name, periodic| periodic.enqueue }
+  end
+
+  def initialize(name, cron_line, job_args, block)
+    @name = name
+    @cron = Rufus::Scheduler::CronLine.new(cron_line)
+    @job_args = { :priority => Delayed::LOW_PRIORITY }.merge(job_args.symbolize_keys)
+    @block = block
+  end
+
+  def enqueue
+    Delayed::Job.enqueue(self, @job_args.merge(:max_attempts => 1, :run_at => @cron.next_time(Delayed::Periodic.now), :singleton => tag))
+  end
+
+  def perform
+    @block.call()
+  ensure
+    begin
+      enqueue
+    rescue
+      # double fail! the auditor will have to catch this.
+      Rails.logger.error "Failure enqueueing periodic job! #{@name} #{$!.inspect}"
+    end
+  end
+
+  def tag
+    "periodic: #{@name}"
+  end
+  alias_method :display_name, :tag
+
+  def self.now
+    Time.zone.now
+  end
+end
+end

data/lib/delayed/plugin.rb

@@ -0,0 +1,22 @@
+require 'active_support/core_ext/class/attribute'
+
+module Delayed
+  class Plugin
+    class_attribute :callback_block
+
+    def self.callbacks(&block)
+      self.callback_block = block
+    end
+
+    def self.inject!
+      unless @injected
+        self.callback_block.call(Delayed::Worker.lifecycle) if self.callback_block
+      end
+      @injected = true
+    end
+
+    def self.reset!
+      @injected = false
+    end
+  end
+end

data/lib/delayed/pool.rb

@@ -0,0 +1,161 @@
+module Delayed
+class Pool
+  mattr_accessor :on_fork
+  self.on_fork = ->{ }
+
+  attr_reader :workers
+
+  def initialize(*args)
+    if args.first.is_a?(Hash)
+      @config = args.first
+    else
+      warn "Calling Delayed::Pool.new directly is deprecated. Use `Delayed::CLI.new.run()` instead."
+    end
+    @workers = {}
+  end
+
+  def run
+    warn "Delayed::Pool#run is deprecated and will be removed. Use `Delayed::CLI.new.run()` instead."
+    Delayed::CLI.new.run()
+  end
+
+  def start
+    say "Started job master", :info
+    $0 = procname
+    # fork to handle unlocking (to prevent polluting the parent with worker objects)
+    unlock_pid = fork_with_reconnects do
+      unlock_orphaned_jobs
+    end
+    Process.wait unlock_pid
+
+    spawn_periodic_auditor
+    spawn_all_workers
+    say "Workers spawned"
+    join
+    say "Shutting down"
+  rescue Interrupt => e
+    say "Signal received, exiting", :info
+  rescue Exception => e
+    say "Job master died with error: #{e.inspect}\n#{e.backtrace.join("\n")}", :fatal
+    raise
+  end
+
+  protected
+
+  def procname
+    "delayed_jobs_pool#{Settings.pool_procname_suffix}"
+  end
+
+  def say(msg, level = :debug)
+    if defined?(Rails.logger) && Rails.logger
+      Rails.logger.send(level, "[#{Process.pid}]P #{msg}")
+    else
+      puts(msg)
+    end
+  end
+
+  def unlock_orphaned_jobs(worker = nil, pid = nil)
+    return if Settings.disable_automatic_orphan_unlocking
+
+    unlocked_jobs = Delayed::Job.unlock_orphaned_jobs(pid)
+    say "Unlocked #{unlocked_jobs} orphaned jobs" if unlocked_jobs > 0
+    ActiveRecord::Base.connection_handler.clear_all_connections! unless Rails.env.test?
+  end
+
+  def spawn_all_workers
+    ActiveRecord::Base.connection_handler.clear_all_connections!
+
+    if @config[:work_queue] == 'parent_process'
+      @work_queue = WorkQueue::ParentProcess.new
+      spawn_work_queue
+    end
+
+    @config[:workers].each do |worker_config|
+      (worker_config[:workers] || 1).times { spawn_worker(worker_config) }
+    end
+  end
+
+  def spawn_work_queue
+    parent_pid = Process.pid
+    pid = fork_with_reconnects do
+      $0 = "delayed_jobs_work_queue#{Settings.pool_procname_suffix}"
+      @work_queue.server(parent_pid: parent_pid).run
+    end
+    workers[pid] = :work_queue
+  end
+
+  def spawn_worker(worker_config)
+    if worker_config[:periodic]
+      return # backwards compat
+    else
+      worker_config[:parent_pid] = Process.pid
+      worker_config[:work_queue] = @work_queue.client if @work_queue
+      worker = Delayed::Worker.new(worker_config)
+    end
+
+    pid = fork_with_reconnects do
+      worker.start
+    end
+    workers[pid] = worker
+  end
+
+  # child processes need to reconnect so they don't accidentally share redis or
+  # db connections with the parent
+  def fork_with_reconnects
+    fork do
+      Pool.on_fork.()
+      Delayed::Job.reconnect!
+      yield
+    end
+  end
+
+  def spawn_periodic_auditor
+    return if Settings.disable_periodic_jobs
+
+    @periodic_thread = Thread.new do
+      # schedule the initial audit immediately on startup
+      schedule_periodic_audit
+      # initial sleep is randomized, for some staggering in the audit calls
+      # since job processors are usually all restarted at the same time
+      sleep(rand(15 * 60))
+      loop do
+        schedule_periodic_audit
+        sleep(15 * 60)
+      end
+    end
+  end
+
+  def schedule_periodic_audit
+    pid = fork_with_reconnects do
+      # we want to avoid db connections in the main pool process
+      $0 = "delayed_periodic_audit_scheduler"
+      Delayed::Periodic.audit_queue
+    end
+    workers[pid] = :periodic_audit
+  end
+
+  def join
+    loop do
+      child = Process.wait
+      if workers.include?(child)
+        worker = workers.delete(child)
+        case worker
+        when :periodic_audit
+          say "ran auditor: #{worker}"
+        when :work_queue
+          say "work queue exited, restarting", :info
+          spawn_work_queue
+        else
+          say "child exited: #{child}, restarting", :info
+          # fork to handle unlocking (to prevent polluting the parent with worker objects)
+          unlock_pid = fork_with_reconnects do
+            unlock_orphaned_jobs(worker, child)
+          end
+          Process.wait unlock_pid
+          spawn_worker(worker.config)
+        end
+      end
+    end
+  end
+end
+end