canvas-jobs 0.9.0

data/lib/delayed/pool.rb

@@ -0,0 +1,335 @@
+require 'optparse'
+require 'yaml'
+require 'fileutils'
+
+module Delayed
+class Pool
+  mattr_accessor :on_fork
+  self.on_fork = ->{ }
+
+  attr_reader :options, :workers
+
+  def initialize(args = ARGV)
+    @args = args
+    @workers = {}
+    @config = { :workers => [] }
+    @options = {
+      :config_file => expand_rails_path("config/delayed_jobs.yml"),
+      :pid_folder => expand_rails_path("tmp/pids"),
+      :tail_logs => true, # only in FG mode
+    }
+  end
+
+  def run
+    if GC.respond_to?(:copy_on_write_friendly=)
+      GC.copy_on_write_friendly = true
+    end
+
+    op = OptionParser.new do |opts|
+      opts.banner = "Usage #{$0} <command> <options>"
+      opts.separator %{\nWhere <command> is one of:
+  start      start the jobs daemon
+  stop       stop the jobs daemon
+  run        start and run in the foreground
+  restart    stop and then start the jobs daemon
+  status     show daemon status
+}
+
+      opts.separator "\n<options>"
+      opts.on("-c", "--config", "Use alternate config file (default #{options[:config_file]})") { |c| options[:config_file] = c }
+      opts.on("-p", "--pid", "Use alternate folder for PID files (default #{options[:pid_folder]})") { |p| options[:pid_folder] = p }
+      opts.on("--no-tail", "Don't tail the logs (only affects non-daemon mode)") { options[:tail_logs] = false }
+      opts.on("--with-prejudice", "When stopping, interrupt jobs in progress, instead of letting them drain") { options[:kill] ||= true }
+      opts.on("--with-extreme-prejudice", "When stopping, immediately kill jobs in progress, instead of letting them drain") { options[:kill] = 9 }
+      opts.on_tail("-h", "--help", "Show this message") { puts opts; exit }
+    end
+    op.parse!(@args)
+
+    read_config(options[:config_file])
+
+    command = @args.shift
+    case command
+    when 'start'
+      exit 1 if status(:alive) == :running
+      daemonize
+      start
+    when 'stop'
+      stop(options[:kill])
+    when 'run'
+      start
+    when 'status'
+      if status
+        exit 0
+      else
+        exit 1
+      end
+    when 'restart'
+      alive = status(false)
+      if alive == :running || (options[:kill] && alive == :draining)
+        stop(options[:kill])
+        if options[:kill]
+          sleep(0.5) while status(false)
+        else
+          sleep(0.5) while status(false) == :running
+        end
+      end
+      daemonize
+      start
+    when nil
+      puts op
+    else
+      raise("Unknown command: #{command.inspect}")
+    end
+  end
+
+  protected
+
+  def procname
+    "delayed_jobs_pool#{Settings.pool_procname_suffix}"
+  end
+
+  def start
+    load_rails
+    tail_rails_log unless @daemon
+
+    say "Started job master", :info
+    $0 = procname
+    apply_config
+
+    # fork to handle unlocking (to prevent polluting the parent with worker objects)
+    unlock_pid = fork_with_reconnects do
+      unlock_orphaned_jobs
+    end
+    Process.wait unlock_pid
+
+    spawn_periodic_auditor
+    spawn_all_workers
+    say "Workers spawned"
+    join
+    say "Shutting down"
+  rescue Interrupt => e
+    say "Signal received, exiting", :info
+  rescue Exception => e
+    say "Job master died with error: #{e.inspect}\n#{e.backtrace.join("\n")}", :fatal
+    raise
+  end
+
+  def say(msg, level = :debug)
+    if defined?(Rails.logger) && Rails.logger
+      Rails.logger.send(level, "[#{Process.pid}]P #{msg}")
+    else
+      puts(msg)
+    end
+  end
+
+  def load_rails
+    require(expand_rails_path("config/environment.rb"))
+    Dir.chdir(Rails.root)
+  end
+
+  def unlock_orphaned_jobs(worker = nil, pid = nil)
+    # don't bother trying to unlock jobs by process name if the name is overridden
+    return if @config.key?(:name)
+    return if @config[:disable_automatic_orphan_unlocking]
+    return if @config[:workers].any? { |worker_config| worker_config.key?(:name) || worker_config.key?('name') }
+
+    unlocked_jobs = Delayed::Job.unlock_orphaned_jobs(pid)
+    say "Unlocked #{unlocked_jobs} orphaned jobs" if unlocked_jobs > 0
+    ActiveRecord::Base.connection_handler.clear_all_connections! unless Rails.env.test?
+  end
+
+  def spawn_all_workers
+    ActiveRecord::Base.connection_handler.clear_all_connections!
+
+    @config[:workers].each do |worker_config|
+      worker_config = worker_config.with_indifferent_access
+      (worker_config[:workers] || 1).times { spawn_worker(@config.merge(worker_config)) }
+    end
+  end
+
+  def spawn_worker(worker_config)
+    if worker_config[:periodic]
+      return # backwards compat
+    else
+      worker_config[:parent_pid] = Process.pid
+      worker = Delayed::Worker.new(worker_config)
+    end
+
+    pid = fork_with_reconnects do
+      worker.start
+    end
+    workers[pid] = worker
+  end
+
+  # child processes need to reconnect so they don't accidentally share redis or
+  # db connections with the parent
+  def fork_with_reconnects
+    fork do
+      Pool.on_fork.()
+      Delayed::Job.reconnect!
+      yield
+    end
+  end
+
+  def spawn_periodic_auditor
+    return if @config[:disable_periodic_jobs]
+
+    @periodic_thread = Thread.new do
+      # schedule the initial audit immediately on startup
+      schedule_periodic_audit
+      # initial sleep is randomized, for some staggering in the audit calls
+      # since job processors are usually all restarted at the same time
+      sleep(rand(15 * 60))
+      loop do
+        schedule_periodic_audit
+        sleep(15 * 60)
+      end
+    end
+  end
+
+  def schedule_periodic_audit
+    pid = fork_with_reconnects do
+      # we want to avoid db connections in the main pool process
+      $0 = "delayed_periodic_audit_scheduler"
+      Delayed::Periodic.audit_queue
+    end
+    workers[pid] = :periodic_audit
+  end
+
+  def join
+    loop do
+      child = Process.wait
+      if child
+        worker = workers.delete(child)
+        if worker.is_a?(Symbol)
+          say "ran auditor: #{worker}"
+        else
+          say "child exited: #{child}, restarting", :info
+          # fork to handle unlocking (to prevent polluting the parent with worker objects)
+          unlock_pid = fork_with_reconnects do
+            unlock_orphaned_jobs(worker, child)
+          end
+          Process.wait unlock_pid
+          spawn_worker(worker.config)
+        end
+      end
+    end
+  end
+
+  def tail_rails_log
+    return if !@options[:tail_logs]
+    return if !Rails.logger.respond_to?(:log_path)
+    Rails.logger.auto_flushing = true if Rails.logger.respond_to?(:auto_flushing=)
+    Thread.new do
+      f = File.open(Rails.logger.log_path, 'r')
+      f.seek(0, IO::SEEK_END)
+      loop do
+        content = f.read
+        content.present? ? STDOUT.print(content) : sleep(0.5)
+      end
+    end
+  end
+
+  def daemonize
+    FileUtils.mkdir_p(pid_folder)
+    puts "Daemonizing..."
+
+    exit if fork
+    Process.setsid
+    exit if fork
+    Process.setpgrp
+
+    @daemon = true
+    File.open(pid_file, 'wb') { |f| f.write(Process.pid.to_s) }
+    # if we blow up so badly that we can't syslog the error, try to send
+    # it somewhere useful
+    last_ditch_logfile = self.last_ditch_logfile || "log/delayed_job.log"
+    if last_ditch_logfile[0] != '|'
+      last_ditch_logfile = expand_rails_path(last_ditch_logfile)
+    end
+    STDIN.reopen("/dev/null")
+    STDOUT.reopen(open(last_ditch_logfile, 'a'))
+    STDERR.reopen(STDOUT)
+    STDOUT.sync = STDERR.sync = true
+  end
+
+  def pid_folder
+    options[:pid_folder]
+  end
+
+  def pid_file
+    File.join(pid_folder, 'delayed_jobs_pool.pid')
+  end
+
+  def remove_pid_file
+    return unless @daemon
+    pid = File.read(pid_file) if File.file?(pid_file)
+    if pid.to_i == Process.pid
+      FileUtils.rm(pid_file)
+    end
+  end
+
+  def last_ditch_logfile
+    @config['last_ditch_logfile']
+  end
+
+  def stop(kill = false)
+    pid = status(false) && File.read(pid_file).to_i if File.file?(pid_file)
+    if pid && pid > 0
+      puts "Stopping pool #{pid}..."
+      signal = 'INT'
+      if kill
+        pid = -pid # send to the whole group
+        if kill == 9
+          signal = 'KILL'
+        else
+          signal = 'TERM'
+        end
+      end
+      begin
+        Process.kill(signal, pid)
+      rescue Errno::ESRCH
+        # ignore if the pid no longer exists
+      end
+    else
+      status
+    end
+  end
+
+  def status(print = true)
+    pid = File.read(pid_file) if File.file?(pid_file)
+    alive = pid && pid.to_i > 0 && (Process.kill(0, pid.to_i) rescue false) && :running
+    alive ||= :draining if pid.to_i > 0 && Process.kill(0, -pid.to_i) rescue false
+    if alive
+      puts "Delayed jobs #{alive}, pool PID: #{pid}" if print
+    else
+      puts "No delayed jobs pool running" if print && print != :alive
+    end
+    alive
+  end
+
+  def read_config(config_filename)
+    config = YAML.load_file(config_filename)
+    env = defined?(RAILS_ENV) ? RAILS_ENV : ENV['RAILS_ENV'] || 'development'
+    @config = config[env] || config['default']
+    # Backwards compatibility from when the config was just an array of queues
+    @config = { :workers => @config } if @config.is_a?(Array)
+    unless @config && @config.is_a?(Hash)
+      raise ArgumentError,
+        "Invalid config file #{config_filename}"
+    end
+  end
+
+  def apply_config
+    @config = @config.with_indifferent_access
+    Settings::SETTINGS.each do |setting|
+      Settings.send("#{setting}=", @config[setting.to_s]) if @config.key?(setting.to_s)
+    end
+  end
+
+  def expand_rails_path(path)
+    File.expand_path("../#{path}", ENV['BUNDLE_GEMFILE'])
+  end
+
+end
+end

data/lib/delayed/settings.rb

@@ -0,0 +1,32 @@
+module Delayed
+  module Settings
+    SETTINGS = [ :queue, :max_attempts, :sleep_delay, :sleep_delay_stagger, :fetch_batch_size, :select_random_from_batch, :worker_procname_prefix, :pool_procname_suffix, :default_job_options ]
+    SETTINGS_WITH_ARGS = [ :num_strands ]
+
+    SETTINGS.each do |setting|
+      mattr_writer(setting)
+      self.send("#{setting}=", nil)
+      define_singleton_method(setting) do
+        val = class_variable_get(:"@@#{setting}")
+      val.respond_to?(:call) ? val.call() : val
+      end
+    end
+
+    mattr_accessor(*SETTINGS_WITH_ARGS)
+
+    def self.queue=(queue_name)
+      raise(ArgumentError, "queue_name must not be blank") if queue_name.blank?
+      @@queue = queue_name
+    end
+
+    self.queue = "queue"
+    self.max_attempts = 1
+    self.sleep_delay = 2.0
+    self.sleep_delay_stagger = 2.0
+    self.fetch_batch_size = 5
+    self.select_random_from_batch = false
+
+    self.num_strands = ->(strand_name){ nil }
+    self.default_job_options = ->{ Hash.new }
+  end
+end

data/lib/delayed/version.rb

@@ -0,0 +1,3 @@
+module Delayed
+  VERSION = "0.9.0"
+end

data/lib/delayed/worker.rb

@@ -0,0 +1,213 @@
+module Delayed
+
+class TimeoutError < RuntimeError; end
+
+require 'tmpdir'
+
+class Worker
+  attr_reader :config, :queue, :min_priority, :max_priority
+
+  # Callback to fire when a delayed job fails max_attempts times. If this
+  # callback is defined, then the value of destroy_failed_jobs is ignored, and
+  # the job is destroyed if this block returns true.
+  #
+  # This allows for destroying "uninteresting" failures, while keeping around
+  # interesting failures to be investigated later.
+  #
+  # The block is called with args(job, last_exception)
+  def self.on_max_failures=(block)
+    @@on_max_failures = block
+  end
+  cattr_reader :on_max_failures
+
+  def self.lifecycle
+    @lifecycle ||= Delayed::Lifecycle.new
+  end
+
+  def initialize(options = {})
+    @exit = false
+    @config = options
+    @parent_pid = options[:parent_pid]
+    @queue = options[:queue] || Settings.queue
+    @min_priority = options[:min_priority]
+    @max_priority = options[:max_priority]
+    @max_job_count = options[:worker_max_job_count].to_i
+    @max_memory_usage = options[:worker_max_memory_usage].to_i
+    @job_count = 0
+  end
+
+  def name=(name)
+    @name = name
+  end
+
+  def name
+    @name ||= "#{Socket.gethostname rescue "X"}:#{self.id}"
+  end
+
+  def set_process_name(new_name)
+    $0 = "delayed:#{new_name}"
+  end
+
+  def exit?
+    @exit || parent_exited?
+  end
+
+  def parent_exited?
+    @parent_pid && @parent_pid != Process.ppid
+  end
+
+  def start
+    say "Starting worker", :info
+
+    trap('INT') { say 'Exiting'; @exit = true }
+
+    loop do
+      run
+      break if exit?
+    end
+
+    say "Stopping worker", :info
+  rescue => e
+    Rails.logger.fatal("Child process died: #{e.inspect}") rescue nil
+    ErrorReport.log_exception(:delayed_jobs, e) rescue nil
+  ensure
+    Delayed::Job.clear_locks!(name)
+  end
+
+  def run
+    job =
+        self.class.lifecycle.run_callbacks(:pop, self) do
+          Delayed::Job.get_and_lock_next_available(
+            name,
+            queue,
+            min_priority,
+            max_priority)
+        end
+
+    if job
+      configure_for_job(job) do
+        @job_count += perform(job)
+
+        if @max_job_count > 0 && @job_count >= @max_job_count
+          say "Max job count of #{@max_job_count} exceeded, dying"
+          @exit = true
+        end
+
+        if @max_memory_usage > 0
+          memory = sample_memory
+          if memory > @max_memory_usage
+            say "Memory usage of #{memory} exceeds max of #{@max_memory_usage}, dying"
+            @exit = true
+          else
+            say "Memory usage: #{memory}"
+          end
+        end
+      end
+    else
+      set_process_name("wait:#{Settings.worker_procname_prefix}#{@queue}:#{min_priority || 0}:#{max_priority || 'max'}")
+      sleep(Settings.sleep_delay + (rand * Settings.sleep_delay_stagger))
+    end
+  end
+
+  def perform(job)
+    count = 1
+    self.class.lifecycle.run_callbacks(:perform, self, job) do
+      set_process_name("run:#{Settings.worker_procname_prefix}#{job.id}:#{job.name}")
+      say("Processing #{log_job(job, :long)}", :info)
+      runtime = Benchmark.realtime do
+        if job.batch?
+          # each job in the batch will have perform called on it, so we don't
+          # need a timeout around this 
+          count = perform_batch(job)
+        else
+          job.invoke_job
+        end
+        job.destroy
+      end
+      say("Completed #{log_job(job)} #{"%.0fms" % (runtime * 1000)}", :info)
+    end
+    count
+  rescue Exception => e
+    handle_failed_job(job, e)
+    count
+  end
+
+  def perform_batch(parent_job)
+    batch = parent_job.payload_object
+    if batch.mode == :serial
+      batch.jobs.each do |job|
+        job.source = parent_job.source
+        job.create_and_lock!(name)
+        configure_for_job(job) do
+          perform(job)
+        end
+      end
+      batch.items.size
+    end
+  end
+
+  def handle_failed_job(job, error)
+    job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
+    say("Failed with #{error.class} [#{error.message}] (#{job.attempts} attempts)", :error)
+    job.reschedule(error)
+  end
+
+  def id
+    Process.pid
+  end
+
+  def say(msg, level = :debug)
+    Rails.logger.send(level, msg)
+  end
+
+  def log_job(job, format = :short)
+    case format
+    when :long
+      "#{job.full_name} #{ job.to_json(:include_root => false, :only => %w(tag strand priority attempts created_at max_attempts source)) }"
+    else
+      job.full_name
+    end
+  end
+
+  # set up the session context information, so that it gets logged with the job log lines
+  # also set up a unique tmpdir, which will get removed at the end of the job.
+  def configure_for_job(job)
+    previous_tmpdir = ENV['TMPDIR']
+    Thread.current[:running_delayed_job] = job
+
+    Dir.mktmpdir("job-#{job.id}-#{self.name.gsub(/[^\w\.]/, '.')}-") do |dir|
+      ENV['TMPDIR'] = dir
+      yield
+    end
+  ensure
+    ENV['TMPDIR'] = previous_tmpdir
+    Thread.current[:running_delayed_job] = nil
+  end
+
+  def self.current_job
+    Thread.current[:running_delayed_job]
+  end
+
+  # `sample` reports KB, not B
+  if File.directory?("/proc")
+    # linux w/ proc fs
+    LINUX_PAGE_SIZE = (size = `getconf PAGESIZE`.to_i; size > 0 ? size : 4096)
+    def sample_memory
+      s = File.read("/proc/#{Process.pid}/statm").to_i rescue 0
+      s * LINUX_PAGE_SIZE / 1024
+    end
+  else
+    # generic unix solution
+    def sample_memory
+      if Rails.env.test?
+        0
+      else
+        # hmm this is actually resident set size, doesn't include swapped-to-disk
+        # memory.
+        `ps -o rss= -p #{Process.pid}`.to_i
+      end
+    end
+  end
+
+end
+end