inst-jobs 0.11.0

data/lib/delayed/work_queue/parent_process.rb

@@ -0,0 +1,180 @@
+require 'socket'
+require 'tempfile'
+require 'timeout'
+
+module Delayed
+module WorkQueue
+# ParentProcess is a WorkQueue implementation that spawns a separate worker
+# process for querying the queue. Each Worker child process sends requests to
+# the ParentProcess via IPC, and receives responses. This centralized queue
+# querying cuts down on db queries and lock contention, and allows the
+# possibility for other centralized logic such as notifications when all workers
+# are idle.
+#
+# The IPC implementation uses Unix stream sockets and Ruby's built-in Marshal
+# functionality. The ParentProcess creates a Unix socket on the filesystem in
+# the tmp directory, so that if a worker process dies and is restarted it can
+# reconnect to the socket.
+#
+# While Unix and IP sockets are API compatible, we take a lot of shortcuts
+# because we know it's just a local Unix socket. If we ever wanted to swap this
+# out for a TCP/IP socket and have the WorkQueue running on another host, we'd
+# want to be a lot more robust about partial reads/writes and timeouts.
+class ParentProcess
+  class ProtocolError < RuntimeError
+  end
+
+  def initialize
+    @path = self.class.generate_socket_path
+  end
+
+  def self.generate_socket_path
+    # We utilize Tempfile as a convenient way to get a socket filename in the
+    # writeable temp directory. However, since we destroy the normal file and
+    # write a unix socket file to the same location, we lose the hard uniqueness
+    # guarantees of Tempfile. This is OK for this use case, we only generate one
+    # Tempfile with this prefix.
+    tmp = Tempfile.new("inst-jobs-#{Process.pid}-")
+    path = tmp.path
+    tmp.close!
+    path
+  end
+
+  def server(parent_pid: nil)
+    # The unix_server_socket method takes care of cleaning up any existing
+    # socket for us if the work queue process dies and is restarted.
+    listen_socket = Socket.unix_server_socket(@path)
+    Server.new(listen_socket, parent_pid: parent_pid)
+  end
+
+  def client
+    Client.new(Addrinfo.unix(@path))
+  end
+
+  class Client
+    attr_reader :addrinfo
+
+    def initialize(addrinfo)
+      @addrinfo = addrinfo
+    end
+
+    def get_and_lock_next_available(name, queue_name, min_priority, max_priority)
+      @socket ||= @addrinfo.connect
+      Marshal.dump([name, queue_name, min_priority, max_priority], @socket)
+      response = Marshal.load(@socket)
+      unless response.nil? || (response.is_a?(Delayed::Job) && response.locked_by == name)
+        raise(ProtocolError, "response is not a locked job: #{response.inspect}")
+      end
+      response
+    rescue SystemCallError, IOError
+      # The work queue process died. Return nil to signal the worker
+      # process should sleep as if no job was found, and then retry.
+      @socket = nil
+      nil
+    end
+  end
+
+  class Server
+    attr_reader :listen_socket
+
+    def initialize(listen_socket, parent_pid: nil)
+      @listen_socket = listen_socket
+      @parent_pid = parent_pid
+      @clients = {}
+    end
+
+    def connected_clients
+      @clients.size
+    end
+
+    def all_workers_idle?
+      !@clients.any? { |_, c| c.working }
+    end
+
+    def say(msg, level = :debug)
+      if defined?(Rails.logger) && Rails.logger
+        Rails.logger.send(level, "[#{Process.pid}]Q #{msg}")
+      else
+        puts(msg)
+      end
+    end
+
+    # run the server queue worker
+    # this method does not return, only exits or raises an exception
+    def run
+      say "Starting work queue process"
+
+      while !exit?
+        run_once
+      end
+
+    rescue => e
+      say "WorkQueue Server died: #{e.inspect}"
+      raise
+    end
+
+    def run_once
+      handles = @clients.keys + [@listen_socket]
+      readable, _, _ = IO.select(handles, nil, nil, 1)
+      if readable
+        readable.each { |s| handle_read(s) }
+      end
+    end
+
+    def handle_read(socket)
+      if socket == @listen_socket
+        handle_accept
+      else
+        handle_request(socket)
+      end
+    end
+
+    # Any error on the listen socket other than WaitReadable will bubble up
+    # and terminate the work queue process, to be restarted by the parent daemon.
+    def handle_accept
+      client, _addr = @listen_socket.accept_nonblock
+      if client
+        @clients[client] = ClientState.new(false)
+      end
+    rescue IO::WaitReadable
+      # ignore and just try accepting again next time through the loop
+    end
+
+    def handle_request(socket)
+      # There is an assumption here that the client will never send a partial
+      # request and then leave the socket open. Doing so would leave us hanging
+      # here forever. This is only a reasonable assumption because we control
+      # the client.
+      request = client_timeout { Marshal.load(socket) }
+      response = nil
+      Delayed::Worker.lifecycle.run_callbacks(:work_queue_pop, self) do
+        response = Delayed::Job.get_and_lock_next_available(*request)
+        @clients[socket].working = !response.nil?
+      end
+      client_timeout { Marshal.dump(response, socket) }
+    rescue SystemCallError, IOError, Timeout::Error
+      # this socket went away
+      begin
+        socket.close
+      rescue IOError
+      end
+      @clients.delete(socket)
+    end
+
+    def exit?
+      parent_exited?
+    end
+
+    def parent_exited?
+      @parent_pid && @parent_pid != Process.ppid
+    end
+
+    def client_timeout
+      Timeout.timeout(Settings.parent_process_client_timeout) { yield }
+    end
+
+    ClientState = Struct.new(:working)
+  end
+end
+end
+end

data/lib/delayed/worker.rb

@@ -0,0 +1,234 @@
+module Delayed
+
+class TimeoutError < RuntimeError; end
+
+require 'tmpdir'
+require 'set'
+
+class Worker
+  attr_reader :config, :queue_name, :min_priority, :max_priority, :work_queue
+
+  # Callback to fire when a delayed job fails max_attempts times. If this
+  # callback is defined, then the value of destroy_failed_jobs is ignored, and
+  # the job is destroyed if this block returns true.
+  #
+  # This allows for destroying "uninteresting" failures, while keeping around
+  # interesting failures to be investigated later.
+  #
+  # The block is called with args(job, last_exception)
+  def self.on_max_failures=(block)
+    @@on_max_failures = block
+  end
+  cattr_reader :on_max_failures
+
+  cattr_accessor :plugins
+  self.plugins = Set.new
+
+  def self.lifecycle
+    @lifecycle ||= Delayed::Lifecycle.new
+  end
+
+  def initialize(options = {})
+    @exit = false
+    @config = options
+    @parent_pid = options[:parent_pid]
+    @queue_name = options[:queue] || Settings.queue
+    @min_priority = options[:min_priority]
+    @max_priority = options[:max_priority]
+    @max_job_count = options[:worker_max_job_count].to_i
+    @max_memory_usage = options[:worker_max_memory_usage].to_i
+    @work_queue = options[:work_queue] || WorkQueue::InProcess.new
+    @job_count = 0
+
+    app = Rails.application
+    if app && !app.config.cache_classes
+      Delayed::Worker.lifecycle.around(:perform) do |&block|
+        reload = app.config.reload_classes_only_on_change != true || app.reloaders.map(&:updated?).any?
+        ActionDispatch::Reloader.prepare! if reload
+        begin
+          block.call
+        ensure
+          ActionDispatch::Reloader.cleanup! if reload
+        end
+      end
+    end
+
+    plugins.each { |plugin| plugin.inject! }
+  end
+
+  def name
+    @name ||= "#{Socket.gethostname rescue "X"}:#{self.id}"
+  end
+
+  def set_process_name(new_name)
+    $0 = "delayed:#{new_name}"
+  end
+
+  def exit?
+    @exit || parent_exited?
+  end
+
+  def parent_exited?
+    @parent_pid && @parent_pid != Process.ppid
+  end
+
+  def start
+    say "Starting worker", :info
+
+    trap('INT') { say 'Exiting'; @exit = true }
+
+    self.class.lifecycle.run_callbacks(:execute, self) do
+      loop do
+        run
+        break if exit?
+      end
+    end
+
+    say "Stopping worker", :info
+  rescue => e
+    Rails.logger.fatal("Child process died: #{e.inspect}") rescue nil
+    self.class.lifecycle.run_callbacks(:exceptional_exit, self, e) { }
+  ensure
+    Delayed::Job.clear_locks!(name)
+  end
+
+  def run
+    self.class.lifecycle.run_callbacks(:loop, self) do
+      job = self.class.lifecycle.run_callbacks(:pop, self) do
+        work_queue.get_and_lock_next_available(name, queue_name, min_priority, max_priority)
+      end
+
+      if job
+        configure_for_job(job) do
+          @job_count += perform(job)
+
+          if @max_job_count > 0 && @job_count >= @max_job_count
+            say "Max job count of #{@max_job_count} exceeded, dying"
+            @exit = true
+          end
+
+          if @max_memory_usage > 0
+            memory = sample_memory
+            if memory > @max_memory_usage
+              say "Memory usage of #{memory} exceeds max of #{@max_memory_usage}, dying"
+              @exit = true
+            else
+              say "Memory usage: #{memory}"
+            end
+          end
+        end
+      else
+        set_process_name("wait:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}")
+        sleep(Settings.sleep_delay + (rand * Settings.sleep_delay_stagger))
+      end
+    end
+  end
+
+  def perform(job)
+    count = 1
+    raise Delayed::Backend::JobExpired, "job expired at #{job.expires_at}" if job.expired?
+    self.class.lifecycle.run_callbacks(:perform, self, job) do
+      set_process_name("run:#{Settings.worker_procname_prefix}#{job.id}:#{job.name}")
+      say("Processing #{log_job(job, :long)}", :info)
+      runtime = Benchmark.realtime do
+        if job.batch?
+          # each job in the batch will have perform called on it, so we don't
+          # need a timeout around this
+          count = perform_batch(job)
+        else
+          job.invoke_job
+        end
+        job.destroy
+      end
+      say("Completed #{log_job(job)} #{"%.0fms" % (runtime * 1000)}", :info)
+    end
+    count
+  rescue Exception => e
+    self.class.lifecycle.run_callbacks(:error, self, job, e) do
+      handle_failed_job(job, e)
+    end
+    count
+  end
+
+  def perform_batch(parent_job)
+    batch = parent_job.payload_object
+    if batch.mode == :serial
+      batch.jobs.each do |job|
+        job.source = parent_job.source
+        job.create_and_lock!(name)
+        configure_for_job(job) do
+          perform(job)
+        end
+      end
+      batch.items.size
+    end
+  end
+
+  def handle_failed_job(job, error)
+    job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
+    say("Failed with #{error.class} [#{error.message}] (#{job.attempts} attempts)", :error)
+    job.reschedule(error)
+  end
+
+  def id
+    Process.pid
+  end
+
+  def say(msg, level = :debug)
+    Rails.logger.send(level, msg)
+  end
+
+  def log_job(job, format = :short)
+    case format
+    when :long
+      "#{job.full_name} #{ job.to_json(:include_root => false, :only => %w(tag strand priority attempts created_at max_attempts source)) }"
+    else
+      job.full_name
+    end
+  end
+
+  # set up the session context information, so that it gets logged with the job log lines
+  # also set up a unique tmpdir, which will get removed at the end of the job.
+  def configure_for_job(job)
+    previous_tmpdir = ENV['TMPDIR']
+    Thread.current[:running_delayed_job] = job
+
+    dir = Dir.mktmpdir("job-#{job.id}-#{self.name.gsub(/[^\w\.]/, '.')}-")
+    begin
+      ENV['TMPDIR'] = dir
+      yield
+    ensure
+      FileUtils.remove_entry(dir, true)
+    end
+  ensure
+    ENV['TMPDIR'] = previous_tmpdir
+    Thread.current[:running_delayed_job] = nil
+  end
+
+  def self.current_job
+    Thread.current[:running_delayed_job]
+  end
+
+  # `sample` reports KB, not B
+  if File.directory?("/proc")
+    # linux w/ proc fs
+    LINUX_PAGE_SIZE = (size = `getconf PAGESIZE`.to_i; size > 0 ? size : 4096)
+    def sample_memory
+      s = File.read("/proc/#{Process.pid}/statm").to_i rescue 0
+      s * LINUX_PAGE_SIZE / 1024
+    end
+  else
+    # generic unix solution
+    def sample_memory
+      if Rails.env.test?
+        0
+      else
+        # hmm this is actually resident set size, doesn't include swapped-to-disk
+        # memory.
+        `ps -o rss= -p #{Process.pid}`.to_i
+      end
+    end
+  end
+
+end
+end

data/lib/delayed/yaml_extensions.rb

@@ -0,0 +1,109 @@
+# New definitions for YAML to aid in serialization and deserialization of delayed jobs.
+
+require 'yaml'
+
+# First, tell YAML how to load a Module. This depends on Rails .constantize and autoloading.
+YAML.add_ruby_type("object:Module") do |type, val|
+  val.constantize
+end
+
+Psych.add_domain_type("ruby/object", "Module") do |type, val|
+  val.constantize
+end
+
+Psych.add_domain_type("ruby/object", "Class") do |type, val|
+  val.constantize
+end
+
+class Module
+  def to_yaml(opts = {})
+    YAML.quick_emit(self.object_id, opts) do |out|
+      out.scalar(taguri, name)
+    end
+  end
+end
+
+# Now we have to do the same for Class.
+YAML.add_ruby_type("object:Class") do |type, val|
+  val.constantize
+end
+
+class Class
+  def to_yaml(opts = {})
+    YAML.quick_emit(self.object_id, opts) do |out|
+      out.scalar(taguri, name)
+    end
+  end
+
+  def encode_with(coder)
+    coder.scalar("!ruby/object:Class", name)
+  end
+end
+
+# Now, tell YAML how to intelligently load ActiveRecord objects, using the
+# database rather than just serializing their attributes to the YAML. This
+# ensures the object is up to date when we use it in the job.
+class ActiveRecord::Base
+  yaml_as "tag:ruby.yaml.org,2002:ActiveRecord"
+
+  def to_yaml(opts = {})
+    if id.nil?
+      raise("Can't serialize unsaved ActiveRecord object for delayed job: #{self.inspect}")
+    end
+    YAML.quick_emit(self.object_id, opts) do |out|
+      out.scalar(taguri, id.to_s)
+    end
+  end
+
+  def encode_with(coder)
+    if id.nil?
+      raise("Can't serialize unsaved ActiveRecord object for delayed job: #{self.inspect}")
+    end
+    coder.scalar("!ruby/ActiveRecord:#{self.class.name}", id.to_s)
+  end
+
+  def self.yaml_new(klass, tag, val)
+    klass.find(val)
+  rescue ActiveRecord::RecordNotFound
+    raise Delayed::Backend::RecordNotFound, "Couldn't find #{klass} with id #{val.inspect}"
+  end
+end
+
+module Delayed
+  module PsychExt
+    module ToRuby
+      def visit_Psych_Nodes_Scalar(object)
+        case object.tag
+        when %r{^!ruby/ActiveRecord:(.+)$}
+          begin
+            klass = resolve_class(Regexp.last_match[1])
+            klass.unscoped.find(object.value)
+          rescue ActiveRecord::RecordNotFound
+            raise Delayed::Backend::RecordNotFound, "Couldn't find #{klass} with id #{object.value.inspect}"
+          end
+        when "tag:ruby.yaml.org,2002:Delayed::Periodic", "!ruby/Delayed::Periodic"
+          Delayed::Periodic.scheduled[object.value] || raise(NameError, "job #{object.value} is no longer scheduled")
+        else
+          super
+        end
+      end
+
+      def resolve_class(klass_name)
+        return nil if !klass_name || klass_name.empty?
+        klass_name.constantize
+      rescue
+        super
+      end
+    end
+  end
+end
+
+Psych::Visitors::ToRuby.prepend(Delayed::PsychExt::ToRuby)
+
+# Load Module/Class from yaml tag.
+class Module
+  def yaml_tag_read_class(name)
+    name.constantize
+    name
+  end
+end

data/lib/delayed_job.rb

@@ -0,0 +1,46 @@
+module Delayed
+  MIN_PRIORITY = 0
+  HIGH_PRIORITY = 0
+  NORMAL_PRIORITY = 10
+  LOW_PRIORITY = 20
+  LOWER_PRIORITY = 50
+  MAX_PRIORITY = 1_000_000
+
+  def self.select_backend(backend)
+    remove_const(:Job) if defined?(::Delayed::Job)
+    const_set(:Job, backend)
+  end
+end
+
+require 'rails'
+require 'active_support/core_ext/module/attribute_accessors'
+require 'active_record'
+require 'after_transaction_commit'
+
+require 'delayed/settings'
+require 'delayed/yaml_extensions'
+
+require 'delayed/backend/base'
+require 'delayed/backend/active_record'
+require 'delayed/backend/redis/job'
+require 'delayed/batch'
+require 'delayed/cli'
+require 'delayed/daemon'
+require 'delayed/job_tracking'
+require 'delayed/lifecycle'
+require 'delayed/log_tailer'
+require 'delayed/message_sending'
+require 'delayed/performable_method'
+require 'delayed/periodic'
+require 'delayed/plugin'
+require 'delayed/pool'
+require 'delayed/worker'
+require 'delayed/work_queue/in_process'
+require 'delayed/work_queue/parent_process'
+
+require 'delayed/engine'
+
+Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
+
+Object.send(:include, Delayed::MessageSending)
+Module.send(:include, Delayed::MessageSending::ClassMethods)

data/lib/inst-jobs.rb

@@ -0,0 +1 @@
+require 'delayed_job'