inst-jobs 2.0.0 → 3.0.0

data/lib/delayed/periodic.rb

@@ -1,88 +1,92 @@
 # frozen_string_literal: true
 
-require 'fugit'
+require "fugit"
 
 module Delayed
-class Periodic
-  attr_reader :name, :cron
+  class Periodic
+    attr_reader :name, :cron
 
-  def encode_with(coder)
-    coder.scalar("!ruby/Delayed::Periodic", @name)
-  end
+    def encode_with(coder)
+      coder.scalar("!ruby/Delayed::Periodic", @name)
+    end
 
-  cattr_accessor :scheduled, :overrides
-  self.scheduled = {}
-  self.overrides = {}
+    cattr_accessor :scheduled, :overrides
+    self.scheduled = {}
+    self.overrides = {}
 
-  def self.add_overrides(overrides)
-    overrides.each do |name, cron_line|
-      # throws error if the line is malformed
-      Fugit.do_parse_cron(cron_line)
+    def self.add_overrides(overrides)
+      overrides.each do |_name, cron_line|
+        # throws error if the line is malformed
+        Fugit.do_parse_cron(cron_line)
+      end
+      self.overrides.merge!(overrides)
     end
-    self.overrides.merge!(overrides)
-  end
 
-  def self.cron(job_name, cron_line, job_args = {}, &block)
-    raise ArgumentError, "job #{job_name} already scheduled!" if self.scheduled[job_name]
-    cron_line = overrides[job_name] || cron_line
-    self.scheduled[job_name] = self.new(job_name, cron_line, job_args, block)
-  end
+    def self.cron(job_name, cron_line, job_args = {}, &block)
+      raise ArgumentError, "job #{job_name} already scheduled!" if scheduled[job_name]
 
-  def self.audit_queue
-    # we used to queue up a job in a strand here, and perform the audit inside that job
-    # however, now that we're using singletons for scheduling periodic jobs,
-    # it's fine to just do the audit in-line here without risk of creating duplicates
-    perform_audit!
-  end
+      cron_line = overrides[job_name] || cron_line
+      scheduled[job_name] = new(job_name, cron_line, job_args, block)
+    end
 
-  # make sure all periodic jobs are scheduled for their next run in the job queue
-  # this auditing should run on the strand
-  def self.perform_audit!
-    self.scheduled.each { |name, periodic| periodic.enqueue }
-  end
+    def self.audit_queue
+      # we used to queue up a job in a strand here, and perform the audit inside that job
+      # however, now that we're using singletons for scheduling periodic jobs,
+      # it's fine to just do the audit in-line here without risk of creating duplicates
+      Delayed::Job.transaction do
+        # for db performance reasons, we only need one process doing this at a time
+        # so if we can't get an advisory lock, just abort. we'll try again soon
+        return unless Delayed::Job.attempt_advisory_lock("Delayed::Periodic#audit_queue")
 
-  def initialize(name, cron_line, job_args, block)
-    @name = name
-    @cron = Fugit.do_parse_cron(cron_line)
-    @job_args = { :priority => Delayed::LOW_PRIORITY }.merge(job_args.symbolize_keys)
-    @block = block
-  end
+        perform_audit!
+      end
+    end
 
-  def enqueue
-    Delayed::Job.enqueue(self, **enqueue_args)
-  end
+    # make sure all periodic jobs are scheduled for their next run in the job queue
+    # this auditing should run on the strand
+    def self.perform_audit!
+      scheduled.each { |_name, periodic| periodic.enqueue }
+    end
 
-  def enqueue_args
-    inferred_args = {
-      max_attempts: 1,
-      run_at: @cron.next_time(Delayed::Periodic.now).utc.to_time,
-      singleton: (@job_args[:singleton] == false ? nil : tag),
-      # yes, checking for whether it is actually the boolean literal false,
-      # which means the consuming code really does not want this job to be
-      # a singleton at all.
-      on_conflict: :patient
-    }
-    @job_args.merge(inferred_args)
-  end
+    def initialize(name, cron_line, job_args, block)
+      @name = name
+      @cron = Fugit.do_parse_cron(cron_line)
+      @job_args = { priority: Delayed::LOW_PRIORITY }.merge(job_args.symbolize_keys)
+      @block = block
+    end
 
-  def perform
-    @block.call()
-  ensure
-    begin
-      enqueue
-    rescue
-      # double fail! the auditor will have to catch this.
-      Rails.logger.error "Failure enqueueing periodic job! #{@name} #{$!.inspect}"
+    def enqueue
+      Delayed::Job.enqueue(self, **enqueue_args)
     end
-  end
 
-  def tag
-    "periodic: #{@name}"
-  end
-  alias_method :display_name, :tag
+    def enqueue_args
+      inferred_args = {
+        max_attempts: 1,
+        run_at: @cron.next_time(Delayed::Periodic.now).utc.to_time,
+        singleton: tag,
+        on_conflict: :patient
+      }
+      @job_args.merge(inferred_args)
+    end
 
-  def self.now
-    Time.zone.now
+    def perform
+      @block.call
+    ensure
+      begin
+        enqueue
+      rescue
+        # double fail! the auditor will have to catch this.
+        Rails.logger.error "Failure enqueueing periodic job! #{@name} #{$!.inspect}"
+      end
+    end
+
+    def tag
+      "periodic: #{@name}"
+    end
+    alias display_name tag
+
+    def self.now
+      Time.zone.now
+    end
   end
 end
-end

data/lib/delayed/plugin.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext/class/attribute'
+require "active_support/core_ext/class/attribute"
 
 module Delayed
   class Plugin
@@ -11,9 +11,7 @@ module Delayed
     end
 
     def self.inject!
-      unless @injected
-        self.callback_block.call(Delayed::Worker.lifecycle) if self.callback_block
-      end
+      callback_block&.call(Delayed::Worker.lifecycle) unless @injected
       @injected = true
     end
 

data/lib/delayed/pool.rb

@@ -1,222 +1,260 @@
 # frozen_string_literal: true
 
 module Delayed
-class Pool
-  include Delayed::Logging
+  class Pool
+    include Delayed::Logging
 
-  mattr_accessor :on_fork
-  self.on_fork = ->{ }
+    mattr_accessor :on_fork
+    self.on_fork = -> {}
 
-  SIGNALS = %i{INT TERM QUIT}
-  POOL_SLEEP_PERIOD = 5
+    SIGNALS = %i[INT TERM QUIT].freeze
+    POOL_SLEEP_PERIOD = 5
 
-  attr_reader :workers
+    attr_reader :workers
 
-  def initialize(*args)
-    if args.first.is_a?(Hash)
-      @config = args.first
-    else
-      warn "Calling Delayed::Pool.new directly is deprecated. Use `Delayed::CLI.new.run()` instead."
+    def initialize(*args)
+      if args.first.is_a?(Hash)
+        @config = args.first
+      else
+        warn "Calling Delayed::Pool.new directly is deprecated. Use `Delayed::CLI.new.run()` instead."
+      end
+      @workers = {}
+      @signal_queue = []
+      @self_pipe = IO.pipe
     end
-    @workers = {}
-    @signal_queue = []
-    @self_pipe = IO.pipe
-  end
 
-  def run
-    warn "Delayed::Pool#run is deprecated and will be removed. Use `Delayed::CLI.new.run()` instead."
-    Delayed::CLI.new.run()
-  end
+    def run
+      warn "Delayed::Pool#run is deprecated and will be removed. Use `Delayed::CLI.new.run()` instead."
+      Delayed::CLI.new.run
+    end
 
-  def start
-    say "Started job master", :info
-    SIGNALS.each { |sig| trap(sig) { @signal_queue << sig; wake_up } }
-    $0 = procname
-    # fork to handle unlocking (to prevent polluting the parent with worker objects)
-    unlock_pid = fork_with_reconnects do
-      unlock_orphaned_jobs
-    end
-    Process.wait unlock_pid
-
-    spawn_periodic_auditor
-    spawn_all_workers
-    say "Workers spawned"
-    join
-    say "Shutting down"
-    stop
-    reap_all_children
-  rescue Exception => e
-    say "Job master died with error: #{e.inspect}\n#{e.backtrace.join("\n")}", :fatal
-    raise
-  end
+    def start
+      say "Started job master", :info
+      SIGNALS.each do |sig|
+        trap(sig) do
+          @signal_queue << sig
+          wake_up
+        end
+      end
+      $0 = procname
+      # fork to handle unlocking (to prevent polluting the parent with worker objects)
+      unlock_pid = fork_with_reconnects do
+        unlock_orphaned_jobs
+      end
+      Process.wait unlock_pid
+
+      spawn_periodic_auditor
+      spawn_abandoned_job_cleanup
+      spawn_all_workers
+      say "Workers spawned"
+      join
+      say "Shutting down"
+      stop
+      reap_all_children
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      say "Job master died with error: #{e.inspect}\n#{e.backtrace.join("\n")}", :fatal
+      raise
+    end
 
-  protected
+    protected
 
-  def procname
-    "delayed_jobs_pool#{Settings.pool_procname_suffix}"
-  end
+    def procname
+      "delayed_jobs_pool#{Settings.pool_procname_suffix}"
+    end
 
-  def unlock_orphaned_jobs(worker = nil, pid = nil)
-    return if Settings.disable_automatic_orphan_unlocking
+    def unlock_orphaned_jobs(_worker = nil, pid = nil)
+      return if Settings.disable_automatic_orphan_unlocking
 
-    unlocked_jobs = Delayed::Job.unlock_orphaned_jobs(pid)
-    say "Unlocked #{unlocked_jobs} orphaned jobs" if unlocked_jobs > 0
-    ActiveRecord::Base.connection_handler.clear_all_connections! unless Rails.env.test?
-  end
+      unlocked_jobs = Delayed::Job.unlock_orphaned_jobs(pid)
+      say "Unlocked #{unlocked_jobs} orphaned jobs" if unlocked_jobs.positive?
+      ActiveRecord::Base.connection_handler.clear_all_connections! unless Rails.env.test?
+    end
 
-  def spawn_all_workers
-    ActiveRecord::Base.connection_handler.clear_all_connections!
+    def spawn_all_workers
+      ActiveRecord::Base.connection_handler.clear_all_connections!
 
-    if @config[:work_queue] == 'parent_process'
-      @work_queue = WorkQueue::ParentProcess.new
-      spawn_work_queue
-    end
+      if @config[:work_queue] == "parent_process"
+        @work_queue = WorkQueue::ParentProcess.new
+        spawn_work_queue
+      end
 
-    @config[:workers].each do |worker_config|
-      (worker_config[:workers] || 1).times { spawn_worker(worker_config) }
+      @config[:workers].each do |worker_config|
+        (worker_config[:workers] || 1).times { spawn_worker(worker_config) }
+      end
     end
-  end
 
-  def spawn_work_queue
-    parent_pid = Process.pid
-    pid = fork_with_reconnects do
-      $0 = "delayed_jobs_work_queue#{Settings.pool_procname_suffix}"
-      @work_queue.server(parent_pid: parent_pid).run
+    def spawn_work_queue
+      parent_pid = Process.pid
+      pid = fork_with_reconnects do
+        $0 = "delayed_jobs_work_queue#{Settings.pool_procname_suffix}"
+        @work_queue.server(parent_pid: parent_pid).run
+      end
+      workers[pid] = :work_queue
     end
-    workers[pid] = :work_queue
-  end
 
-  def spawn_worker(worker_config)
-    if worker_config[:periodic]
-      return # backwards compat
-    else
+    def spawn_worker(worker_config)
+      return if worker_config[:periodic] # backwards compat
+
       worker_config[:parent_pid] = Process.pid
       worker_config[:work_queue] = @work_queue.client if @work_queue
       worker = Delayed::Worker.new(worker_config)
+
+      pid = fork_with_reconnects do
+        worker.start
+      end
+      workers[pid] = worker
     end
 
-    pid = fork_with_reconnects do
-      worker.start
+    # child processes need to reconnect so they don't accidentally share redis or
+    # db connections with the parent
+    def fork_with_reconnects
+      fork do
+        @self_pipe.each(&:close) # sub-processes don't need to wake us up; keep their FDs clean
+        Pool.on_fork.call
+        Delayed::Job.reconnect!
+        yield
+      end
     end
-    workers[pid] = worker
-  end
 
-  # child processes need to reconnect so they don't accidentally share redis or
-  # db connections with the parent
-  def fork_with_reconnects
-    fork do
-      Pool.on_fork.()
-      Delayed::Job.reconnect!
-      yield
+    def spawn_abandoned_job_cleanup
+      return if Settings.disable_abandoned_job_cleanup
+
+      cleanup_interval_in_minutes = 60
+      @abandoned_cleanup_thread = Thread.new do
+        # every hour (staggered by process)
+        # check for dead jobs and cull them.
+        # Will actually be more often based on the
+        # number of worker nodes in the pool.  This will actually
+        # be a max of N times per hour where N is the number of workers,
+        # but they won't overrun each other because the health check
+        # takes an advisory lock internally
+        sleep(rand(cleanup_interval_in_minutes * 60))
+        loop do
+          schedule_abandoned_job_cleanup
+          sleep(cleanup_interval_in_minutes * 60)
+        end
+      end
     end
-  end
 
-  def spawn_periodic_auditor
-    return if Settings.disable_periodic_jobs
+    def schedule_abandoned_job_cleanup
+      pid = fork_with_reconnects do
+        # we want to avoid db connections in the main pool process
+        $0 = "delayed_abandoned_job_cleanup"
+        Delayed::Worker::HealthCheck.reschedule_abandoned_jobs
+      end
+      workers[pid] = :abandoned_job_cleanup
+    end
 
-    @periodic_thread = Thread.new do
-      # schedule the initial audit immediately on startup
-      schedule_periodic_audit
-      # initial sleep is randomized, for some staggering in the audit calls
-      # since job processors are usually all restarted at the same time
-      sleep(rand(15 * 60))
-      loop do
+    def spawn_periodic_auditor
+      return if Settings.disable_periodic_jobs
+
+      @periodic_thread = Thread.new do
+        # schedule the initial audit immediately on startup
         schedule_periodic_audit
-        sleep(15 * 60)
+        # initial sleep is randomized, for some staggering in the audit calls
+        # since job processors are usually all restarted at the same time
+        sleep(rand(15 * 60))
+        loop do
+          schedule_periodic_audit
+          sleep(15 * 60)
+        end
       end
     end
-  end
 
-  def schedule_periodic_audit
-    pid = fork_with_reconnects do
-      # we want to avoid db connections in the main pool process
-      $0 = "delayed_periodic_audit_scheduler"
-      Delayed::Periodic.audit_queue
+    def schedule_periodic_audit
+      pid = fork_with_reconnects do
+        # we want to avoid db connections in the main pool process
+        $0 = "delayed_periodic_audit_scheduler"
+        Delayed::Periodic.audit_queue
+      end
+      workers[pid] = :periodic_audit
     end
-    workers[pid] = :periodic_audit
-  end
 
-  def join
-    loop do
-      maintain_children
-      case sig = @signal_queue.shift
-      when nil
-        pool_sleep
-      when :QUIT
-        break
-      when :TERM, :INT
-        stop(graceful: false) if Settings.kill_workers_on_exit
-        break
-      else
-        logger.warn("Unexpected signal received: #{sig}")
+    def join
+      loop do
+        maintain_children
+        case sig = @signal_queue.shift
+        when nil
+          pool_sleep
+        when :QUIT
+          break
+        when :TERM, :INT
+          stop(graceful: false) if Settings.kill_workers_on_exit
+          break
+        else
+          logger.warn("Unexpected signal received: #{sig}")
+        end
       end
     end
-  end
 
-  def pool_sleep
-    IO.select([@self_pipe[0]], nil, nil, POOL_SLEEP_PERIOD)
-    @self_pipe[0].read_nonblock(11, exception: false)
-  end
+    def pool_sleep
+      @self_pipe[0].wait_readable(POOL_SLEEP_PERIOD)
+      @self_pipe[0].read_nonblock(11, exception: false)
+    end
+
+    def stop(graceful: true, timeout: Settings.slow_exit_timeout)
+      signal_for_children = graceful ? :QUIT : :TERM
+      if Settings.kill_workers_on_exit
+        limit = Time.zone.now + timeout
+        until @workers.empty? || Time.zone.now >= limit
+          signal_all_children(signal_for_children)
+          # Give our children some time to process the signal before checking if
+          # they've exited
+          sleep(0.5)
+          reap_all_children.each { |pid| @workers.delete(pid) }
+        end
 
-  def stop(graceful: true, timeout: Settings.slow_exit_timeout)
-    signal_for_children = graceful ? :QUIT : :TERM
-    if Settings.kill_workers_on_exit
-      limit = Time.now + timeout
-      until @workers.empty? || Time.now >= limit
+        # We really want to give the workers every oportunity to clean up after
+        # themselves before murdering them.
+        stop(graceful: false, timeout: 2) if graceful
+        signal_all_children(:KILL)
+      else
         signal_all_children(signal_for_children)
-        # Give our children some time to process the signal before checking if
-        # they've exited
-        sleep(0.5)
-        reap_all_children.each { |pid| @workers.delete(pid) }
       end
+    end
 
-      # We really want to give the workers every oportunity to clean up after
-      # themselves before murdering them.
-      stop(graceful: false, timeout: 2) if graceful
-      signal_all_children(:KILL)
-    else
-      signal_all_children(signal_for_children)
+    def signal_all_children(signal)
+      workers.each_key { |pid| signal_child(signal, pid) }
     end
-  end
 
-  def signal_all_children(signal)
-    workers.keys.each { |pid| signal_child(signal, pid) }
-  end
+    def signal_child(signal, pid)
+      Process.kill(signal, pid)
+    rescue Erron::ESRCH
+      workers.delete(pid)
+    end
 
-  def signal_child(signal, pid)
-    Process.kill(signal, pid)
-  rescue Erron::ESRCH
-    workers.delete(pid)
-  end
+    # Respawn all children that have exited since we last checked
+    def maintain_children
+      reap_all_children.each do |pid|
+        respawn_child(pid)
+      end
+    end
+
+    # Reaps processes that have exited or just returns if none have exited
+    #
+    # @return Array An array of child pids that have exited
+    def reap_all_children
+      exited_pids = []
+      loop do
+        pid = Process.wait(-1, Process::WNOHANG)
+        break unless pid
 
-  # Respawn all children that have exited since we last checked
-  def maintain_children
-    reap_all_children.each do |pid|
-      respawn_child(pid)
+        exited_pids << pid
+      rescue Errno::ECHILD
+        break
+      end
+      exited_pids
     end
-  end
 
-  # Reaps processes that have exited or just returns if none have exited
-  #
-  # @return Array An array of child pids that have exited
-  def reap_all_children
-    exited_pids = []
-    begin
-      pid = Process.wait(-1, Process::WNOHANG)
-      break unless pid
-      exited_pids << pid
-    rescue Errno::ECHILD
-      break
-    end while true # I hate doing loops this way but it's the only way to make the rescue work
-    exited_pids
-  end
+    def respawn_child(child)
+      return unless workers.include?(child)
 
-  def respawn_child(child)
-    if workers.include?(child)
       worker = workers.delete(child)
       case worker
       when :periodic_audit
         say "ran auditor: #{worker}"
+      when :abandoned_job_cleanup
+        say "ran cleanup: #{worker}"
       when :work_queue
         say "work queue exited, restarting", :info
         spawn_work_queue
@@ -230,10 +268,9 @@ class Pool
         spawn_worker(worker.config)
       end
     end
-  end
 
-  def wake_up
-    @self_pipe[1].write_nonblock('.', exception: false)
+    def wake_up
+      @self_pipe[1].write_nonblock(".", exception: false)
+    end
   end
 end
-end

data/lib/delayed/server/helpers.rb

@@ -8,21 +8,21 @@ module Delayed
       end
 
       def url_path(*path_parts)
-        [path_prefix, path_parts].join('/').squeeze('/')
+        [path_prefix, path_parts].join("/").squeeze("/")
       end
 
       def path_prefix
-        request.env['SCRIPT_NAME']
+        request.env["SCRIPT_NAME"]
       end
 
       def render_javascript_env
         {
           Routes: {
             root: path_prefix,
-            running: url_path('running'),
-            tags: url_path('tags'),
-            jobs: url_path('jobs'),
-            bulkUpdate: url_path('bulk_update'),
+            running: url_path("running"),
+            tags: url_path("tags"),
+            jobs: url_path("jobs"),
+            bulkUpdate: url_path("bulk_update")
           }
         }.to_json
       end