inst-jobs 2.2.1 → 2.4.0

data/lib/delayed/work_queue/parent_process.rb

@@ -1,69 +1,68 @@
 # frozen_string_literal: true
 
-require 'pathname'
-require 'socket'
-require 'timeout'
+require "pathname"
+require "socket"
+require "timeout"
 
-require_relative 'parent_process/client'
-require_relative 'parent_process/server'
+require_relative "parent_process/client"
+require_relative "parent_process/server"
 
 module Delayed
-module WorkQueue
-# ParentProcess is a WorkQueue implementation that spawns a separate worker
-# process for querying the queue. Each Worker child process sends requests to
-# the ParentProcess via IPC, and receives responses. This centralized queue
-# querying cuts down on db queries and lock contention, and allows the
-# possibility for other centralized logic such as notifications when all workers
-# are idle.
-#
-# The IPC implementation uses Unix stream sockets and Ruby's built-in Marshal
-# functionality. The ParentProcess creates a Unix socket on the filesystem in
-# the tmp directory, so that if a worker process dies and is restarted it can
-# reconnect to the socket.
-#
-# While Unix and IP sockets are API compatible, we take a lot of shortcuts
-# because we know it's just a local Unix socket. If we ever wanted to swap this
-# out for a TCP/IP socket and have the WorkQueue running on another host, we'd
-# want to be a lot more robust about partial reads/writes and timeouts.
-class ParentProcess
-  class ProtocolError < RuntimeError
-  end
+  module WorkQueue
+    # ParentProcess is a WorkQueue implementation that spawns a separate worker
+    # process for querying the queue. Each Worker child process sends requests to
+    # the ParentProcess via IPC, and receives responses. This centralized queue
+    # querying cuts down on db queries and lock contention, and allows the
+    # possibility for other centralized logic such as notifications when all workers
+    # are idle.
+    #
+    # The IPC implementation uses Unix stream sockets and Ruby's built-in Marshal
+    # functionality. The ParentProcess creates a Unix socket on the filesystem in
+    # the tmp directory, so that if a worker process dies and is restarted it can
+    # reconnect to the socket.
+    #
+    # While Unix and IP sockets are API compatible, we take a lot of shortcuts
+    # because we know it's just a local Unix socket. If we ever wanted to swap this
+    # out for a TCP/IP socket and have the WorkQueue running on another host, we'd
+    # want to be a lot more robust about partial reads/writes and timeouts.
+    class ParentProcess
+      class ProtocolError < RuntimeError
+      end
 
-  attr_reader :server_address
+      attr_reader :server_address
 
-  DEFAULT_SOCKET_NAME = 'inst-jobs.sock'.freeze
-  private_constant :DEFAULT_SOCKET_NAME
+      DEFAULT_SOCKET_NAME = "inst-jobs.sock"
+      private_constant :DEFAULT_SOCKET_NAME
 
-  def initialize(config = Settings.parent_process)
-    @config = config
-    @server_address = generate_socket_path(config['server_address'])
-  end
+      def initialize(config = Settings.parent_process)
+        @config = config
+        @server_address = generate_socket_path(config["server_address"])
+      end
 
-  def server(parent_pid: nil)
-    # The unix_server_socket method takes care of cleaning up any existing
-    # socket for us if the work queue process dies and is restarted.
-    listen_socket = Socket.unix_server_socket(@server_address)
-    Server.new(listen_socket, parent_pid: parent_pid, config: @config)
-  end
+      def server(parent_pid: nil)
+        # The unix_server_socket method takes care of cleaning up any existing
+        # socket for us if the work queue process dies and is restarted.
+        listen_socket = Socket.unix_server_socket(@server_address)
+        Server.new(listen_socket, parent_pid: parent_pid, config: @config)
+      end
 
-  def client
-    Client.new(Addrinfo.unix(@server_address), config: @config)
-  end
+      def client
+        Client.new(Addrinfo.unix(@server_address), config: @config)
+      end
 
-  private
+      private
 
-  def generate_socket_path(supplied_path)
-    pathname = Pathname.new(supplied_path)
+      def generate_socket_path(supplied_path)
+        pathname = Pathname.new(supplied_path)
 
-    if pathname.absolute? && pathname.directory?
-      pathname.join(DEFAULT_SOCKET_NAME).to_s
-    elsif pathname.absolute?
-      supplied_path
-    else
-      generate_socket_path(Settings.expand_rails_path(supplied_path))
+        if pathname.absolute? && pathname.directory?
+          pathname.join(DEFAULT_SOCKET_NAME).to_s
+        elsif pathname.absolute?
+          supplied_path
+        else
+          generate_socket_path(Settings.expand_rails_path(supplied_path))
+        end
+      end
     end
   end
 end
-end
-end
-

data/lib/delayed/worker/consul_health_check.rb

@@ -1,23 +1,23 @@
 # frozen_string_literal: true
 
-require_relative 'health_check'
-require_relative 'process_helper'
-require 'socket'
+require_relative "health_check"
+require_relative "process_helper"
+require "socket"
 
 module Delayed
   class Worker
     class ConsulHealthCheck < HealthCheck
       self.type_name = :consul
 
-      CONSUL_CONFIG_KEYS = %w{url acl_token}.map(&:freeze).freeze
-      DEFAULT_SERVICE_NAME = 'inst-jobs_worker'.freeze
+      CONSUL_CONFIG_KEYS = %w[url acl_token].map(&:freeze).freeze
+      DEFAULT_SERVICE_NAME = "inst-jobs_worker"
       attr_reader :service_client, :health_client
 
       def initialize(*, **)
         super
         # Because we don't want the consul client to be a hard dependency we're
         # only requiring it once it's absolutely needed
-        require 'diplomat'
+        require "diplomat"
 
         if config.keys.any? { |k| CONSUL_CONFIG_KEYS.include?(k) }
           consul_config = Diplomat::Configuration.new.tap do |conf|
@@ -35,10 +35,10 @@ module Delayed
 
       def start
         @service_client.register({
-          id: worker_name,
-          name: service_name,
-          check: check_attributes
-        })
+                                   id: worker_name,
+                                   name: service_name,
+                                   check: check_attributes
+                                 })
       end
 
       def stop
@@ -48,27 +48,28 @@ module Delayed
       def live_workers
         # Filter out critical workers (probably nodes failing their serf health check)
         live_nodes = @health_client.service(service_name, {
-          filter: 'not Checks.Status == critical'
-        })
+                                              filter: "not Checks.Status == critical"
+                                            })
 
-        live_nodes.map { |n| n.Service['ID']}
+        live_nodes.map { |n| n.Service["ID"] }
       end
 
       private
 
       def check_attributes
         {
-          args: ['bash', '-c', check_script],
-          status: 'passing',
-          interval: @config.fetch(:check_interval, '5m'),
-          deregister_critical_service_after: @config.fetch(:deregister_service_delay, '10m'),
+          args: ["bash", "-c", check_script],
+          status: "passing",
+          interval: @config.fetch(:check_interval, "5m"),
+          deregister_critical_service_after: @config.fetch(:deregister_service_delay, "10m")
         }.tap do |h|
-          h[:docker_container_id] = docker_container_id if @config['docker']
+          h[:docker_container_id] = docker_container_id if @config["docker"]
         end
       end
 
       def check_script
         return @check_script if @check_script
+
         mtime = ProcessHelper.mtime(Process.pid)
         @check_script = ProcessHelper.check_script(Process.pid, mtime)
       end
@@ -77,12 +78,13 @@ module Delayed
       # cgroups for part of its magic and also uses the container id as the cgroup name
       def docker_container_id
         return @docker_container_id if @docker_container_id
+
         content = File.read("/proc/1/cgroup").split("\n")
         @docker_container_id = content.last.split("/").last
       end
 
       def service_name
-        @service_name ||= @config.fetch('service_name', DEFAULT_SERVICE_NAME)
+        @service_name ||= @config.fetch("service_name", DEFAULT_SERVICE_NAME)
       end
     end
   end

data/lib/delayed/worker/health_check.rb

@@ -11,17 +11,20 @@ module Delayed
 
         def inherited(subclass)
           @subclasses << subclass
+          super
         end
 
         def build(type:, worker_name:, config: {})
           type = type.to_sym
           klass = @subclasses.find { |sc| sc.type_name == type }
           raise ArgumentError, "Unable to build a HealthCheck for type #{type}" unless klass
+
           klass.new(worker_name: worker_name, config: config)
         end
 
         def reschedule_abandoned_jobs
           return if Settings.worker_health_check_type == :none
+
           Delayed::Job.transaction do
             # this action is a special case, and SHOULD NOT be a periodic job
             # because if it gets wiped out suddenly during execution
@@ -32,27 +35,32 @@ module Delayed
             # operation, the transaction will end, releasing the advisory lock).
             result = attempt_advisory_lock
             return unless result
+
             checker = Worker::HealthCheck.build(
               type: Settings.worker_health_check_type,
               config: Settings.worker_health_check_config,
-              worker_name: 'cleanup-crew'
+              worker_name: "cleanup-crew"
             )
             live_workers = checker.live_workers
 
             Delayed::Job.running_jobs.each do |job|
               # prefetched jobs have their own way of automatically unlocking themselves
               next if job.locked_by.start_with?("prefetch:")
-              unless live_workers.include?(job.locked_by)
-                begin
-                  Delayed::Job.transaction do
-                    # double check that the job is still there. locked_by will immediately be reset
-                    # to nil in this transaction by Job#reschedule
-                    next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
-                    job.reschedule
-                  end
-                rescue
-                  ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
+
+              next if live_workers.include?(job.locked_by)
+
+              begin
+                Delayed::Job.transaction do
+                  # double check that the job is still there. locked_by will immediately be reset
+                  # to nil in this transaction by Job#reschedule
+                  next unless Delayed::Job.where(id: job,
+                                                 locked_by: job.locked_by)
+                                          .update_all(locked_by: "abandoned job cleanup") == 1
+
+                  job.reschedule
                 end
+              rescue
+                ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
               end
             end
           end
@@ -61,7 +69,8 @@ module Delayed
         def attempt_advisory_lock
           lock_name = "Delayed::Worker::HealthCheck#reschedule_abandoned_jobs"
           conn = ActiveRecord::Base.connection
-          conn.select_value("SELECT pg_try_advisory_xact_lock(#{conn.quote_table_name('half_md5_as_bigint')}('#{lock_name}'));")
+          fn_name = conn.quote_table_name("half_md5_as_bigint")
+          conn.select_value("SELECT pg_try_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
         end
       end
 

data/lib/delayed/worker/null_health_check.rb

@@ -13,7 +13,9 @@ module Delayed
         true
       end
 
-      def live_workers; []; end
+      def live_workers
+        []
+      end
     end
   end
 end

data/lib/delayed/worker/process_helper.rb

@@ -3,14 +3,13 @@
 module Delayed
   class Worker
     module ProcessHelper
-
-      STAT_LINUX = 'stat --format=%%Y /proc/$WORKER_PID'
-      STAT_MAC = 'ps -o lstart -p $WORKER_PID'
+      STAT_LINUX = "stat --format=%%Y /proc/$WORKER_PID"
+      STAT_MAC = "ps -o lstart -p $WORKER_PID"
       STAT = RUBY_PLATFORM =~ /darwin/ ? STAT_MAC : STAT_LINUX
       ALIVE_CHECK_LINUX = '[ -d "/proc/$WORKER_PID" ]'
-      ALIVE_CHECK_MAC = 'ps -p $WORKER_PID > /dev/null'
+      ALIVE_CHECK_MAC = "ps -p $WORKER_PID > /dev/null"
       ALIVE_CHECK = RUBY_PLATFORM =~ /darwin/ ? ALIVE_CHECK_MAC : ALIVE_CHECK_LINUX
-      SCRIPT_TEMPLATE = <<-BASH.freeze
+      SCRIPT_TEMPLATE = <<-BASH
         WORKER_PID="%<pid>d" # an example, filled from ruby when the check is created
         ORIGINAL_MTIME="%<mtime>s" # an example, filled from ruby when the check is created
 
@@ -31,19 +30,19 @@ module Delayed
 
       def self.mtime(pid)
         if RUBY_PLATFORM =~ /darwin/
-          `ps -o lstart -p #{pid}`.sub(/\n$/, '').presence
+          `ps -o lstart -p #{pid}`.sub(/\n$/, "").presence
         else
           File::Stat.new("/proc/#{pid}").mtime.to_i.to_s rescue nil
         end
       end
 
       def self.check_script(pid, mtime)
-        sprintf(SCRIPT_TEMPLATE, {pid: pid, mtime: mtime})
+        format(SCRIPT_TEMPLATE, { pid: pid, mtime: mtime })
       end
 
       def self.process_is_still_running?(pid, mtime)
-        system(self.check_script(pid, mtime))
+        system(check_script(pid, mtime))
       end
     end
   end
-end
+end

data/lib/delayed/worker.rb

@@ -1,257 +1,267 @@
 # frozen_string_literal: true
 
 module Delayed
-
-class TimeoutError < RuntimeError; end
-class RetriableError < RuntimeError
-  # this error is a special case.  You _should_ raise
-  # it from inside the rescue block for another error, 
-  # because it indicates: "something made this job fail
-  # but we're pretty sure it's transient and it's safe to try again".
-  # the workflow is still the same (retry will happen unless
-  # retries are exhausted), but it won't call the :error
-  # callback unless it can't retry anymore.  It WILL call the
-  # separate ":retry" callback, which is ONLY activated
-  # for this kind of error.
-end
-
-require 'tmpdir'
-require 'set'
-
-class Worker
-  include Delayed::Logging
-  SIGNALS = %i{INT TERM QUIT}
-
-  attr_reader :config, :queue_name, :min_priority, :max_priority, :work_queue
-
-  # Callback to fire when a delayed job fails max_attempts times. If this
-  # callback is defined, then the value of destroy_failed_jobs is ignored, and
-  # the job is destroyed if this block returns true.
-  #
-  # This allows for destroying "uninteresting" failures, while keeping around
-  # interesting failures to be investigated later.
-  #
-  # The block is called with args(job, last_exception)
-  def self.on_max_failures=(block)
-    @@on_max_failures = block
+  class TimeoutError < RuntimeError; end
+
+  class RetriableError < RuntimeError
+    # this error is a special case.  You _should_ raise
+    # it from inside the rescue block for another error,
+    # because it indicates: "something made this job fail
+    # but we're pretty sure it's transient and it's safe to try again".
+    # the workflow is still the same (retry will happen unless
+    # retries are exhausted), but it won't call the :error
+    # callback unless it can't retry anymore.  It WILL call the
+    # separate ":retry" callback, which is ONLY activated
+    # for this kind of error.
   end
-  cattr_reader :on_max_failures
 
-  cattr_accessor :plugins
-  self.plugins = Set.new
+  require "tmpdir"
+  require "set"
+
+  class Worker
+    include Delayed::Logging
+    SIGNALS = %i[INT TERM QUIT].freeze
+
+    attr_reader :config, :queue_name, :min_priority, :max_priority, :work_queue
+
+    class << self
+      # Callback to fire when a delayed job fails max_attempts times. If this
+      # callback is defined, then the value of destroy_failed_jobs is ignored, and
+      # the job is destroyed if this block returns true.
+      #
+      # This allows for destroying "uninteresting" failures, while keeping around
+      # interesting failures to be investigated later.
+      #
+      # The block is called with args(job, last_exception)
+      attr_accessor :on_max_failures
+    end
 
-  def self.lifecycle
-    @lifecycle ||= Delayed::Lifecycle.new
-  end
+    cattr_accessor :plugins
+    self.plugins = Set.new
 
-  def self.current_job
-    Thread.current[:running_delayed_job]
-  end
+    def self.lifecycle
+      @lifecycle ||= Delayed::Lifecycle.new
+    end
 
-  def self.running_job(job)
-    Thread.current[:running_delayed_job] = job
-    yield
-  ensure
-    Thread.current[:running_delayed_job] = nil
-  end
+    def self.current_job
+      Thread.current[:running_delayed_job]
+    end
 
-  def initialize(options = {})
-    @exit = false
-    @parent_pid = options[:parent_pid]
-    @queue_name = options[:queue] ||= Settings.queue
-    @min_priority = options[:min_priority]
-    @max_priority = options[:max_priority]
-    @max_job_count = options[:worker_max_job_count].to_i
-    @max_memory_usage = options[:worker_max_memory_usage].to_i
-    @work_queue = options.delete(:work_queue) || WorkQueue::InProcess.new
-    @health_check_type = Settings.worker_health_check_type
-    @health_check_config = Settings.worker_health_check_config
-    @config = options
-    @job_count = 0
-
-    @self_pipe = IO.pipe
-    @signal_queue = []
-
-    app = Rails.application
-    if app && !app.config.cache_classes
-      Delayed::Worker.lifecycle.around(:perform) do |worker, job, &block|
-        reload = app.config.reload_classes_only_on_change != true || app.reloaders.map(&:updated?).any?
-
-        if reload
-          if defined?(ActiveSupport::Reloader)
-            Rails.application.reloader.reload!
-          else
-            ActionDispatch::Reloader.prepare!
+    def self.running_job(job)
+      Thread.current[:running_delayed_job] = job
+      yield
+    ensure
+      Thread.current[:running_delayed_job] = nil
+    end
+
+    def initialize(options = {})
+      @exit = false
+      @parent_pid = options[:parent_pid]
+      @queue_name = options[:queue] ||= Settings.queue
+      @min_priority = options[:min_priority]
+      @max_priority = options[:max_priority]
+      @max_job_count = options[:worker_max_job_count].to_i
+      @max_memory_usage = options[:worker_max_memory_usage].to_i
+      @work_queue = options.delete(:work_queue) || WorkQueue::InProcess.new
+      @health_check_type = Settings.worker_health_check_type
+      @health_check_config = Settings.worker_health_check_config
+      @config = options
+      @job_count = 0
+
+      @signal_queue = []
+
+      app = Rails.application
+      if app && !app.config.cache_classes
+        Delayed::Worker.lifecycle.around(:perform) do |worker, job, &block|
+          reload = app.config.reload_classes_only_on_change != true || app.reloaders.map(&:updated?).any?
+
+          if reload
+            if defined?(ActiveSupport::Reloader)
+              Rails.application.reloader.reload!
+            else
+              ActionDispatch::Reloader.prepare!
+            end
           end
-        end
 
-        begin
-          block.call(worker, job)
-        ensure
-          ActionDispatch::Reloader.cleanup! if reload && !defined?(ActiveSupport::Reloader)
+          begin
+            block.call(worker, job)
+          ensure
+            ActionDispatch::Reloader.cleanup! if reload && !defined?(ActiveSupport::Reloader)
+          end
         end
       end
-    end
-
-    plugins.each { |plugin| plugin.inject! }
-  end
-
-  def name
-    @name ||= "#{Socket.gethostname rescue "X"}:#{self.id}"
-  end
 
-  def set_process_name(new_name)
-    $0 = "delayed:#{new_name}"
-  end
+      plugins.each(&:inject!)
+    end
 
-  def exit?
-    !!@exit || parent_exited?
-  end
+    def name
+      @name ||= "#{Socket.gethostname rescue 'X'}:#{id}"
+    end
 
-  def parent_exited?
-    @parent_pid && @parent_pid != Process.ppid
-  end
+    def process_name=(new_name)
+      $0 = "delayed:#{new_name}"
+    end
 
-  def wake_up
-    @self_pipe[1].write_nonblock('.', exception: false)
-    work_queue.wake_up
-  end
+    def exit?
+      !!@exit || parent_exited?
+    end
 
-  def start
-    logger.info "Starting worker"
-    set_process_name("start:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}")
+    def parent_exited?
+      @parent_pid && @parent_pid != Process.ppid
+    end
 
-    work_thread = Thread.current
-    SIGNALS.each do |sig|
-      trap(sig) { @signal_queue << sig; wake_up }
+    def wake_up
+      @self_pipe[1].write_nonblock(".", exception: false)
+      work_queue.wake_up
     end
 
-    raise 'Could not register health_check' unless health_check.start
-
-    signal_processor = Thread.new do
-      loop do
-        @self_pipe[0].read(1)
-        case @signal_queue.pop
-        when :INT, :TERM
-          @exit = true # get the main thread to bail early if it's waiting for a job
-          work_thread.raise(SystemExit) # Force the main thread to bail out of the current job
-          cleanup! # we're going to get SIGKILL'd in a moment, so clean up asap
-          break
-        when :QUIT
-          @exit = true
-        else
-          logger.error "Unknown signal '#{sig}' received"
+    def start
+      logger.info "Starting worker"
+      self.process_name =
+        "start:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}"
+      @self_pipe = IO.pipe
+      work_queue.init
+
+      work_thread = Thread.current
+      SIGNALS.each do |sig|
+        trap(sig) do
+          @signal_queue << sig
+          wake_up
         end
       end
-    end
 
-    self.class.lifecycle.run_callbacks(:execute, self) do
-      until exit? do
-        run
+      raise "Could not register health_check" unless health_check.start
+
+      signal_processor = Thread.new do
+        loop do
+          @self_pipe[0].read(1)
+          case @signal_queue.pop
+          when :INT, :TERM
+            @exit = true # get the main thread to bail early if it's waiting for a job
+            work_thread.raise(SystemExit) # Force the main thread to bail out of the current job
+            cleanup! # we're going to get SIGKILL'd in a moment, so clean up asap
+            break
+          when :QUIT
+            @exit = true
+          else
+            logger.error "Unknown signal '#{sig}' received"
+          end
+        end
+      end
+
+      self.class.lifecycle.run_callbacks(:execute, self) do
+        run until exit?
       end
-    end
 
-    logger.info "Stopping worker"
-  rescue => e
-    Rails.logger.fatal("Child process died: #{e.inspect}") rescue nil
-    self.class.lifecycle.run_callbacks(:exceptional_exit, self, e) { }
-  ensure
-    cleanup!
+      logger.info "Stopping worker"
+    rescue => e
+      Rails.logger.fatal("Child process died: #{e.inspect}") rescue nil
+      self.class.lifecycle.run_callbacks(:exceptional_exit, self, e) { nil }
+    ensure
+      cleanup!
 
-    if signal_processor
-      signal_processor.kill
-      signal_processor.join
+      if signal_processor
+        signal_processor.kill
+        signal_processor.join
+      end
+
+      @self_pipe&.each(&:close)
+      @self_pipe = nil
     end
-  end
 
-  def cleanup!
-    return if cleaned?
+    def cleanup!
+      return if cleaned?
 
-    health_check.stop
-    work_queue.close
-    Delayed::Job.clear_locks!(name)
+      health_check.stop
+      work_queue.close
+      Delayed::Job.clear_locks!(name)
 
-    @cleaned = true
-  end
+      @cleaned = true
+    end
 
-  def cleaned?
-    @cleaned
-  end
+    def cleaned?
+      @cleaned
+    end
 
-  def run
-    return if exit?
-    self.class.lifecycle.run_callbacks(:loop, self) do
-      set_process_name("pop:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}")
-      job = self.class.lifecycle.run_callbacks(:pop, self) do
-        work_queue.get_and_lock_next_available(name, config)
-      end
+    def run
+      return if exit?
 
-      if job
-        configure_for_job(job) do
-          @job_count += perform(job)
+      self.class.lifecycle.run_callbacks(:loop, self) do
+        self.process_name =
+          "pop:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}"
+        job = self.class.lifecycle.run_callbacks(:pop, self) do
+          work_queue.get_and_lock_next_available(name, config)
+        end
 
-          if @max_job_count > 0 && @job_count >= @max_job_count
-            logger.debug "Max job count of #{@max_job_count} exceeded, dying"
-            @exit = true
-          end
+        if job
+          configure_for_job(job) do
+            @job_count += perform(job)
 
-          if @max_memory_usage > 0
-            memory = sample_memory
-            if memory > @max_memory_usage
-              logger.debug "Memory usage of #{memory} exceeds max of #{@max_memory_usage}, dying"
+            if @max_job_count.positive? && @job_count >= @max_job_count
+              logger.debug "Max job count of #{@max_job_count} exceeded, dying"
               @exit = true
-            else
-              logger.debug "Memory usage: #{memory}"
+            end
+
+            if @max_memory_usage.positive?
+              memory = sample_memory
+              if memory > @max_memory_usage
+                logger.debug "Memory usage of #{memory} exceeds max of #{@max_memory_usage}, dying"
+                @exit = true
+              else
+                logger.debug "Memory usage: #{memory}"
+              end
             end
           end
+        else
+          self.process_name =
+            "wait:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}"
+          sleep(Settings.sleep_delay + (rand * Settings.sleep_delay_stagger)) unless exit?
         end
-      else
-        set_process_name("wait:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}")
-        sleep(Settings.sleep_delay + (rand * Settings.sleep_delay_stagger)) unless exit?
       end
     end
-  end
 
-  def perform(job)
-    begin
-      count = 1
-      raise Delayed::Backend::JobExpired, "job expired at #{job.expires_at}" if job.expired?
-      self.class.lifecycle.run_callbacks(:perform, self, job) do
-        set_process_name("run:#{Settings.worker_procname_prefix}#{job.id}:#{job.name}")
-        logger.info("Processing #{log_job(job, :long)}")
-        runtime = Benchmark.realtime do
-          if job.batch?
-            # each job in the batch will have perform called on it, so we don't
-            # need a timeout around this
-            count = perform_batch(job)
-          else
-            job.invoke_job
+    def perform(job)
+      begin
+        count = 1
+        raise Delayed::Backend::JobExpired, "job expired at #{job.expires_at}" if job.expired?
+
+        self.class.lifecycle.run_callbacks(:perform, self, job) do
+          self.process_name = "run:#{Settings.worker_procname_prefix}#{job.id}:#{job.name}"
+          logger.info("Processing #{log_job(job, :long)}")
+          runtime = Benchmark.realtime do
+            if job.batch?
+              # each job in the batch will have perform called on it, so we don't
+              # need a timeout around this
+              count = perform_batch(job)
+            else
+              job.invoke_job
+            end
+            job.destroy
           end
-          job.destroy
+          logger.info("Completed #{log_job(job)} #{format('%.0fms', (runtime * 1000))}")
+        end
+      rescue ::Delayed::RetriableError => e
+        can_retry = job.attempts + 1 < job.inferred_max_attempts
+        callback_type = can_retry ? :retry : :error
+        self.class.lifecycle.run_callbacks(callback_type, self, job, e) do
+          handle_failed_job(job, e)
+        end
+      rescue SystemExit => e
+        # There wasn't really a failure here so no callbacks and whatnot needed,
+        # still reschedule the job though.
+        job.reschedule(e)
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        self.class.lifecycle.run_callbacks(:error, self, job, e) do
+          handle_failed_job(job, e)
         end
-        logger.info("Completed #{log_job(job)} #{"%.0fms" % (runtime * 1000)}")
-      end
-    rescue ::Delayed::RetriableError => re
-      can_retry = job.attempts + 1 < job.inferred_max_attempts
-      callback_type = can_retry ? :retry : :error
-      self.class.lifecycle.run_callbacks(callback_type, self, job, re) do
-        handle_failed_job(job, re)
-      end
-    rescue SystemExit => se
-      # There wasn't really a failure here so no callbacks and whatnot needed,
-      # still reschedule the job though.
-      job.reschedule(se)
-    rescue Exception => e
-      self.class.lifecycle.run_callbacks(:error, self, job, e) do
-        handle_failed_job(job, e)
       end
+      count
     end
-    count
-  end
 
-  def perform_batch(parent_job)
-    batch = parent_job.payload_object
-    if batch.mode == :serial
+    def perform_batch(parent_job)
+      batch = parent_job.payload_object
+      return unless batch.mode == :serial
+
       batch.jobs.each do |job|
         job.source = parent_job.source
         job.create_and_lock!(name)
@@ -261,72 +271,72 @@ class Worker
       end
       batch.items.size
     end
-  end
 
-  def handle_failed_job(job, error)
-    job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
-    logger.error("Failed with #{error.class} [#{error.message}] (#{job.attempts} attempts)")
-    job.reschedule(error)
-  end
+    def handle_failed_job(job, error)
+      job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
+      logger.error("Failed with #{error.class} [#{error.message}] (#{job.attempts} attempts)")
+      job.reschedule(error)
+    end
 
-  def id
-    Process.pid
-  end
+    def id
+      Process.pid
+    end
 
-  def log_job(job, format = :short)
-    case format
-    when :long
-      "#{job.full_name} #{ Settings.job_detailed_log_format.call(job) }"
-    else
-      job.full_name
+    def log_job(job, format = :short)
+      case format
+      when :long
+        "#{job.full_name} #{Settings.job_detailed_log_format.call(job)}"
+      else
+        job.full_name
+      end
     end
-  end
 
-  # set up the session context information, so that it gets logged with the job log lines
-  # also set up a unique tmpdir, which will get removed at the end of the job.
-  def configure_for_job(job)
-    previous_tmpdir = ENV['TMPDIR']
+    # set up the session context information, so that it gets logged with the job log lines
+    # also set up a unique tmpdir, which will get removed at the end of the job.
+    def configure_for_job(job)
+      previous_tmpdir = ENV["TMPDIR"]
 
-    self.class.running_job(job) do
-      dir = Dir.mktmpdir("job-#{job.id}-#{self.name.gsub(/[^\w\.]/, '.')}-")
-      begin
-        ENV['TMPDIR'] = dir
-        yield
-      ensure
-        FileUtils.remove_entry(dir, true)
+      self.class.running_job(job) do
+        dir = Dir.mktmpdir("job-#{job.id}-#{name.gsub(/[^\w.]/, '.')}-")
+        begin
+          ENV["TMPDIR"] = dir
+          yield
+        ensure
+          FileUtils.remove_entry(dir, true)
+        end
       end
+    ensure
+      ENV["TMPDIR"] = previous_tmpdir
     end
-  ensure
-    ENV['TMPDIR'] = previous_tmpdir
-  end
-
-  def health_check
-    @health_check ||= HealthCheck.build(
-      type: @health_check_type,
-      worker_name: name,
-      config: @health_check_config
-    )
-  end
 
-  # `sample` reports KB, not B
-  if File.directory?("/proc")
-    # linux w/ proc fs
-    LINUX_PAGE_SIZE = (size = `getconf PAGESIZE`.to_i; size > 0 ? size : 4096)
-    def sample_memory
-      s = File.read("/proc/#{Process.pid}/statm").to_i rescue 0
-      s * LINUX_PAGE_SIZE / 1024
+    def health_check
+      @health_check ||= HealthCheck.build(
+        type: @health_check_type,
+        worker_name: name,
+        config: @health_check_config
+      )
     end
-  else
-    # generic unix solution
-    def sample_memory
-      if Rails.env.test?
-        0
-      else
-        # hmm this is actually resident set size, doesn't include swapped-to-disk
-        # memory.
-        `ps -o rss= -p #{Process.pid}`.to_i
+
+    # `sample` reports KB, not B
+    if File.directory?("/proc")
+      # linux w/ proc fs
+      LINUX_PAGE_SIZE = (size = `getconf PAGESIZE`.to_i
+                         size.positive? ? size : 4096)
+      def sample_memory
+        s = File.read("/proc/#{Process.pid}/statm").to_i rescue 0
+        s * LINUX_PAGE_SIZE / 1024
+      end
+    else
+      # generic unix solution
+      def sample_memory
+        if Rails.env.test?
+          0
+        else
+          # hmm this is actually resident set size, doesn't include swapped-to-disk
+          # memory.
+          `ps -o rss= -p #{Process.pid}`.to_i
+        end
       end
     end
   end
 end
-end