inst-jobs 2.3.1 → 2.4.8

data/lib/delayed/worker/consul_health_check.rb

@@ -1,23 +1,23 @@
 # frozen_string_literal: true
 
-require_relative 'health_check'
-require_relative 'process_helper'
-require 'socket'
+require_relative "health_check"
+require_relative "process_helper"
+require "socket"
 
 module Delayed
   class Worker
     class ConsulHealthCheck < HealthCheck
       self.type_name = :consul
 
-      CONSUL_CONFIG_KEYS = %w{url acl_token}.map(&:freeze).freeze
-      DEFAULT_SERVICE_NAME = 'inst-jobs_worker'.freeze
+      CONSUL_CONFIG_KEYS = %w[url acl_token].map(&:freeze).freeze
+      DEFAULT_SERVICE_NAME = "inst-jobs_worker"
       attr_reader :service_client, :health_client
 
       def initialize(*, **)
         super
         # Because we don't want the consul client to be a hard dependency we're
         # only requiring it once it's absolutely needed
-        require 'diplomat'
+        require "diplomat"
 
         if config.keys.any? { |k| CONSUL_CONFIG_KEYS.include?(k) }
           consul_config = Diplomat::Configuration.new.tap do |conf|
@@ -35,10 +35,10 @@ module Delayed
 
       def start
         @service_client.register({
-          id: worker_name,
-          name: service_name,
-          check: check_attributes
-        })
+                                   id: worker_name,
+                                   name: service_name,
+                                   check: check_attributes
+                                 })
       end
 
       def stop
@@ -48,27 +48,28 @@ module Delayed
       def live_workers
         # Filter out critical workers (probably nodes failing their serf health check)
         live_nodes = @health_client.service(service_name, {
-          filter: 'not Checks.Status == critical'
-        })
+                                              filter: "not Checks.Status == critical"
+                                            })
 
-        live_nodes.map { |n| n.Service['ID']}
+        live_nodes.map { |n| n.Service["ID"] }
       end
 
       private
 
       def check_attributes
         {
-          args: ['bash', '-c', check_script],
-          status: 'passing',
-          interval: @config.fetch(:check_interval, '5m'),
-          deregister_critical_service_after: @config.fetch(:deregister_service_delay, '10m'),
+          args: ["bash", "-c", check_script],
+          status: "passing",
+          interval: @config.fetch(:check_interval, "5m"),
+          deregister_critical_service_after: @config.fetch(:deregister_service_delay, "10m")
         }.tap do |h|
-          h[:docker_container_id] = docker_container_id if @config['docker']
+          h[:docker_container_id] = docker_container_id if @config["docker"]
         end
       end
 
       def check_script
         return @check_script if @check_script
+
         mtime = ProcessHelper.mtime(Process.pid)
         @check_script = ProcessHelper.check_script(Process.pid, mtime)
       end
@@ -77,12 +78,13 @@ module Delayed
       # cgroups for part of its magic and also uses the container id as the cgroup name
       def docker_container_id
         return @docker_container_id if @docker_container_id
+
         content = File.read("/proc/1/cgroup").split("\n")
         @docker_container_id = content.last.split("/").last
       end
 
       def service_name
-        @service_name ||= @config.fetch('service_name', DEFAULT_SERVICE_NAME)
+        @service_name ||= @config.fetch("service_name", DEFAULT_SERVICE_NAME)
       end
     end
   end

data/lib/delayed/worker/health_check.rb

@@ -11,58 +11,65 @@ module Delayed
 
         def inherited(subclass)
           @subclasses << subclass
+          super
         end
 
         def build(type:, worker_name:, config: {})
           type = type.to_sym
           klass = @subclasses.find { |sc| sc.type_name == type }
           raise ArgumentError, "Unable to build a HealthCheck for type #{type}" unless klass
+
           klass.new(worker_name: worker_name, config: config)
         end
 
         def reschedule_abandoned_jobs
           return if Settings.worker_health_check_type == :none
+
           Delayed::Job.transaction do
             # this action is a special case, and SHOULD NOT be a periodic job
             # because if it gets wiped out suddenly during execution
-            # it can't go clean up it's abandoned self.  Therefore,
+            # it can't go clean up its abandoned self.  Therefore,
             # we expect it to get run from it's own process forked from the job pool
             # and we try to get an advisory lock when it runs.  If we succeed,
             # no other worker is trying to do this right now (and if we abandon the
             # operation, the transaction will end, releasing the advisory lock).
-            result = attempt_advisory_lock
+            result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
             return unless result
+
+            horizon = 5.minutes.ago
+
             checker = Worker::HealthCheck.build(
               type: Settings.worker_health_check_type,
               config: Settings.worker_health_check_config,
-              worker_name: 'cleanup-crew'
+              worker_name: "cleanup-crew"
             )
             live_workers = checker.live_workers
 
-            Delayed::Job.running_jobs.each do |job|
-              # prefetched jobs have their own way of automatically unlocking themselves
-              next if job.locked_by.start_with?("prefetch:")
-              unless live_workers.include?(job.locked_by)
-                begin
-                  Delayed::Job.transaction do
-                    # double check that the job is still there. locked_by will immediately be reset
-                    # to nil in this transaction by Job#reschedule
-                    next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
-                    job.reschedule
-                  end
-                rescue
-                  ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
+            loop do
+              batch = Delayed::Job.running_jobs
+                                  .where("locked_at<?", horizon)
+                                  .where.not("locked_by LIKE 'prefetch:%'")
+                                  .where.not(locked_by: live_workers)
+                                  .limit(100)
+                                  .to_a
+              break if batch.empty?
+
+              batch.each do |job|
+                Delayed::Job.transaction do
+                  # double check that the job is still there. locked_by will immediately be reset
+                  # to nil in this transaction by Job#reschedule
+                  next unless Delayed::Job.where(id: job,
+                                                 locked_by: job.locked_by)
+                                          .update_all(locked_by: "abandoned job cleanup") == 1
+
+                  job.reschedule
                 end
               end
+            rescue
+              ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
             end
           end
         end
-
-        def attempt_advisory_lock
-          lock_name = "Delayed::Worker::HealthCheck#reschedule_abandoned_jobs"
-          conn = ActiveRecord::Base.connection
-          conn.select_value("SELECT pg_try_advisory_xact_lock(#{conn.quote_table_name('half_md5_as_bigint')}('#{lock_name}'));")
-        end
       end
 
       attr_accessor :config, :worker_name

data/lib/delayed/worker/null_health_check.rb

@@ -13,7 +13,9 @@ module Delayed
         true
       end
 
-      def live_workers; []; end
+      def live_workers
+        []
+      end
     end
   end
 end

data/lib/delayed/worker/process_helper.rb

@@ -3,14 +3,13 @@
 module Delayed
   class Worker
     module ProcessHelper
-
-      STAT_LINUX = 'stat --format=%%Y /proc/$WORKER_PID'
-      STAT_MAC = 'ps -o lstart -p $WORKER_PID'
+      STAT_LINUX = "stat --format=%%Y /proc/$WORKER_PID"
+      STAT_MAC = "ps -o lstart -p $WORKER_PID"
       STAT = RUBY_PLATFORM =~ /darwin/ ? STAT_MAC : STAT_LINUX
       ALIVE_CHECK_LINUX = '[ -d "/proc/$WORKER_PID" ]'
-      ALIVE_CHECK_MAC = 'ps -p $WORKER_PID > /dev/null'
+      ALIVE_CHECK_MAC = "ps -p $WORKER_PID > /dev/null"
       ALIVE_CHECK = RUBY_PLATFORM =~ /darwin/ ? ALIVE_CHECK_MAC : ALIVE_CHECK_LINUX
-      SCRIPT_TEMPLATE = <<-BASH.freeze
+      SCRIPT_TEMPLATE = <<-BASH
         WORKER_PID="%<pid>d" # an example, filled from ruby when the check is created
         ORIGINAL_MTIME="%<mtime>s" # an example, filled from ruby when the check is created
 
@@ -31,19 +30,19 @@ module Delayed
 
       def self.mtime(pid)
         if RUBY_PLATFORM =~ /darwin/
-          `ps -o lstart -p #{pid}`.sub(/\n$/, '').presence
+          `ps -o lstart -p #{pid}`.sub(/\n$/, "").presence
         else
           File::Stat.new("/proc/#{pid}").mtime.to_i.to_s rescue nil
         end
       end
 
       def self.check_script(pid, mtime)
-        sprintf(SCRIPT_TEMPLATE, {pid: pid, mtime: mtime})
+        format(SCRIPT_TEMPLATE, { pid: pid, mtime: mtime })
       end
 
       def self.process_is_still_running?(pid, mtime)
-        system(self.check_script(pid, mtime))
+        system(check_script(pid, mtime))
       end
     end
   end
-end
+end