inst-jobs 2.0.0 → 3.0.0

data/lib/delayed/worker/consul_health_check.rb

@@ -1,77 +1,75 @@
 # frozen_string_literal: true
 
-require_relative 'health_check'
-require_relative 'process_helper'
-require 'socket'
+require_relative "health_check"
+require_relative "process_helper"
+require "socket"
 
 module Delayed
   class Worker
     class ConsulHealthCheck < HealthCheck
       self.type_name = :consul
 
-      CONSUL_CONFIG_KEYS = %w{url host port ssl token connect_timeout receive_timeout send_timeout}.map(&:freeze).freeze
-      DEFAULT_SERVICE_NAME = 'inst-jobs_worker'.freeze
-      attr_reader :agent_client, :catalog_client
+      CONSUL_CONFIG_KEYS = %w[url acl_token].map(&:freeze).freeze
+      DEFAULT_SERVICE_NAME = "inst-jobs_worker"
+      attr_reader :service_client, :health_client
 
       def initialize(*, **)
         super
         # Because we don't want the consul client to be a hard dependency we're
         # only requiring it once it's absolutely needed
-        require 'imperium'
+        require "diplomat"
 
         if config.keys.any? { |k| CONSUL_CONFIG_KEYS.include?(k) }
-          consul_config = Imperium::Configuration.new.tap do |conf|
+          consul_config = Diplomat::Configuration.new.tap do |conf|
             CONSUL_CONFIG_KEYS.each do |key|
               conf.send("#{key}=", config[key]) if config[key]
             end
           end
-          @agent_client = Imperium::Agent.new(consul_config)
-          @catalog_client = Imperium::Catalog.new(consul_config)
+          @service_client = Diplomat::Service.new(configuration: consul_config)
+          @health_client = Diplomat::Health.new(configuration: consul_config)
         else
-          @agent_client = Imperium::Agent.default_client
-          @catalog_client = Imperium::Catalog.default_client
+          @service_client = Diplomat::Service.new
+          @health_client = Diplomat::Health.new
         end
       end
 
       def start
-        service = Imperium::Service.new({
-          id: worker_name,
-          name: service_name,
-        })
-        service.add_check(check_attributes)
-        response = @agent_client.register_service(service)
-        response.ok?
+        @service_client.register({
+                                   id: worker_name,
+                                   name: service_name,
+                                   check: check_attributes
+                                 })
       end
 
       def stop
-        response = @agent_client.deregister_service(worker_name)
-        response.ok? || response.not_found?
+        @service_client.deregister(worker_name)
       end
 
       def live_workers
-        live_nodes = @catalog_client.list_nodes_for_service(service_name)
-        if live_nodes.ok?
-          live_nodes.map(&:service_id)
-        else
-          raise "Unable to read from Consul catalog: #{live_nodes.content}"
-        end
+        # Filter out critical workers (probably nodes failing their serf health check)
+        live_nodes = @health_client.service(service_name, {
+                                              filter: "not Checks.Status == critical"
+                                            })
+
+        live_nodes.map { |n| n.Service["ID"] }
       end
 
       private
 
       def check_attributes
         {
-          args: ['bash', '-c', check_script],
-          status: 'passing',
-          interval: @config.fetch(:check_interval, '5m'),
-          deregister_critical_service_after: @config.fetch(:deregister_service_delay, '10m'),
+          args: ["bash", "-c", check_script],
+          status: "passing",
+          interval: @config.fetch(:check_interval, "5m"),
+          deregister_critical_service_after: @config.fetch(:deregister_service_delay, "10m")
         }.tap do |h|
-          h[:docker_container_id] = docker_container_id if @config['docker']
+          h[:docker_container_id] = docker_container_id if @config["docker"]
         end
       end
 
       def check_script
         return @check_script if @check_script
+
         mtime = ProcessHelper.mtime(Process.pid)
         @check_script = ProcessHelper.check_script(Process.pid, mtime)
       end
@@ -80,12 +78,13 @@ module Delayed
       # cgroups for part of its magic and also uses the container id as the cgroup name
       def docker_container_id
         return @docker_container_id if @docker_container_id
+
         content = File.read("/proc/1/cgroup").split("\n")
         @docker_container_id = content.last.split("/").last
       end
 
       def service_name
-        @service_name ||= @config.fetch('service_name', DEFAULT_SERVICE_NAME)
+        @service_name ||= @config.fetch("service_name", DEFAULT_SERVICE_NAME)
       end
     end
   end

data/lib/delayed/worker/health_check.rb

@@ -11,57 +11,65 @@ module Delayed
 
         def inherited(subclass)
           @subclasses << subclass
+          super
         end
 
         def build(type:, worker_name:, config: {})
           type = type.to_sym
           klass = @subclasses.find { |sc| sc.type_name == type }
           raise ArgumentError, "Unable to build a HealthCheck for type #{type}" unless klass
+
           klass.new(worker_name: worker_name, config: config)
         end
 
         def reschedule_abandoned_jobs
           return if Settings.worker_health_check_type == :none
+
           Delayed::Job.transaction do
-            # this job is a special case, and is not a singleton
+            # this action is a special case, and SHOULD NOT be a periodic job
             # because if it gets wiped out suddenly during execution
-            # it can't go clean up it's abandoned self.  Therefore,
-            # we try to get an advisory lock when it runs.  If we succeed,
-            # no other job is trying to do this right now (and if we abandon the
-            # job, the transaction will end, releasing the advisory lock).
-            result = attempt_advisory_lock
+            # it can't go clean up its abandoned self.  Therefore,
+            # we expect it to get run from it's own process forked from the job pool
+            # and we try to get an advisory lock when it runs.  If we succeed,
+            # no other worker is trying to do this right now (and if we abandon the
+            # operation, the transaction will end, releasing the advisory lock).
+            result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
             return unless result
+
+            horizon = 5.minutes.ago
+
             checker = Worker::HealthCheck.build(
               type: Settings.worker_health_check_type,
               config: Settings.worker_health_check_config,
-              worker_name: 'cleanup-crew'
+              worker_name: "cleanup-crew"
             )
             live_workers = checker.live_workers
 
-            Delayed::Job.running_jobs.each do |job|
-              # prefetched jobs have their own way of automatically unlocking themselves
-              next if job.locked_by.start_with?("prefetch:")
-              unless live_workers.include?(job.locked_by)
-                begin
-                  Delayed::Job.transaction do
-                    # double check that the job is still there. locked_by will immediately be reset
-                    # to nil in this transaction by Job#reschedule
-                    next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
-                    job.reschedule
-                  end
-                rescue
-                  ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
+            loop do
+              batch = Delayed::Job.running_jobs
+                                  .where("locked_at<?", horizon)
+                                  .where.not("locked_by LIKE 'prefetch:%'")
+                                  .where.not(locked_by: live_workers)
+                                  .limit(100)
+                                  .to_a
+              break if batch.empty?
+
+              batch.each do |job|
+                Delayed::Job.transaction do
+                  # double check that the job is still there. locked_by will immediately be reset
+                  # to nil in this transaction by Job#reschedule
+                  next unless Delayed::Job.where(id: job,
+                                                 locked_by: job.locked_by)
+                                          .update_all(locked_by: "abandoned job cleanup") == 1
+
+                  job.reschedule
                 end
               end
+            rescue
+              ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
             end
           end
         end
-
-        def attempt_advisory_lock
-          lock_name = "Delayed::Worker::HealthCheck#reschedule_abandoned_jobs"
-          output = ActiveRecord::Base.connection.execute("SELECT pg_try_advisory_xact_lock(half_md5_as_bigint('#{lock_name}'));")
-          output.getvalue(0, 0)
-        end
       end
 
       attr_accessor :config, :worker_name

data/lib/delayed/worker/null_health_check.rb

@@ -13,7 +13,9 @@ module Delayed
         true
       end
 
-      def live_workers; []; end
+      def live_workers
+        []
+      end
     end
   end
 end

data/lib/delayed/worker/process_helper.rb

@@ -3,14 +3,13 @@
 module Delayed
   class Worker
     module ProcessHelper
-
-      STAT_LINUX = 'stat --format=%%Y /proc/$WORKER_PID'
-      STAT_MAC = 'ps -o lstart -p $WORKER_PID'
+      STAT_LINUX = "stat --format=%%Y /proc/$WORKER_PID"
+      STAT_MAC = "ps -o lstart -p $WORKER_PID"
       STAT = RUBY_PLATFORM =~ /darwin/ ? STAT_MAC : STAT_LINUX
       ALIVE_CHECK_LINUX = '[ -d "/proc/$WORKER_PID" ]'
-      ALIVE_CHECK_MAC = 'ps -p $WORKER_PID > /dev/null'
+      ALIVE_CHECK_MAC = "ps -p $WORKER_PID > /dev/null"
       ALIVE_CHECK = RUBY_PLATFORM =~ /darwin/ ? ALIVE_CHECK_MAC : ALIVE_CHECK_LINUX
-      SCRIPT_TEMPLATE = <<-BASH.freeze
+      SCRIPT_TEMPLATE = <<-BASH
         WORKER_PID="%<pid>d" # an example, filled from ruby when the check is created
         ORIGINAL_MTIME="%<mtime>s" # an example, filled from ruby when the check is created
 
@@ -31,19 +30,19 @@ module Delayed
 
       def self.mtime(pid)
         if RUBY_PLATFORM =~ /darwin/
-          `ps -o lstart -p #{pid}`.sub(/\n$/, '').presence
+          `ps -o lstart -p #{pid}`.sub(/\n$/, "").presence
         else
           File::Stat.new("/proc/#{pid}").mtime.to_i.to_s rescue nil
         end
       end
 
       def self.check_script(pid, mtime)
-        sprintf(SCRIPT_TEMPLATE, {pid: pid, mtime: mtime})
+        format(SCRIPT_TEMPLATE, { pid: pid, mtime: mtime })
       end
 
       def self.process_is_still_running?(pid, mtime)
-        system(self.check_script(pid, mtime))
+        system(check_script(pid, mtime))
       end
     end
   end
-end
+end