ductwork 0.25.0 → 1.0.0

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -3,7 +3,7 @@
 module Ductwork
   module Processes
     class PipelineAdvancer
-      attr_reader :thread, :last_heartbeat_at, :pipeline
+      attr_reader :thread, :last_heartbeat_at, :branch
 
       def initialize(klass, index = nil)
         @klass = klass
@@ -18,12 +18,19 @@ module Ductwork
         @thread.name = name
       end
 
-      alias restart start
+      def restart
+        cleanup_dead_thread!
+        start
+      end
 
       def alive?
         thread&.alive? || false
       end
 
+      def stuck?
+        branch.nil? && last_heartbeat_too_long_ago?
+      end
+
       def stop
         running_context.shutdown!
       end
@@ -43,9 +50,9 @@ module Ductwork
 
       private
 
-      attr_reader :klass, :index, :running_context
+      attr_reader :klass, :index, :running_context, :original_claim_token
 
-      def work_loop # rubocop:todo Metrics
+      def work_loop
         run_hooks_for(:start)
 
         Ductwork.logger.debug(
@@ -55,85 +62,27 @@ module Ductwork
         )
 
         while running_context.running?
-          id = Ductwork.wrap_with_app_executor do
-            Ductwork::Pipeline
-              .in_progress
-              .where(klass: klass, claimed_for_advancing_at: nil)
-              .where(steps: Ductwork::Step.where(status: :advancing))
-              .where.not(steps: Ductwork::Step.where.not(status: %w[advancing completed]))
-              .order(:last_advanced_at)
-              .limit(1)
-              .pluck(:id)
-              .first
-          end
+          claimed = false
 
-          if id.present?
-            rows_updated = Ductwork.wrap_with_app_executor do
-              Ductwork::Pipeline
-                .where(id: id, claimed_for_advancing_at: nil)
-                .update_all(
-                  claimed_for_advancing_at: Time.current,
-                  status: "advancing"
-                )
-            end
+          Ductwork.wrap_with_app_executor do
+            claimed = Branch.with_latest_claimed(klass) do |branch, transition, advancement|
+              @branch = branch
+              @original_claim_token = branch.claim_token
+
+              branch.advance!(transition, advancement)
 
-            if rows_updated == 1
-              Ductwork.logger.debug(
-                msg: "Pipeline claimed",
-                pipeline_id: id,
-                pipeline: klass,
-                role: :pipeline_advancer
-              )
-
-              @pipeline = Ductwork.wrap_with_app_executor do
-                @pipeline = Ductwork::Pipeline.find(id)
-                pipeline.advance!
-
-                Ductwork.logger.debug(
-                  msg: "Pipeline advanced",
-                  pipeline_id: id,
-                  pipeline: klass,
-                  role: :pipeline_advancer
-                )
-
-                # rubocop:todo Metrics/BlockNesting
-                status = if pipeline.completed?
-                           "completed"
-                         elsif pipeline.dampened?
-                           "dampened"
-                         else
-                           "in_progress"
-                         end
-                # rubocop:enable Metrics/BlockNesting
-              ensure
-                # release the pipeline and set last advanced at so it doesn't
-                # block. we're not using a queue so we have to use a db
-                # timestamp
-                pipeline.update!(
-                  claimed_for_advancing_at: nil,
-                  last_advanced_at: Time.current,
-                  status: status || "in_progress"
-                )
-              end
-            else
-              Ductwork.logger.debug(
-                msg: "Did not claim pipeline, avoided race condition",
-                pipeline_id: id,
-                pipeline: klass,
-                role: :pipeline_advancer
-              )
+              Ductwork::FaultInjection.checkpoint(:after_branch_advancement)
+            ensure
+              @branch = nil
+              @original_claim_token = nil
             end
-          else
-            Ductwork.logger.debug(
-              msg: "No pipeline needs advancing",
-              pipeline: klass,
-              role: :pipeline_advancer
-            )
           end
 
           @last_heartbeat_at = Time.current
 
-          sleep(polling_timeout)
+          if !claimed
+            sleep(polling_timeout)
+          end
         end
 
         Ductwork.logger.debug(
@@ -145,6 +94,29 @@ module Ductwork
         run_hooks_for(:stop)
       end
 
+      def cleanup_dead_thread!
+        if branch.present?
+          advancement = branch
+                        .transitions
+                        .joins(:advancements)
+                        .where(ductwork_advancements: { completed_at: nil })
+                        .first
+                        &.advancements
+                        &.find_by(completed_at: nil)
+
+          advancement&.thread_crashed!(original_claim_token)
+        end
+      ensure
+        @branch = nil
+        @original_claim_token = nil
+      end
+
+      def last_heartbeat_too_long_ago?
+        threshold = Ductwork.configuration.supervisor_reaper_timeout
+
+        (Time.current - last_heartbeat_at) > threshold
+      end
+
       def run_hooks_for(event)
         Ductwork.hooks[:advancer].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -24,7 +24,7 @@ module Ductwork
       end
 
       def run
-        create_process!
+        adopt_or_create_process!
         start_pipeline_advancers
 
         Ductwork.logger.debug(
@@ -36,7 +36,7 @@ module Ductwork
         while running_context.running?
           # TODO: Increase or make configurable
           sleep(5)
-          check_thread_health
+          check_workers_health
           report_heartbeat!
         end
 
@@ -49,60 +49,53 @@ module Ductwork
 
       def start_pipeline_advancers
         klasses.each do |klass|
-          advancer = Ductwork::Processes::PipelineAdvancer.new(klass)
-          advancers.push(advancer)
-          advancer.start
-
-          Ductwork.logger.debug(
-            msg: "Created new pipeline advancer",
-            role: :pipeline_advancer_runner,
-            pipeline: klass,
-            thread: advancer.name
-          )
-        end
-      end
+          Ductwork.configuration.pipeline_advancer_count(klass).times do |index|
+            advancer = Ductwork::Processes::PipelineAdvancer.new(klass, index)
+            advancers.push(advancer)
+            advancer.start
 
-      def check_thread_health
-        Ductwork.logger.debug(
-          msg: "Checking threads health",
-          role: :pipeline_advancer_runner,
-          pipelines: klasses
-        )
-        advancers.each do |advancer|
-          if !advancer.alive?
-            advancer.restart
-
-            Ductwork.logger.warn(
-              msg: "Restarted pipeline advancer",
+            Ductwork.logger.debug(
+              msg: "Created new pipeline advancer",
               role: :pipeline_advancer_runner,
-              pipeline: advancer.pipeline.class.to_s,
+              pipeline: klass,
               thread: advancer.name
             )
           end
         end
-        Ductwork.logger.debug(
-          msg: "Checked thread health",
-          role: :pipeline_advancer_runner,
-          pipelines: klasses
+      end
+
+      def check_workers_health
+        Ductwork::Processes::WorkerHealthCheck
+          .new(advancers, :pipeline_advancer_runner)
+          .check
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Checking worker health failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :pipeline_advancer_runner
         )
       end
 
-      def create_process!
+      def adopt_or_create_process!
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.create!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch,
-            last_heartbeat_at: Time.current
-          )
+          Ductwork::Process.adopt_or_create_current!(:pipeline_advancer)
         end
       end
 
       def report_heartbeat!
         Ductwork.logger.debug(msg: "Reporting heartbeat", role: :pipeline_advancer_runner)
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.report_heartbeat!
+          Ductwork::Process.report_heartbeat!(:pipeline_advancer)
         end
         Ductwork.logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reporting heartbeat failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :pipeline_advancer_runner
+        )
       end
 
       def shutdown!
@@ -111,7 +104,7 @@ module Ductwork
         advancers.each(&:stop)
         await_threads_graceful_shutdown
         kill_remaining_threads
-        delete_process!
+        reap_process_record!
       end
 
       def log_shutting_down
@@ -154,12 +147,9 @@ module Ductwork
         end
       end
 
-      def delete_process!
+      def reap_process_record!
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.find_by!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch
-          ).delete
+          Ductwork::Process.current&.reap!(:pipeline_advancer_runner, force: true)
         end
       end
     end

data/lib/ductwork/processes/process_supervisor.rb

@@ -2,7 +2,12 @@
 
 module Ductwork
   module Processes
-    class ProcessSupervisor
+    class ProcessSupervisor # rubocop:todo Metrics/ClassLength
+      # Upper bound on how long sig_kill_process will block the supervisor
+      # loop waiting for a KILL'd child to be reaped (see sig_kill_process).
+      KILL_REAP_TIMEOUT = 5 # seconds
+      KILL_REAP_INTERVAL = 0.1
+
       attr_reader :workers
 
       def initialize
@@ -29,6 +34,8 @@ module Ductwork
       end
 
       def run
+        adopt_or_create_process!
+
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :process_supervisor,
@@ -38,6 +45,7 @@ module Ductwork
         while running_context.running?
           sleep(Ductwork.configuration.supervisor_polling_timeout)
           check_workers
+          reap_process_records
         end
 
         shutdown
@@ -60,6 +68,12 @@ module Ductwork
 
       attr_reader :running_context
 
+      def adopt_or_create_process!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.adopt_or_create_current!(:supervisor)
+        end
+      end
+
       def check_workers
         Ductwork.logger.debug(
           msg: "Checking workers are alive",
@@ -67,8 +81,10 @@ module Ductwork
         )
 
         workers.each do |worker|
-          if process_dead?(worker[:pid])
+          if heartbeat_stale?(worker[:pid])
             old_pid = worker[:pid]
+            sig_kill_process(old_pid)
+            reap_process_record!(old_pid)
             new_pid = fork do
               worker[:block].call(worker[:metadata])
             end
@@ -86,6 +102,27 @@ module Ductwork
           msg: "All workers are alive or revived",
           role: :process_supervisor
         )
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Checking workers failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :process_supervisor
+        )
+      end
+
+      def reap_process_records
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.reap_all!(:process_supervisor)
+          Ductwork::Process.sweep_orphaned_claims!(:process_supervisor)
+        end
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reaping process records failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :process_supervisor
+        )
       end
 
       def terminate_gracefully
@@ -97,6 +134,8 @@ module Ductwork
             signal: :TERM
           )
           ::Process.kill(:TERM, worker[:pid])
+        rescue Errno::ESRCH
+          # Child already exited; nothing to terminate
         end
       end
 
@@ -121,38 +160,76 @@ module Ductwork
 
       def terminate_immediately
         workers.each_with_index do |worker, index|
-          Ductwork.logger.debug(
+          Ductwork.logger.warn(
             msg: "Sending KILL signal to process (#{worker[:pid]})",
             role: :process_supervisor,
             pid: worker[:pid],
             signal: :KILL
           )
-          ::Process.kill(:KILL, worker[:pid])
-          ::Process.wait(worker[:pid])
+          sig_kill_process(worker[:pid])
+          reap_process_record!(worker[:pid])
           workers[index] = nil
-          Ductwork.logger.debug(
+          Ductwork.logger.warn(
             msg: "Child process (#{worker[:pid]}) killed after timeout",
             role: :process_supervisor,
             pid: worker[:pid]
           )
-        rescue Errno::ESRCH, Errno::ECHILD
-          # no-op because process is already dead
         end
 
         @workers = workers.compact
       end
 
-      def process_dead?(pid)
+      def heartbeat_stale?(pid)
         machine_identifier = Ductwork::MachineIdentifier.fetch
+        threshold = Ductwork.configuration.supervisor_reaper_timeout - 10.seconds
+        sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", threshold)
 
         Ductwork.wrap_with_app_executor do
           Ductwork::Process
             .where(pid:, machine_identifier:)
-            .where("last_heartbeat_at < ?", 5.minutes.ago)
+            .where(sql)
             .exists?
         end
       end
 
+      def sig_kill_process(pid)
+        ::Process.kill(:KILL, pid)
+
+        deadline = now + KILL_REAP_TIMEOUT
+        loop do
+          return if ::Process.wait(pid, ::Process::WNOHANG)
+
+          if now >= deadline
+            # Likely wedged in uninterruptible sleep (D state): the SIGKILL is
+            # pending but undeliverable until its syscall returns. Don't block
+            # the supervisor loop on it - the kill still applies, but the OS
+            # process is left unreaped (a zombie) until the supervisor itself
+            # exits and init reaps it. The DB process record is reaped
+            # separately by the caller.
+            Ductwork.logger.warn(
+              msg: "Process (#{pid}) did not exit after KILL; deferring reap",
+              role: :process_supervisor,
+              pid: pid
+            )
+            return
+          end
+
+          sleep(KILL_REAP_INTERVAL)
+        end
+      rescue Errno::ECHILD, Errno::ESRCH
+        # Process is already dead or reaped
+      end
+
+      def reap_process_record!(pid)
+        machine_identifier = Ductwork::MachineIdentifier.fetch
+
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process
+            .find_by(pid:, machine_identifier:)
+            &.reap!(:process_supervisor, force: true)
+        end
+      end
+
       def run_hooks_for(event)
         Ductwork.hooks[:supervisor].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/thread_supervisor.rb

@@ -9,6 +9,7 @@ module Ductwork
         @running_context = Ductwork::RunningContext.new
         @workers = []
 
+        adopt_or_create_process!
         run_hooks_for(:start)
 
         Signal.trap(:INT) { @running_context.shutdown! }
@@ -48,7 +49,9 @@ module Ductwork
 
         while running_context.running?
           sleep(Ductwork.configuration.supervisor_polling_timeout)
-          check_worker_health
+          check_workers_health
+          report_heartbeat!
+          reap_process_records
         end
 
         shutdown
@@ -58,26 +61,15 @@ module Ductwork
 
       attr_reader :running_context
 
-      def check_worker_health
-        Ductwork.logger.debug(
-          msg: "Checking workers are alive",
-          role: :thread_supervisor
-        )
-
-        workers.each do |worker|
-          if !worker.alive?
-            worker.restart
-
-            Ductwork.logger.warn(
-              msg: "Restarted supervised thread",
-              role: :thread_supervisor,
-              thread: worker.name
-            )
-          end
-        end
-
-        Ductwork.logger.debug(
-          msg: "Checked workers are alive",
+      def check_workers_health
+        Ductwork::Processes::WorkerHealthCheck
+          .new(workers, :thread_supervisor)
+          .check
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Checking worker health failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
           role: :thread_supervisor
         )
       end
@@ -88,6 +80,7 @@ module Ductwork
         workers.each(&:stop)
         await_threads_graceful_shutdown
         kill_remaining_threads
+        reap_process_record!
         run_hooks_for(:stop)
       end
 
@@ -130,6 +123,47 @@ module Ductwork
         end
       end
 
+      def adopt_or_create_process!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.adopt_or_create_current!(:supervisor)
+        end
+      end
+
+      def report_heartbeat!
+        Ductwork.logger.debug(msg: "Reporting heartbeat", role: :thread_supervisor)
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.report_heartbeat!(:supervisor)
+        end
+        Ductwork.logger.debug(msg: "Reported heartbeat", role: :thread_supervisor)
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reporting heartbeat failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :thread_supervisor
+        )
+      end
+
+      def reap_process_records
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.reap_all!(:thread_supervisor)
+          Ductwork::Process.sweep_orphaned_claims!(:thread_supervisor)
+        end
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reaping process records failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :thread_supervisor
+        )
+      end
+
+      def reap_process_record!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.current&.reap!(:thread_supervisor, force: true)
+        end
+      end
+
       def run_hooks_for(event)
         Ductwork.hooks[:supervisor].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/thread_supervisor_runner.rb

@@ -11,9 +11,11 @@ module Ductwork
       def run
         if Ductwork.configuration.role.in?(%w[all advancer])
           pipelines.each do |pipeline|
-            log_created_pipeline_advancer(pipeline)
-            supervisor.add_worker(metadata: { pipeline: }) do
-              pipeline_advancer.new(pipeline)
+            Ductwork.configuration.pipeline_advancer_count(pipeline).times do |index|
+              log_created_pipeline_advancer(pipeline, index)
+              supervisor.add_worker(metadata: { pipeline: }) do
+                pipeline_advancer.new(pipeline, index)
+              end
             end
           end
         end
@@ -36,11 +38,12 @@ module Ductwork
 
       attr_reader :pipelines, :supervisor
 
-      def log_created_pipeline_advancer(pipeline)
+      def log_created_pipeline_advancer(pipeline, index)
         Ductwork.logger.debug(
           msg: "Created new pipeline advancer",
           role: :thread_supervisor_runner,
-          pipeline: pipeline
+          pipeline: pipeline,
+          index: index
         )
       end
 

data/lib/ductwork/processes/worker_health_check.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class WorkerHealthCheck
+      def initialize(workers, role)
+        @workers = workers
+        @role = role
+      end
+
+      def check
+        workers.each do |worker|
+          if !worker.alive?
+            restart_dead_worker(worker)
+          elsif worker.stuck?
+            restart_stuck_worker(worker)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :workers, :role
+
+      def restart_dead_worker(worker)
+        worker.restart
+
+        claimed_args = if worker.is_a?(Ductwork::Processes::PipelineAdvancer)
+                         { branch_id: worker.branch&.id }
+                       else
+                         { job_id: worker.execution&.job_id }
+                       end
+
+        Ductwork.logger.warn(
+          msg: "Restarted dead thread",
+          role: role,
+          thread: worker.name,
+          **claimed_args
+        )
+      end
+
+      def restart_stuck_worker(worker)
+        worker.kill if worker.alive?
+        worker.join(1)
+        worker.restart
+
+        Ductwork.logger.warn(
+          msg: "Killed and restarted stuck thread",
+          role: role,
+          thread: worker.name
+        )
+      end
+    end
+  end
+end