ductwork 0.26.0 → 1.1.0

data/lib/ductwork/processes/job_worker.rb

@@ -3,7 +3,7 @@
 module Ductwork
   module Processes
     class JobWorker
-      attr_reader :thread, :last_heartbeat_at, :job, :pipeline
+      attr_reader :thread, :last_heartbeat_at, :execution, :pipeline
 
       def initialize(pipeline, id)
         @pipeline = pipeline
@@ -18,12 +18,19 @@ module Ductwork
         @thread.name = name
       end
 
-      alias restart start
+      def restart
+        cleanup_dead_thread!
+        start
+      end
 
       def alive?
         thread&.alive? || false
       end
 
+      def stuck?
+        execution.nil? && last_heartbeat_too_long_ago?
+      end
+
       def stop
         running_context.shutdown!
       end
@@ -45,7 +52,7 @@ module Ductwork
 
       attr_reader :id, :running_context
 
-      def work_loop
+      def work_loop # rubocop:todo Metrics
         run_hooks_for(:start)
 
         Ductwork.logger.debug(
@@ -55,29 +62,74 @@ module Ductwork
         )
 
         while running_context.running?
-          Ductwork.logger.debug(
-            msg: "Attempting to claim job",
-            role: :job_worker,
-            pipeline: pipeline
-          )
-
-          @job = Ductwork.wrap_with_app_executor do
-            Job.claim_latest(pipeline)
-          end
+          owner_process_id = nil
 
-          if job.present?
-            Ductwork.wrap_with_app_executor do
-              job.execute(pipeline)
+          begin
+            Ductwork.logger.debug(
+              msg: "Attempting to claim job",
+              role: :job_worker,
+              pipeline: pipeline
+            )
+
+            @execution = Ductwork.wrap_with_app_executor do
+              owner_process_id = Ductwork::Process.current.id
+              Ductwork::ExecutionClaim.new(pipeline, owner_process_id).latest
             end
 
-            @job = nil
-          else
-            Ductwork.logger.debug(
-              msg: "No job to claim, looping",
+            if execution.present?
+              Ductwork::FaultInjection.checkpoint(:after_job_claim)
+
+              Ductwork.wrap_with_app_executor do
+                execution.call(pipeline, owner_process_id)
+              end
+
+              @execution = nil
+            else
+              Ductwork.logger.debug(
+                msg: "No job to claim, looping",
+                role: :job_worker,
+                pipeline: pipeline
+              )
+              sleep(polling_timeout)
+            end
+          rescue Ductwork::Execution::CommitFailed => e
+            Ductwork.logger.error(
+              msg: "Execution commit failed",
+              error_klass: e.class.name,
+              error_message: e.message,
+              job_id: execution&.job_id,
+              job_klass: execution&.job&.klass,
               role: :job_worker,
               pipeline: pipeline
             )
-            sleep(polling_timeout)
+
+            @execution = nil
+          rescue StandardError => e
+            if execution.present?
+              Ductwork.wrap_with_app_executor do
+                execution.crashed!
+              end
+            end
+
+            Ductwork.logger.error(
+              msg: "Unexpected error in work loop",
+              error_klass: e.class.name,
+              error_message: e.message,
+              job_id: execution&.job_id,
+              job_klass: execution&.job&.klass,
+              role: :job_worker,
+              pipeline: pipeline
+            )
+          ensure
+            if execution.present? && execution.reload.completed_at.nil?
+              Ductwork.wrap_with_app_executor do
+                execution.crashed!
+              rescue StandardError
+                nil
+              end
+            end
+
+            @execution = nil
           end
 
           @last_heartbeat_at = Time.current
@@ -92,6 +144,20 @@ module Ductwork
         run_hooks_for(:stop)
       end
 
+      def cleanup_dead_thread!
+        if execution.present? && execution.reload.completed_at.nil?
+          execution.crashed!
+        end
+      ensure
+        @execution = nil
+      end
+
+      def last_heartbeat_too_long_ago?
+        threshold = Ductwork.configuration.supervisor_reaper_timeout
+
+        (Time.current - last_heartbeat_at) > threshold
+      end
+
       def run_hooks_for(event)
         Ductwork.hooks[:worker].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/job_worker_runner.rb

@@ -24,7 +24,7 @@ module Ductwork
       end
 
       def run
-        create_process_record!
+        adopt_or_create_process!
         start_job_workers
 
         Ductwork.logger.debug(
@@ -36,7 +36,7 @@ module Ductwork
         while running?
           # TODO: Increase or make configurable
           sleep(5)
-          check_thread_health
+          check_workers_health
           report_heartbeat!
         end
 
@@ -47,13 +47,9 @@ module Ductwork
 
       attr_reader :pipelines, :running_context, :job_workers
 
-      def create_process_record!
+      def adopt_or_create_process!
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.create!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch,
-            last_heartbeat_at: Time.current
-          )
+          Ductwork::Process.adopt_or_create_current!(:job_worker)
         end
       end
 
@@ -78,37 +74,32 @@ module Ductwork
         running_context.running?
       end
 
-      def check_thread_health
-        Ductwork.logger.debug(
-          msg: "Checking thread health",
-          role: :job_worker_runner,
-          pipelines: pipelines
-        )
-        job_workers.each do |job_worker|
-          if !job_worker.alive?
-            job_worker.restart
-
-            Ductwork.logger.warn(
-              msg: "Restarted job worker",
-              role: :job_worker_runner,
-              pipeline: job_worker.pipeline,
-              thread: job_worker.name
-            )
-          end
-        end
-        Ductwork.logger.debug(
-          msg: "Checked thread health",
-          role: :job_worker_runner,
-          pipelines: pipelines
+      def check_workers_health
+        Ductwork::Processes::WorkerHealthCheck
+          .new(job_workers, :job_worker_runner)
+          .check
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Checking worker health failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :job_worker_runner
         )
       end
 
       def report_heartbeat!
         Ductwork.logger.debug(msg: "Reporting heartbeat", role: :job_worker_runner)
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.report_heartbeat!
+          Ductwork::Process.report_heartbeat!(:job_worker)
         end
         Ductwork.logger.debug(msg: "Reported heartbeat", role: :job_worker_runner)
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reporting heartbeat failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :job_worker_runner
+        )
       end
 
       def shutdown!
@@ -116,7 +107,7 @@ module Ductwork
         job_workers.each(&:stop)
         await_threads_graceful_shutdown
         kill_remaining_job_workers
-        delete_process_record!
+        reap_process_record!
       end
 
       def await_threads_graceful_shutdown
@@ -152,12 +143,9 @@ module Ductwork
         end
       end
 
-      def delete_process_record!
+      def reap_process_record!
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.find_by!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch
-          ).delete
+          Ductwork::Process.current&.reap!(:job_worker_runner, force: true)
         end
       end
     end

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -3,7 +3,7 @@
 module Ductwork
   module Processes
     class PipelineAdvancer
-      attr_reader :thread, :last_heartbeat_at, :pipeline
+      attr_reader :thread, :last_heartbeat_at, :branch
 
       def initialize(klass, index = nil)
         @klass = klass
@@ -18,12 +18,19 @@ module Ductwork
         @thread.name = name
       end
 
-      alias restart start
+      def restart
+        cleanup_dead_thread!
+        start
+      end
 
       def alive?
         thread&.alive? || false
       end
 
+      def stuck?
+        branch.nil? && last_heartbeat_too_long_ago?
+      end
+
       def stop
         running_context.shutdown!
       end
@@ -43,9 +50,9 @@ module Ductwork
 
       private
 
-      attr_reader :klass, :index, :running_context
+      attr_reader :klass, :index, :running_context, :original_claim_token
 
-      def work_loop # rubocop:todo Metrics
+      def work_loop
         run_hooks_for(:start)
 
         Ductwork.logger.debug(
@@ -55,85 +62,27 @@ module Ductwork
         )
 
         while running_context.running?
-          id = Ductwork.wrap_with_app_executor do
-            Ductwork::Pipeline
-              .in_progress
-              .where(klass: klass, claimed_for_advancing_at: nil)
-              .where(steps: Ductwork::Step.where(status: :advancing))
-              .where.not(steps: Ductwork::Step.where.not(status: %w[advancing completed]))
-              .order(:last_advanced_at)
-              .limit(1)
-              .pluck(:id)
-              .first
-          end
+          claimed = false
 
-          if id.present?
-            rows_updated = Ductwork.wrap_with_app_executor do
-              Ductwork::Pipeline
-                .where(id: id, claimed_for_advancing_at: nil)
-                .update_all(
-                  claimed_for_advancing_at: Time.current,
-                  status: "advancing"
-                )
-            end
+          Ductwork.wrap_with_app_executor do
+            claimed = Branch.with_latest_claimed(klass) do |branch, transition, advancement|
+              @branch = branch
+              @original_claim_token = branch.claim_token
+
+              branch.advance!(transition, advancement)
 
-            if rows_updated == 1
-              Ductwork.logger.debug(
-                msg: "Pipeline claimed",
-                pipeline_id: id,
-                pipeline: klass,
-                role: :pipeline_advancer
-              )
-
-              @pipeline = Ductwork.wrap_with_app_executor do
-                @pipeline = Ductwork::Pipeline.find(id)
-                pipeline.advance!
-
-                Ductwork.logger.debug(
-                  msg: "Pipeline advanced",
-                  pipeline_id: id,
-                  pipeline: klass,
-                  role: :pipeline_advancer
-                )
-
-                # rubocop:todo Metrics/BlockNesting
-                status = if pipeline.completed?
-                           "completed"
-                         elsif pipeline.dampened?
-                           "dampened"
-                         else
-                           "in_progress"
-                         end
-                # rubocop:enable Metrics/BlockNesting
-              ensure
-                # release the pipeline and set last advanced at so it doesn't
-                # block. we're not using a queue so we have to use a db
-                # timestamp
-                pipeline.update!(
-                  claimed_for_advancing_at: nil,
-                  last_advanced_at: Time.current,
-                  status: status || "in_progress"
-                )
-              end
-            else
-              Ductwork.logger.debug(
-                msg: "Did not claim pipeline, avoided race condition",
-                pipeline_id: id,
-                pipeline: klass,
-                role: :pipeline_advancer
-              )
+              Ductwork::FaultInjection.checkpoint(:after_branch_advancement)
+            ensure
+              @branch = nil
+              @original_claim_token = nil
             end
-          else
-            Ductwork.logger.debug(
-              msg: "No pipeline needs advancing",
-              pipeline: klass,
-              role: :pipeline_advancer
-            )
           end
 
           @last_heartbeat_at = Time.current
 
-          sleep(polling_timeout)
+          if !claimed
+            sleep(polling_timeout)
+          end
         end
 
         Ductwork.logger.debug(
@@ -145,6 +94,29 @@ module Ductwork
         run_hooks_for(:stop)
       end
 
+      def cleanup_dead_thread!
+        if branch.present?
+          advancement = branch
+                        .transitions
+                        .joins(:advancements)
+                        .where(ductwork_advancements: { completed_at: nil })
+                        .first
+                        &.advancements
+                        &.find_by(completed_at: nil)
+
+          advancement&.thread_crashed!(original_claim_token)
+        end
+      ensure
+        @branch = nil
+        @original_claim_token = nil
+      end
+
+      def last_heartbeat_too_long_ago?
+        threshold = Ductwork.configuration.supervisor_reaper_timeout
+
+        (Time.current - last_heartbeat_at) > threshold
+      end
+
       def run_hooks_for(event)
         Ductwork.hooks[:advancer].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -24,7 +24,7 @@ module Ductwork
       end
 
       def run
-        create_process!
+        adopt_or_create_process!
         start_pipeline_advancers
 
         Ductwork.logger.debug(
@@ -36,7 +36,7 @@ module Ductwork
         while running_context.running?
           # TODO: Increase or make configurable
           sleep(5)
-          check_thread_health
+          check_workers_health
           report_heartbeat!
         end
 
@@ -49,60 +49,53 @@ module Ductwork
 
       def start_pipeline_advancers
         klasses.each do |klass|
-          advancer = Ductwork::Processes::PipelineAdvancer.new(klass)
-          advancers.push(advancer)
-          advancer.start
-
-          Ductwork.logger.debug(
-            msg: "Created new pipeline advancer",
-            role: :pipeline_advancer_runner,
-            pipeline: klass,
-            thread: advancer.name
-          )
-        end
-      end
+          Ductwork.configuration.pipeline_advancer_count(klass).times do |index|
+            advancer = Ductwork::Processes::PipelineAdvancer.new(klass, index)
+            advancers.push(advancer)
+            advancer.start
 
-      def check_thread_health
-        Ductwork.logger.debug(
-          msg: "Checking threads health",
-          role: :pipeline_advancer_runner,
-          pipelines: klasses
-        )
-        advancers.each do |advancer|
-          if !advancer.alive?
-            advancer.restart
-
-            Ductwork.logger.warn(
-              msg: "Restarted pipeline advancer",
+            Ductwork.logger.debug(
+              msg: "Created new pipeline advancer",
               role: :pipeline_advancer_runner,
-              pipeline: advancer.pipeline.class.to_s,
+              pipeline: klass,
               thread: advancer.name
             )
           end
         end
-        Ductwork.logger.debug(
-          msg: "Checked thread health",
-          role: :pipeline_advancer_runner,
-          pipelines: klasses
+      end
+
+      def check_workers_health
+        Ductwork::Processes::WorkerHealthCheck
+          .new(advancers, :pipeline_advancer_runner)
+          .check
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Checking worker health failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :pipeline_advancer_runner
         )
       end
 
-      def create_process!
+      def adopt_or_create_process!
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.create!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch,
-            last_heartbeat_at: Time.current
-          )
+          Ductwork::Process.adopt_or_create_current!(:pipeline_advancer)
         end
       end
 
       def report_heartbeat!
         Ductwork.logger.debug(msg: "Reporting heartbeat", role: :pipeline_advancer_runner)
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.report_heartbeat!
+          Ductwork::Process.report_heartbeat!(:pipeline_advancer)
         end
         Ductwork.logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reporting heartbeat failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :pipeline_advancer_runner
+        )
       end
 
       def shutdown!
@@ -111,7 +104,7 @@ module Ductwork
         advancers.each(&:stop)
         await_threads_graceful_shutdown
         kill_remaining_threads
-        delete_process!
+        reap_process_record!
       end
 
       def log_shutting_down
@@ -154,12 +147,9 @@ module Ductwork
         end
       end
 
-      def delete_process!
+      def reap_process_record!
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.find_by!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch
-          ).delete
+          Ductwork::Process.current&.reap!(:pipeline_advancer_runner, force: true)
         end
       end
     end