ductwork 0.25.0 → 1.0.0

data/lib/ductwork/models/result.rb

@@ -9,6 +9,7 @@ module Ductwork
     enum :result_type,
          success: "success",
          failure: "failure",
-         timed_out: "timed_out"
+         timed_out: "timed_out",
+         process_crashed: "process_crashed"
   end
 end

data/lib/ductwork/models/run.rb

@@ -2,8 +2,126 @@
 
 module Ductwork
   class Run < Ductwork::Record
-    belongs_to :execution, class_name: "Ductwork::Execution"
+    belongs_to :pipeline, class_name: "Ductwork::Pipeline"
+    has_many :branches,
+             class_name: "Ductwork::Branch",
+             foreign_key: "run_id",
+             dependent: :destroy
+    has_many :steps,
+             class_name: "Ductwork::Step",
+             foreign_key: "run_id",
+             dependent: :destroy
+    has_many :tuples,
+             class_name: "Ductwork::Tuple",
+             foreign_key: "run_id",
+             dependent: :destroy
 
+    validates :pipeline_klass, presence: true
+    validates :definition, presence: true
+    validates :definition_sha1, presence: true
+    validates :status, presence: true
     validates :started_at, presence: true
+    validates :triggered_at, presence: true
+
+    enum :status,
+         pending: "pending",
+         in_progress: "in_progress",
+         waiting: "waiting",
+         advancing: "advancing",
+         halted: "halted",
+         dampened: "dampened",
+         completed: "completed"
+
+    def parsed_definition
+      @parsed_definition ||= JSON.parse(definition).with_indifferent_access
+    end
+
+    def resolve_terminal_state!
+      Ductwork::Record.transaction do
+        lock_for_terminal_resolution!
+
+        next if halted? || completed?
+        next if branches.where.not(status: %w[completed halted]).exists?
+
+        if branches.halted.exists?
+          pipeline.update!(status: "halted")
+          update!(status: "halted", halted_at: Time.current)
+
+          Ductwork.logger.warn(
+            msg: "Pipeline halted",
+            pipeline_id: pipeline.id,
+            run_id: id
+          )
+        else
+          pipeline.update!(status: "completed")
+          update!(status: "completed", completed_at: Time.current)
+
+          Ductwork.logger.info(
+            msg: "Pipeline completed",
+            pipeline_id: pipeline.id,
+            run_id: id
+          )
+        end
+      end
+    end
+
+    # NOTE: Fires the host `on_halt` handler (at-most) once after the run has halted
+    #
+    # The handler is an observable side effect (it may page, refund, or emit an
+    # external event), so it must run only after the halt is durably committed
+    # and outside the advancer's `with_claim_fence` transaction; otherwise a
+    # rolled-back commit (deadlock victim, dropped connection) would leave it
+    # spuriously fired and double-firing on re-claim. `Branch#advance!` calls
+    # this once the advancement has committed.
+    #
+    # The dispatch is claimed with an atomic `UPDATE ... WHERE`: only the
+    # advancer that flips `on_halt_dispatched_at` from NULL runs the handler, and
+    # only while the run is actually halted (a rolled-back halt never persists
+    # `status`). That makes the handler at-most-once even across concurrent
+    # advancers and re-claims.
+    def dispatch_on_halt!
+      klass = parsed_definition.dig(:metadata, :on_halt, :klass)
+
+      return if klass.blank?
+
+      claimed = self.class
+                    .where(id: id, status: "halted", on_halt_dispatched_at: nil)
+                    .update_all(on_halt_dispatched_at: Time.current)
+
+      return if claimed.zero?
+
+      begin
+        reasons = branches.halted.pluck(:halt_reason)
+
+        Object.const_get(klass).new(reasons).execute
+      rescue StandardError => e
+        Ductwork.logger.error(
+          msg: "on_halt handler errored",
+          run_id: id,
+          error_klass: e.class.to_s,
+          error_message: e.message
+        )
+      end
+    end
+
+    private
+
+    # NOTE: a branch transition creates run-referencing rows (branch/step/job)
+    # whose FK takes a `FOR KEY SHARE` lock on this run row, then resolves the
+    # terminal state in the same transaction. The default `FOR UPDATE` conflicts
+    # with `FOR KEY SHARE`, so two transitions each holding the share-lock and
+    # upgrading here deadlock on the run row. On Postgres/Cockroach we take
+    # `FOR NO KEY UPDATE`, which is compatible with `FOR KEY SHARE` (no upgrade
+    # cycle) yet still conflicts with itself (concurrent resolutions stay
+    # serialized). MySQL has no equivalent mode, so it keeps `FOR UPDATE` and
+    # relies on the advancer retrying a deadlock victim; SQLite has no row-level
+    # locks, so `lock!` is a no-op there and the deadlock cannot occur.
+    def lock_for_terminal_resolution!
+      if Ductwork::Record.connection.adapter_name.downcase.match?(/postgresql|cockroach/)
+        lock!("FOR NO KEY UPDATE")
+      else
+        lock!
+      end
+    end
   end
 end

data/lib/ductwork/models/step.rb

@@ -2,8 +2,13 @@
 
 module Ductwork
   class Step < Ductwork::Record
-    belongs_to :pipeline, class_name: "Ductwork::Pipeline"
+    belongs_to :run, class_name: "Ductwork::Run"
+    belongs_to :branch, class_name: "Ductwork::Branch"
+    belongs_to :source_step, class_name: "Ductwork::Step", optional: true
+    has_many :derived_steps, class_name: "Ductwork::Step", foreign_key: :source_step_id, dependent: :destroy
     has_one :job, class_name: "Ductwork::Job", foreign_key: "step_id", dependent: :destroy
+    has_one :in_transition, class_name: "Ductwork::Transition", foreign_key: "in_step_id", dependent: :destroy
+    has_one :out_transition, class_name: "Ductwork::Transition", foreign_key: "out_step_id", dependent: :destroy
 
     validates :node, presence: true
     validates :klass, presence: true
@@ -29,19 +34,34 @@ module Ductwork
          converge: "converge",
          dampen: "dampen"
 
-    def self.build_for_execution(pipeline_id, *, **)
+    def self.build_for_execution(run_id, *, **)
       instance = allocate
-      instance.instance_variable_set(:@pipeline_id, pipeline_id)
+      instance.instance_variable_set(:@run_id, run_id)
       instance.send(:initialize, *, **)
       instance
     end
 
-    def pipeline_id
-      @pipeline_id || (@attributes && super)
+    alias_attribute :idempotency_key, :id
+
+    def run_id
+      @run_id || (@attributes && super)
+    end
+
+    # The result_type of the most recent execution to finish for this step's
+    # job, used to distinguish *why* a step failed (e.g. `errored!` writes
+    # "failure", `crashed!` writes "process_crashed"). Returns nil when no
+    # execution has produced a result yet.
+    def terminal_result_type
+      return if job.blank?
+
+      job.executions
+         .joins(:result)
+         .order(created_at: :desc)
+         .pick("ductwork_results.result_type")
     end
 
     def context
-      @_context ||= Ductwork::Context.new(pipeline_id)
+      @_context ||= Ductwork::Context.new(run_id)
     end
   end
 end

data/lib/ductwork/models/transition.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Transition < Ductwork::Record
+    belongs_to :branch, class_name: "Ductwork::Branch"
+    belongs_to :in_step, class_name: "Ductwork::Step"
+    belongs_to :out_step, class_name: "Ductwork::Step", optional: true
+    has_many :advancements,
+             class_name: "Ductwork::Advancement",
+             foreign_key: "transition_id",
+             dependent: :destroy
+
+    validates :started_at, presence: true
+  end
+end

data/lib/ductwork/models/tuple.rb

@@ -2,12 +2,9 @@
 
 module Ductwork
   class Tuple < Ductwork::Record
-    belongs_to :pipeline, class_name: "Ductwork::Pipeline"
+    belongs_to :run, class_name: "Ductwork::Run"
 
-    # NOTE: These model-level validations are here for symmetry. The current
-    # methods used to create `Tuple` records skip validations lol
     validates :key, presence: true
-    validates :key, uniqueness: { scope: :pipeline_id }
     validates :first_set_at, presence: true
     validates :last_set_at, presence: true
 

data/lib/ductwork/models/workflow.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Ductwork::Workflow = Ductwork::Pipeline

data/lib/ductwork/optimistic_locking_execution_claim.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class OptimisticLockingExecutionClaim
+    def initialize(klass, owner_process_id)
+      @id = nil
+      @execution = nil
+      @klass = klass
+      @process_id = owner_process_id
+    end
+
+    def latest
+      Ductwork::Record.transaction do # rubocop:todo Metrics/BlockLength
+        @id = latest_availability_id
+
+        if id.present?
+          rows_updated = claim_availability
+
+          if rows_updated == 1
+            Ductwork.logger.debug(
+              msg: "Job claimed",
+              role: :job_worker,
+              process_id: process_id,
+              availability_id: id
+            )
+
+            @execution = find_execution
+            execution.update_columns(process_id:)
+
+            update_state
+          else
+            Ductwork.logger.debug(
+              msg: "Did not claim job, avoided race condition",
+              role: :job_worker,
+              process_id: process_id,
+              availability_id: id
+            )
+          end
+        else
+          Ductwork.logger.debug(
+            msg: "No available job to claim",
+            role: :job_worker,
+            process_id: process_id,
+            pipeline: klass
+          )
+        end
+      end
+
+      execution
+    end
+
+    private
+
+    attr_reader :id, :execution, :klass, :process_id
+
+    def latest_availability_id
+      sql = Ductwork::DatabaseClock.now_sql("ductwork_availabilities.started_at")
+
+      Ductwork::Availability
+        .where(sql)
+        .where(completed_at: nil, pipeline_klass: klass)
+        .order(:started_at)
+        .limit(1)
+        .pluck(:id)
+        .first
+    end
+
+    def claim_availability
+      Ductwork::Availability
+        .where(id: id, completed_at: nil)
+        .update_all(completed_at: Time.current, process_id: process_id)
+    end
+
+    def find_execution
+      Ductwork::Execution
+        .joins(:availability)
+        .find_by!(ductwork_availabilities: { id:, process_id: })
+    end
+
+    def update_state
+      step = execution.job.step
+
+      Ductwork::Step
+        .where(id: step.id)
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::Run
+        .where(id: step.run_id)
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::Pipeline
+        .where(id: Ductwork::Run.where(id: step.run_id).select(:pipeline_id))
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+    end
+  end
+end

data/lib/ductwork/processes/health_check.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class HealthCheck
+      def self.run(**)
+        new(**).run
+      end
+
+      def initialize(pid: nil, verbose: false)
+        @pid = pid
+        @machine_identifier = Ductwork::MachineIdentifier.fetch
+        @verbose = verbose
+      end
+
+      def run
+        exit_code = supervisor_processes.reduce(0) do |acc, process|
+          if process.healthy?
+            puts_healthy(process)
+            acc
+          else
+            puts_unhealthy(process)
+            acc + 1
+          end
+        end
+
+        if pid.present? && supervisor_processes.none?
+          puts_dead
+          exit_code = 1
+        end
+
+        exit(exit_code)
+      end
+
+      private
+
+      attr_reader :pid, :machine_identifier, :verbose
+
+      def supervisor_processes
+        Ductwork::Process.supervisor.then do |relation|
+          if pid.present?
+            relation.where(pid:, machine_identifier:)
+          else
+            relation
+          end
+        end
+      end
+
+      def puts_dead
+        if verbose
+          puts <<~STDOUT
+            PID #{pid} (#{machine_identifier})
+              Status: dead
+          STDOUT
+        else
+          puts "dead"
+        end
+      end
+
+      def puts_healthy(process)
+        if verbose
+          puts_verbose(process, "healthy")
+        else
+          puts "healthy"
+        end
+      end
+
+      def puts_unhealthy(process)
+        if verbose
+          puts_verbose(process, "unhealthy")
+        else
+          puts "unhealthy"
+        end
+      end
+
+      def puts_verbose(process, status)
+        puts <<~STDOUT
+          PID #{process.pid} (#{process.machine_identifier})
+            ID: #{process.id}
+            Created At: #{process.created_at.iso8601}
+            Last Heartbeat At: #{process.last_heartbeat_at.iso8601}
+            Status: #{status}
+        STDOUT
+      end
+    end
+  end
+end

data/lib/ductwork/processes/job_worker.rb

@@ -3,7 +3,7 @@
 module Ductwork
   module Processes
     class JobWorker
-      attr_reader :thread, :last_heartbeat_at, :job, :pipeline
+      attr_reader :thread, :last_heartbeat_at, :execution, :pipeline
 
       def initialize(pipeline, id)
         @pipeline = pipeline
@@ -18,12 +18,19 @@ module Ductwork
         @thread.name = name
       end
 
-      alias restart start
+      def restart
+        cleanup_dead_thread!
+        start
+      end
 
       def alive?
         thread&.alive? || false
       end
 
+      def stuck?
+        execution.nil? && last_heartbeat_too_long_ago?
+      end
+
       def stop
         running_context.shutdown!
       end
@@ -45,7 +52,7 @@ module Ductwork
 
       attr_reader :id, :running_context
 
-      def work_loop
+      def work_loop # rubocop:todo Metrics
         run_hooks_for(:start)
 
         Ductwork.logger.debug(
@@ -55,29 +62,74 @@ module Ductwork
         )
 
         while running_context.running?
-          Ductwork.logger.debug(
-            msg: "Attempting to claim job",
-            role: :job_worker,
-            pipeline: pipeline
-          )
-
-          @job = Ductwork.wrap_with_app_executor do
-            Job.claim_latest(pipeline)
-          end
+          owner_process_id = nil
 
-          if job.present?
-            Ductwork.wrap_with_app_executor do
-              job.execute(pipeline)
+          begin
+            Ductwork.logger.debug(
+              msg: "Attempting to claim job",
+              role: :job_worker,
+              pipeline: pipeline
+            )
+
+            @execution = Ductwork.wrap_with_app_executor do
+              owner_process_id = Ductwork::Process.current.id
+              Ductwork::ExecutionClaim.new(pipeline, owner_process_id).latest
             end
 
-            @job = nil
-          else
-            Ductwork.logger.debug(
-              msg: "No job to claim, looping",
+            if execution.present?
+              Ductwork::FaultInjection.checkpoint(:after_job_claim)
+
+              Ductwork.wrap_with_app_executor do
+                execution.call(pipeline, owner_process_id)
+              end
+
+              @execution = nil
+            else
+              Ductwork.logger.debug(
+                msg: "No job to claim, looping",
+                role: :job_worker,
+                pipeline: pipeline
+              )
+              sleep(polling_timeout)
+            end
+          rescue Ductwork::Execution::CommitFailed => e
+            Ductwork.logger.error(
+              msg: "Execution commit failed",
+              error_klass: e.class.name,
+              error_message: e.message,
+              job_id: execution&.job_id,
+              job_klass: execution&.job&.klass,
               role: :job_worker,
               pipeline: pipeline
             )
-            sleep(polling_timeout)
+
+            @execution = nil
+          rescue StandardError => e
+            if execution.present?
+              Ductwork.wrap_with_app_executor do
+                execution.crashed!
+              end
+            end
+
+            Ductwork.logger.error(
+              msg: "Unexpected error in work loop",
+              error_klass: e.class.name,
+              error_message: e.message,
+              job_id: execution&.job_id,
+              job_klass: execution&.job&.klass,
+              role: :job_worker,
+              pipeline: pipeline
+            )
+          ensure
+            if execution.present? && execution.reload.completed_at.nil?
+              Ductwork.wrap_with_app_executor do
+                execution.crashed!
+              rescue StandardError
+                nil
+              end
+            end
+
+            @execution = nil
           end
 
           @last_heartbeat_at = Time.current
@@ -92,6 +144,20 @@ module Ductwork
         run_hooks_for(:stop)
       end
 
+      def cleanup_dead_thread!
+        if execution.present? && execution.reload.completed_at.nil?
+          execution.crashed!
+        end
+      ensure
+        @execution = nil
+      end
+
+      def last_heartbeat_too_long_ago?
+        threshold = Ductwork.configuration.supervisor_reaper_timeout
+
+        (Time.current - last_heartbeat_at) > threshold
+      end
+
       def run_hooks_for(event)
         Ductwork.hooks[:worker].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/job_worker_runner.rb

@@ -24,7 +24,7 @@ module Ductwork
       end
 
       def run
-        create_process_record!
+        adopt_or_create_process!
         start_job_workers
 
         Ductwork.logger.debug(
@@ -36,7 +36,7 @@ module Ductwork
         while running?
           # TODO: Increase or make configurable
           sleep(5)
-          check_thread_health
+          check_workers_health
           report_heartbeat!
         end
 
@@ -47,13 +47,9 @@ module Ductwork
 
       attr_reader :pipelines, :running_context, :job_workers
 
-      def create_process_record!
+      def adopt_or_create_process!
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.create!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch,
-            last_heartbeat_at: Time.current
-          )
+          Ductwork::Process.adopt_or_create_current!(:job_worker)
         end
       end
 
@@ -78,37 +74,32 @@ module Ductwork
         running_context.running?
       end
 
-      def check_thread_health
-        Ductwork.logger.debug(
-          msg: "Checking thread health",
-          role: :job_worker_runner,
-          pipelines: pipelines
-        )
-        job_workers.each do |job_worker|
-          if !job_worker.alive?
-            job_worker.restart
-
-            Ductwork.logger.warn(
-              msg: "Restarted job worker",
-              role: :job_worker_runner,
-              pipeline: job_worker.pipeline,
-              thread: job_worker.name
-            )
-          end
-        end
-        Ductwork.logger.debug(
-          msg: "Checked thread health",
-          role: :job_worker_runner,
-          pipelines: pipelines
+      def check_workers_health
+        Ductwork::Processes::WorkerHealthCheck
+          .new(job_workers, :job_worker_runner)
+          .check
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Checking worker health failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :job_worker_runner
         )
       end
 
       def report_heartbeat!
         Ductwork.logger.debug(msg: "Reporting heartbeat", role: :job_worker_runner)
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.report_heartbeat!
+          Ductwork::Process.report_heartbeat!(:job_worker)
         end
         Ductwork.logger.debug(msg: "Reported heartbeat", role: :job_worker_runner)
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reporting heartbeat failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :job_worker_runner
+        )
       end
 
       def shutdown!
@@ -116,7 +107,7 @@ module Ductwork
         job_workers.each(&:stop)
         await_threads_graceful_shutdown
         kill_remaining_job_workers
-        delete_process_record!
+        reap_process_record!
       end
 
       def await_threads_graceful_shutdown
@@ -152,12 +143,9 @@ module Ductwork
         end
       end
 
-      def delete_process_record!
+      def reap_process_record!
         Ductwork.wrap_with_app_executor do
-          Ductwork::Process.find_by!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch
-          ).delete
+          Ductwork::Process.current&.reap!(:job_worker_runner, force: true)
         end
       end
     end