ductwork 0.26.0 → 1.0.0

data/lib/ductwork/models/process.rb

@@ -2,18 +2,190 @@
 
 module Ductwork
   class Process < Ductwork::Record
-    class NotFoundError < StandardError; end
+    has_many :advancements,
+             class_name: "Ductwork::Advancement",
+             foreign_key: "process_id",
+             dependent: :nullify
+    has_many :availabilities,
+             class_name: "Ductwork::Availability",
+             foreign_key: "process_id",
+             dependent: :nullify
+    has_many :executions,
+             class_name: "Ductwork::Execution",
+             foreign_key: "process_id",
+             dependent: :nullify
 
     validates :pid, uniqueness: { scope: :machine_identifier }
 
-    def self.report_heartbeat!
+    enum :role,
+         supervisor: "supervisor",
+         pipeline_advancer: "pipeline_advancer",
+         job_worker: "job_worker"
+
+    ORPHANED_CLAIM_SWEEP_MULTIPLIER = 3
+
+    def self.adopt_or_create_current!(role)
       pid = ::Process.pid
       machine_identifier = Ductwork::MachineIdentifier.fetch
+      last_heartbeat_at = Ductwork::DatabaseClock.now
+      existing = Ductwork::Process.find_by(pid:, machine_identifier:)
+
+      # NOTE: Same pid + machine_identifier can only mean the OS reused this
+      # pid, which happens after the prior process at this identity has
+      # exited -- a live process can never be replaced while still running.
+      # So its in-flight claims belong to a dead incarnation and must always
+      # be recovered here, even when the heartbeat still looks fresh because
+      # the reaper's timeout hasn't elapsed yet.
+      existing&.recover_crashed_claims!(role)
+
+      Ductwork::Process
+        .find_or_initialize_by(pid:, machine_identifier:)
+        .tap { |process| process.update!(last_heartbeat_at:, role:) }
+    end
+
+    def self.current
+      pid = ::Process.pid
+      machine_identifier = Ductwork::MachineIdentifier.fetch
+
+      find_by(pid:, machine_identifier:)
+    end
+
+    def self.reap_all!(role)
+      count = 0
+      timeout = Ductwork.configuration.supervisor_reaper_timeout
+      sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", timeout)
+
+      Ductwork.logger.debug(
+        msg: "Reaping orphaned process records",
+        role: role
+      )
+
+      where(sql).find_each do |process|
+        process.reap!(role)
+        count += 1
+      end
+
+      Ductwork.logger.debug(
+        msg: "Reaped #{count} orphaned process records",
+        count: count,
+        role: role
+      )
+    end
+
+    # NOTE: backstop for claims that lost their process_id (dependent: :nullify
+    # on a process destroy racing a fresh claim) and so are unreachable through
+    # any Process association -- `reap!`/`reap_all!` above only ever look
+    # through a Process record's own advancements/executions, and a nil
+    # process_id means there is no Process record to look through. This scans
+    # the orphaned rows directly instead.
+    #
+    # Executions are additionally scoped to availabilities with a completed_at
+    # present: an unclaimed execution legitimately has process_id: nil while
+    # it waits to be picked up (see RowLockingExecutionClaim#claim_availability,
+    # which only sets process_id at claim time), so process_id: nil alone
+    # would misidentify perfectly healthy queued work as orphaned.
+    def self.sweep_orphaned_claims!(role)
+      timeout = Ductwork.configuration.supervisor_reaper_timeout * ORPHANED_CLAIM_SWEEP_MULTIPLIER
+      count = 0
+      advancement_sql = Ductwork::DatabaseClock.ago_sql("started_at", timeout)
+
+      Ductwork::Advancement
+        .where(process_id: nil, completed_at: nil)
+        .where(advancement_sql)
+        .find_each do |advancement|
+          advancement.process_crashed!
+          count += 1
+        end
+
+      execution_sql = Ductwork::DatabaseClock.ago_sql("ductwork_availabilities.completed_at", timeout)
 
-      find_by!(pid:, machine_identifier:)
-        .update!(last_heartbeat_at: Time.current)
+      Ductwork::Execution
+        .joins(:availability)
+        .where(process_id: nil, completed_at: nil)
+        .where.not(ductwork_availabilities: { completed_at: nil })
+        .where(execution_sql)
+        .find_each do |execution|
+          execution.crashed!
+          count += 1
+        end
+
+      Ductwork.logger.debug(
+        msg: "Swept #{count} orphaned claims",
+        count: count,
+        role: role
+      )
+    end
+
+    def self.report_heartbeat!(role)
+      process = current
+
+      if process.present?
+        process.update!(last_heartbeat_at: Ductwork::DatabaseClock.now)
+        process
+      else
+        Ductwork.logger.warn(
+          msg: "Process record missing, re-adopting (likely reaped after host suspend)",
+          pid: ::Process.pid
+        )
+        adopt_or_create_current!(role)
+      end
+    end
+
+    def reap!(role, force: false)
+      timeout = Ductwork.configuration.supervisor_reaper_timeout
+      sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", timeout)
+
+      Ductwork.logger.debug(
+        msg: "Reaping orphaned process record #{id}",
+        id: id,
+        role: role
+      )
+
+      Ductwork::Record.transaction do
+        # NOTE: Callers that have already killed/stopped the process hold proof
+        # of death and pass force: true to skip the staleness guard. The row
+        # lock and existence check are kept either way to stay atomic and to
+        # avoid double-reaping a record another parent already cleaned up
+        scope = Ductwork::Process.where(id:).lock
+        scope = scope.where(sql) unless force
+
+        return unless scope.exists?
+
+        recover_crashed_claims!(role)
+        destroy
+      end
+
+      Ductwork.logger.debug(
+        msg: "Reaped orphaned process record #{id}",
+        id: id,
+        role: role
+      )
     rescue ActiveRecord::RecordNotFound
-      raise NotFoundError, "Process #{pid} not found"
+      Ductwork.logger.debug(
+        msg: "Process already reaped by another parent",
+        id: id,
+        role: role
+      )
+    end
+
+    def recover_crashed_claims!(role)
+      Ductwork.logger.debug(
+        msg: "Recovering in-flight claims on reused process record #{id}",
+        id: id,
+        role: role
+      )
+
+      Ductwork::Record.transaction do
+        advancements.where(completed_at: nil).find_each(&:process_crashed!)
+        executions.where(completed_at: nil).find_each(&:crashed!)
+      end
+    end
+
+    def healthy?
+      timeout = Ductwork.configuration.supervisor_reaper_timeout
+      sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", timeout)
+
+      self.class.where(id:).where(sql).none?
     end
   end
 end

data/lib/ductwork/models/result.rb

@@ -9,6 +9,7 @@ module Ductwork
     enum :result_type,
          success: "success",
          failure: "failure",
-         timed_out: "timed_out"
+         timed_out: "timed_out",
+         process_crashed: "process_crashed"
   end
 end

data/lib/ductwork/models/run.rb

@@ -2,8 +2,126 @@
 
 module Ductwork
   class Run < Ductwork::Record
-    belongs_to :execution, class_name: "Ductwork::Execution"
+    belongs_to :pipeline, class_name: "Ductwork::Pipeline"
+    has_many :branches,
+             class_name: "Ductwork::Branch",
+             foreign_key: "run_id",
+             dependent: :destroy
+    has_many :steps,
+             class_name: "Ductwork::Step",
+             foreign_key: "run_id",
+             dependent: :destroy
+    has_many :tuples,
+             class_name: "Ductwork::Tuple",
+             foreign_key: "run_id",
+             dependent: :destroy
 
+    validates :pipeline_klass, presence: true
+    validates :definition, presence: true
+    validates :definition_sha1, presence: true
+    validates :status, presence: true
     validates :started_at, presence: true
+    validates :triggered_at, presence: true
+
+    enum :status,
+         pending: "pending",
+         in_progress: "in_progress",
+         waiting: "waiting",
+         advancing: "advancing",
+         halted: "halted",
+         dampened: "dampened",
+         completed: "completed"
+
+    def parsed_definition
+      @parsed_definition ||= JSON.parse(definition).with_indifferent_access
+    end
+
+    def resolve_terminal_state!
+      Ductwork::Record.transaction do
+        lock_for_terminal_resolution!
+
+        next if halted? || completed?
+        next if branches.where.not(status: %w[completed halted]).exists?
+
+        if branches.halted.exists?
+          pipeline.update!(status: "halted")
+          update!(status: "halted", halted_at: Time.current)
+
+          Ductwork.logger.warn(
+            msg: "Pipeline halted",
+            pipeline_id: pipeline.id,
+            run_id: id
+          )
+        else
+          pipeline.update!(status: "completed")
+          update!(status: "completed", completed_at: Time.current)
+
+          Ductwork.logger.info(
+            msg: "Pipeline completed",
+            pipeline_id: pipeline.id,
+            run_id: id
+          )
+        end
+      end
+    end
+
+    # NOTE: Fires the host `on_halt` handler (at-most) once after the run has halted
+    #
+    # The handler is an observable side effect (it may page, refund, or emit an
+    # external event), so it must run only after the halt is durably committed
+    # and outside the advancer's `with_claim_fence` transaction; otherwise a
+    # rolled-back commit (deadlock victim, dropped connection) would leave it
+    # spuriously fired and double-firing on re-claim. `Branch#advance!` calls
+    # this once the advancement has committed.
+    #
+    # The dispatch is claimed with an atomic `UPDATE ... WHERE`: only the
+    # advancer that flips `on_halt_dispatched_at` from NULL runs the handler, and
+    # only while the run is actually halted (a rolled-back halt never persists
+    # `status`). That makes the handler at-most-once even across concurrent
+    # advancers and re-claims.
+    def dispatch_on_halt!
+      klass = parsed_definition.dig(:metadata, :on_halt, :klass)
+
+      return if klass.blank?
+
+      claimed = self.class
+                    .where(id: id, status: "halted", on_halt_dispatched_at: nil)
+                    .update_all(on_halt_dispatched_at: Time.current)
+
+      return if claimed.zero?
+
+      begin
+        reasons = branches.halted.pluck(:halt_reason)
+
+        Object.const_get(klass).new(reasons).execute
+      rescue StandardError => e
+        Ductwork.logger.error(
+          msg: "on_halt handler errored",
+          run_id: id,
+          error_klass: e.class.to_s,
+          error_message: e.message
+        )
+      end
+    end
+
+    private
+
+    # NOTE: a branch transition creates run-referencing rows (branch/step/job)
+    # whose FK takes a `FOR KEY SHARE` lock on this run row, then resolves the
+    # terminal state in the same transaction. The default `FOR UPDATE` conflicts
+    # with `FOR KEY SHARE`, so two transitions each holding the share-lock and
+    # upgrading here deadlock on the run row. On Postgres/Cockroach we take
+    # `FOR NO KEY UPDATE`, which is compatible with `FOR KEY SHARE` (no upgrade
+    # cycle) yet still conflicts with itself (concurrent resolutions stay
+    # serialized). MySQL has no equivalent mode, so it keeps `FOR UPDATE` and
+    # relies on the advancer retrying a deadlock victim; SQLite has no row-level
+    # locks, so `lock!` is a no-op there and the deadlock cannot occur.
+    def lock_for_terminal_resolution!
+      if Ductwork::Record.connection.adapter_name.downcase.match?(/postgresql|cockroach/)
+        lock!("FOR NO KEY UPDATE")
+      else
+        lock!
+      end
+    end
   end
 end

data/lib/ductwork/models/step.rb

@@ -2,8 +2,13 @@
 
 module Ductwork
   class Step < Ductwork::Record
-    belongs_to :pipeline, class_name: "Ductwork::Pipeline"
+    belongs_to :run, class_name: "Ductwork::Run"
+    belongs_to :branch, class_name: "Ductwork::Branch"
+    belongs_to :source_step, class_name: "Ductwork::Step", optional: true
+    has_many :derived_steps, class_name: "Ductwork::Step", foreign_key: :source_step_id, dependent: :destroy
     has_one :job, class_name: "Ductwork::Job", foreign_key: "step_id", dependent: :destroy
+    has_one :in_transition, class_name: "Ductwork::Transition", foreign_key: "in_step_id", dependent: :destroy
+    has_one :out_transition, class_name: "Ductwork::Transition", foreign_key: "out_step_id", dependent: :destroy
 
     validates :node, presence: true
     validates :klass, presence: true
@@ -29,19 +34,34 @@ module Ductwork
          converge: "converge",
          dampen: "dampen"
 
-    def self.build_for_execution(pipeline_id, *, **)
+    def self.build_for_execution(run_id, *, **)
       instance = allocate
-      instance.instance_variable_set(:@pipeline_id, pipeline_id)
+      instance.instance_variable_set(:@run_id, run_id)
       instance.send(:initialize, *, **)
       instance
     end
 
-    def pipeline_id
-      @pipeline_id || (@attributes && super)
+    alias_attribute :idempotency_key, :id
+
+    def run_id
+      @run_id || (@attributes && super)
+    end
+
+    # The result_type of the most recent execution to finish for this step's
+    # job, used to distinguish *why* a step failed (e.g. `errored!` writes
+    # "failure", `crashed!` writes "process_crashed"). Returns nil when no
+    # execution has produced a result yet.
+    def terminal_result_type
+      return if job.blank?
+
+      job.executions
+         .joins(:result)
+         .order(created_at: :desc)
+         .pick("ductwork_results.result_type")
     end
 
     def context
-      @_context ||= Ductwork::Context.new(pipeline_id)
+      @_context ||= Ductwork::Context.new(run_id)
     end
   end
 end

data/lib/ductwork/models/transition.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Transition < Ductwork::Record
+    belongs_to :branch, class_name: "Ductwork::Branch"
+    belongs_to :in_step, class_name: "Ductwork::Step"
+    belongs_to :out_step, class_name: "Ductwork::Step", optional: true
+    has_many :advancements,
+             class_name: "Ductwork::Advancement",
+             foreign_key: "transition_id",
+             dependent: :destroy
+
+    validates :started_at, presence: true
+  end
+end

data/lib/ductwork/models/tuple.rb

@@ -2,12 +2,9 @@
 
 module Ductwork
   class Tuple < Ductwork::Record
-    belongs_to :pipeline, class_name: "Ductwork::Pipeline"
+    belongs_to :run, class_name: "Ductwork::Run"
 
-    # NOTE: These model-level validations are here for symmetry. The current
-    # methods used to create `Tuple` records skip validations lol
     validates :key, presence: true
-    validates :key, uniqueness: { scope: :pipeline_id }
     validates :first_set_at, presence: true
     validates :last_set_at, presence: true
 

data/lib/ductwork/models/workflow.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Ductwork::Workflow = Ductwork::Pipeline

data/lib/ductwork/optimistic_locking_execution_claim.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class OptimisticLockingExecutionClaim
+    def initialize(klass, owner_process_id)
+      @id = nil
+      @execution = nil
+      @klass = klass
+      @process_id = owner_process_id
+    end
+
+    def latest
+      Ductwork::Record.transaction do # rubocop:todo Metrics/BlockLength
+        @id = latest_availability_id
+
+        if id.present?
+          rows_updated = claim_availability
+
+          if rows_updated == 1
+            Ductwork.logger.debug(
+              msg: "Job claimed",
+              role: :job_worker,
+              process_id: process_id,
+              availability_id: id
+            )
+
+            @execution = find_execution
+            execution.update_columns(process_id:)
+
+            update_state
+          else
+            Ductwork.logger.debug(
+              msg: "Did not claim job, avoided race condition",
+              role: :job_worker,
+              process_id: process_id,
+              availability_id: id
+            )
+          end
+        else
+          Ductwork.logger.debug(
+            msg: "No available job to claim",
+            role: :job_worker,
+            process_id: process_id,
+            pipeline: klass
+          )
+        end
+      end
+
+      execution
+    end
+
+    private
+
+    attr_reader :id, :execution, :klass, :process_id
+
+    def latest_availability_id
+      sql = Ductwork::DatabaseClock.now_sql("ductwork_availabilities.started_at")
+
+      Ductwork::Availability
+        .where(sql)
+        .where(completed_at: nil, pipeline_klass: klass)
+        .order(:started_at)
+        .limit(1)
+        .pluck(:id)
+        .first
+    end
+
+    def claim_availability
+      Ductwork::Availability
+        .where(id: id, completed_at: nil)
+        .update_all(completed_at: Time.current, process_id: process_id)
+    end
+
+    def find_execution
+      Ductwork::Execution
+        .joins(:availability)
+        .find_by!(ductwork_availabilities: { id:, process_id: })
+    end
+
+    def update_state
+      step = execution.job.step
+
+      Ductwork::Step
+        .where(id: step.id)
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::Run
+        .where(id: step.run_id)
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::Pipeline
+        .where(id: Ductwork::Run.where(id: step.run_id).select(:pipeline_id))
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+    end
+  end
+end

data/lib/ductwork/processes/health_check.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class HealthCheck
+      def self.run(**)
+        new(**).run
+      end
+
+      def initialize(pid: nil, verbose: false)
+        @pid = pid
+        @machine_identifier = Ductwork::MachineIdentifier.fetch
+        @verbose = verbose
+      end
+
+      def run
+        exit_code = supervisor_processes.reduce(0) do |acc, process|
+          if process.healthy?
+            puts_healthy(process)
+            acc
+          else
+            puts_unhealthy(process)
+            acc + 1
+          end
+        end
+
+        if pid.present? && supervisor_processes.none?
+          puts_dead
+          exit_code = 1
+        end
+
+        exit(exit_code)
+      end
+
+      private
+
+      attr_reader :pid, :machine_identifier, :verbose
+
+      def supervisor_processes
+        Ductwork::Process.supervisor.then do |relation|
+          if pid.present?
+            relation.where(pid:, machine_identifier:)
+          else
+            relation
+          end
+        end
+      end
+
+      def puts_dead
+        if verbose
+          puts <<~STDOUT
+            PID #{pid} (#{machine_identifier})
+              Status: dead
+          STDOUT
+        else
+          puts "dead"
+        end
+      end
+
+      def puts_healthy(process)
+        if verbose
+          puts_verbose(process, "healthy")
+        else
+          puts "healthy"
+        end
+      end
+
+      def puts_unhealthy(process)
+        if verbose
+          puts_verbose(process, "unhealthy")
+        else
+          puts "unhealthy"
+        end
+      end
+
+      def puts_verbose(process, status)
+        puts <<~STDOUT
+          PID #{process.pid} (#{process.machine_identifier})
+            ID: #{process.id}
+            Created At: #{process.created_at.iso8601}
+            Last Heartbeat At: #{process.last_heartbeat_at.iso8601}
+            Status: #{status}
+        STDOUT
+      end
+    end
+  end
+end