ductwork 0.25.0 → 1.0.0

data/lib/ductwork/models/pipeline.rb

@@ -2,16 +2,13 @@
 
 module Ductwork
   class Pipeline < Ductwork::Record # rubocop:todo Metrics/ClassLength
-    has_many :steps, class_name: "Ductwork::Step", foreign_key: "pipeline_id", dependent: :destroy
-    has_many :tuples, class_name: "Ductwork::Tuple", foreign_key: "pipeline_id", dependent: :destroy
+    has_many :runs,
+             class_name: "Ductwork::Run",
+             foreign_key: "pipeline_id",
+             dependent: :destroy
 
     validates :klass, presence: true
-    validates :definition, presence: true
-    validates :definition_sha1, presence: true
     validates :status, presence: true
-    validates :started_at, presence: true
-    validates :triggered_at, presence: true
-    validates :last_advanced_at, presence: true
 
     enum :status,
          pending: "pending",
@@ -31,6 +28,7 @@ module Ductwork
     end
 
     class DefinitionError < StandardError; end
+    class ReviveError < StandardError; end
 
     class << self
       attr_reader :pipeline_definition
@@ -53,31 +51,42 @@ module Ductwork
         Ductwork.defined_pipelines << name.to_s
       end
 
-      def trigger(*args)
+      def trigger(*args) # rubocop:todo Metrics
         if pipeline_definition.nil?
           raise DefinitionError, "Pipeline must be defined before triggering"
         end
 
+        now = Time.current
         node = pipeline_definition.dig(:nodes, 0)
         klass = pipeline_definition.dig(:edges, node, :klass)
         definition = JSON.dump(pipeline_definition)
 
-        pipeline = Record.transaction do
+        pipeline = Record.transaction do # rubocop:todo Metrics/BlockLength
           p = create!(
             klass: name.to_s,
+            status: :in_progress
+          )
+          run = p.runs.create!(
+            pipeline_klass: name.to_s,
             status: :in_progress,
             definition: definition,
             definition_sha1: Digest::SHA1.hexdigest(definition),
-            triggered_at: Time.current,
-            started_at: Time.current,
-            last_advanced_at: Time.current
+            triggered_at: now,
+            started_at: now
+          )
+          branch = run.branches.create!(
+            pipeline_klass: name.to_s,
+            status: :in_progress,
+            started_at: now,
+            last_advanced_at: now
           )
-          step = p.steps.create!(
+          step = branch.steps.create!(
+            run: run,
             node: node,
             klass: klass,
             status: :in_progress,
             to_transition: :start,
-            started_at: Time.current
+            started_at: now
           )
           Ductwork::Job.enqueue(step, *args)
 
@@ -94,287 +103,175 @@ module Ductwork
       end
     end
 
-    def advance!
-      # NOTE: if we've expanded the pipeline there could be a lot of
-      # advancing records which may cause memory issues. something to
-      # watch out for here and maybe add in config to use AR relation
-      # at certain counts or even memory limits.
-      advancing_steps = steps.advancing.pluck(:id, :node, :klass)
-      advancing_ids = advancing_steps.map(&:first)
-      edges = find_edges(advancing_steps)
+    def current_run
+      runs.in_progress.sole
+    end
 
-      Ductwork::Record.transaction do
-        if edges.nil? || edges.all? { |_, attrs| attrs[:to].blank? }
-          conditionally_complete_pipeline(advancing_ids)
-        else
-          advance_to_next_steps_by_type(edges, advancing_ids)
-        end
+    def revive!(duplicate_context: false)
+      if !halted?
+        raise ReviveError, "Cannot revive #{status} pipeline"
       end
-    end
 
-    def parsed_definition
-      @parsed_definition ||= JSON.parse(definition).with_indifferent_access
-    end
+      last_run = runs.order(started_at: :desc).first
 
-    def complete!
-      update!(status: :completed, completed_at: Time.current)
+      if last_run.blank?
+        raise ReviveError, "Cannot revive pipeline without previous run"
+      end
 
-      Ductwork.logger.info(
-        msg: "Pipeline completed",
-        pipeline_id: id,
-        role: :pipeline_advancer
-      )
-    end
+      now = Time.current
+      new_run = last_run.dup
+      new_run.triggered_at = now
+      new_run.started_at = now
+      new_run.status = "in_progress"
+      new_run.on_halt_dispatched_at = nil
 
-    def halt!
-      update!(status: :halted, halted_at: Time.current)
+      Ductwork::Record.transaction do
+        lock!
+        new_run.save!
+        duplicate_successful_branches_and_steps(new_run, last_run, now)
+        duplicate_halted_branches_and_steps(new_run, last_run, now)
+        conditionally_duplicate_context(last_run, new_run, now, duplicate_context)
+        duplicate_all_branch_links(new_run, last_run)
+        in_progress!
+      end
 
-      Ductwork.logger.info(
-        msg: "Pipeline halted",
-        pipeline_id: id,
-        pipeline_klass: klass
-      )
+      self
     end
 
     private
 
-    def create_step_and_enqueue_job(edge:, input_arg:, node: nil)
-      status = :in_progress
-      started_at = Time.current
-      # NOTE: "chain" is used by ActiveRecord so we have to call
-      # this enum value "default" :sad:
-      to_transition = edge[:type] == "chain" ? "default" : edge[:type]
-      node ||= edge[:to].sole
-      klass = parsed_definition.dig(:edges, node, :klass)
-
-      next_step = steps.create!(node:, klass:, status:, to_transition:, started_at:)
-      Ductwork::Job.enqueue(next_step, input_arg)
-    end
-
-    def find_edges(advancing_steps)
-      if advancing_steps.any?
-        nodes = advancing_steps.map(&:second)
-
-        parsed_definition.fetch(:edges, {}).select { |k| k.in?(nodes) }
-      end
-    end
-
-    def conditionally_complete_pipeline(advancing_ids)
-      steps
-        .where(id: advancing_ids)
-        .update_all(status: :completed, completed_at: Time.current)
+    def duplicate_successful_branches_and_steps(new_run, last_run, now)
+      status = %i[advancing waiting completed]
 
-      remaining = steps
-                  .where(status: %w[in_progress pending advancing])
-                  .where.not(id: advancing_ids)
-                  .exists?
+      last_run.branches.where(status:).find_each do |branch|
+        new_branch = branch.dup
+        new_branch.run = new_run
+        new_branch.started_at = now
+        new_branch.completed_at = now
 
-      if !remaining
-        complete!
-      end
-    end
+        new_branch.save!
 
-    def advance_to_next_steps_by_type(edges, advancing_ids)
-      steps.where(id: advancing_ids).update_all(status: :completed, completed_at: Time.current)
+        branch.steps.where(status:).find_each do |step|
+          new_step = step.dup
+          new_step.source_step = step
+          new_step.branch = new_branch
+          new_step.run = new_run
+          new_step.started_at = now
+          new_step.completed_at = now
 
-      if edges.all? { |_, attrs| attrs[:type] == "combine" }
-        conditionally_combine_next_steps(edges, advancing_ids)
-      else
-        edges.each do |node, attrs|
-          if attrs[:type] == "collapse"
-            conditionally_collapse_next_steps(node, attrs, advancing_ids)
-          else
-            advance_non_merging_steps(attrs, advancing_ids)
-          end
+          new_step.save!
         end
       end
-      log_pipeline_advanced(edges)
     end
 
-    def advance_non_merging_steps(edge, advancing_ids)
-      to_transition = edge[:type]
-      klass = edge[:klass]
-
-      steps.where(id: advancing_ids, klass: klass).find_each do |step|
-        if to_transition.in?(%w[chain divide])
-          advance_to_next_steps(step.id, edge)
-        elsif to_transition == "expand"
-          expand_to_next_steps(step.id, edge)
-        else
-          Ductwork.logger.error(
-            msg: "Invalid To Transition",
-            to_transition: to_transition,
-            pipeline_id: id,
-            role: :pipeline_advancer
-          )
-        end
+    def duplicate_halted_branches_and_steps(new_run, last_run, now)
+      last_run.branches.where(status: :halted).find_each do |branch|
+        new_branch = branch.dup
+        new_branch.run = new_run
+        new_branch.status = "in_progress"
+        new_branch.started_at = now
+        new_branch.completed_at = nil
+        new_branch.last_advanced_at = now
+
+        new_branch.save!
+        revive_branch_steps(branch, new_branch, new_run, now)
       end
     end
 
-    def advance_to_next_steps(step_id, edge)
-      too_many = edge[:to].tally.any? do |to_klass, count|
-        depth = Ductwork
-                .configuration
-                .steps_max_depth(pipeline: klass, step: to_klass)
-
-        depth != -1 && count > depth
-      end
-
-      if too_many
-        halt!
-      else
-        edge[:to].each do |node|
-          input_arg = Ductwork::Job.find_by(step_id:).return_value
-          create_step_and_enqueue_job(edge:, input_arg:, node:)
-        end
-      end
-    end
+    def revive_branch_steps(branch, new_branch, new_run, now)
+      duplicate_prior_steps_as_completed(branch, new_branch, new_run, now)
 
-    def conditionally_combine_next_steps(edges, advancing_ids)
-      if steps.where(status: %w[pending in_progress], node: edges.keys).none?
-        combine_next_steps(edges, advancing_ids)
+      if branch.job_retries_exhausted?
+        re_enqueue_failed_step(branch, new_branch, new_run, now)
       else
-        Ductwork.logger.debug(
-          msg: "Not all divided steps have completed; not combining",
-          pipeline_id: id,
-          role: :pipeline_advancer
-        )
+        re_advance_latest_step(branch, new_branch, new_run, now)
       end
     end
 
-    def combine_next_steps(edges, advancing_ids)
-      edge = edges.values.sample
-      groups = steps
-               .where(id: advancing_ids)
-               .group(:node)
-               .count
-               .keys
-               .map { |node| steps.where(id: advancing_ids).where(node:) }
-
-      groups.first.zip(*groups[1..]).each do |group|
-        input_arg = Ductwork::Job
-                    .where(step_id: group.map(&:id))
-                    .map(&:return_value)
-        create_step_and_enqueue_job(edge:, input_arg:)
-      end
-    end
+    def duplicate_prior_steps_as_completed(branch, new_branch, new_run, now)
+      latest_step = branch.steps.order(started_at: :desc).first
 
-    def expand_to_next_steps(step_id, edge)
-      next_klass = parsed_definition.dig(:edges, edge[:to].sole, :klass)
-      return_value = Ductwork::Job
-                     .find_by(step_id:)
-                     .return_value
-      max_depth = Ductwork.configuration.steps_max_depth(pipeline: klass, step: next_klass)
-
-      if max_depth != -1 && return_value.count > max_depth
-        halt!
-      elsif return_value.none?
-        complete!
-      else
-        bulk_create_steps_and_jobs(edge:, return_value:)
-      end
-    end
+      return if latest_step.blank?
 
-    def bulk_create_steps_and_jobs(edge:, return_value:) # rubocop:todo Metrics
-      # NOTE: "chain" is used by ActiveRecord so we have to call
-      # this enum value "default" :sad:
-      to_transition = edge[:type] == "chain" ? "default" : edge[:type]
-      node ||= edge[:to].sole
-      step_klass = parsed_definition.dig(:edges, node, :klass)
-      now = Time.current
+      status = %i[advancing waiting completed]
+      id = latest_step.id
 
-      Array(return_value).each_slice(1_000).each do |batch| # rubocop:todo Metrics/BlockLength
-        step_rows = []
-        job_rows = []
-        execution_rows = []
-        availability_rows = []
-
-        batch.each do |value| # rubocop:todo Metrics/BlockLength
-          step_id = SecureRandom.uuid_v7
-          job_id = SecureRandom.uuid_v7
-          execution_id = SecureRandom.uuid_v7
-          availability_id = SecureRandom.uuid_v7
-
-          step_rows << {
-            id: step_id,
-            pipeline_id: id,
-            node: node,
-            klass: step_klass,
-            status: "in_progress",
-            to_transition: to_transition,
-            started_at: now,
-            created_at: now,
-            updated_at: now,
-          }
-          job_rows << {
-            id: job_id,
-            step_id: step_id,
-            input_args: JSON.dump({ args: [value] }),
-            klass: step_klass,
-            started_at: now,
-            created_at: now,
-            updated_at: now,
-          }
-          execution_rows << {
-            id: execution_id,
-            job_id: job_id,
-            retry_count: 0,
-            started_at: now,
-            created_at: now,
-            updated_at: now,
-          }
-          availability_rows << {
-            id: availability_id,
-            execution_id: execution_id,
-            pipeline_klass: klass,
-            started_at: now,
-            created_at: now,
-            updated_at: now,
-          }
-        end
+      branch.steps.where(status:).where.not(id:).find_each do |step|
+        new_step = step.dup
+        new_step.source_step = step
+        new_step.branch = new_branch
+        new_step.run = new_run
+        new_step.started_at = now
+        new_step.completed_at = now
 
-        Ductwork::Step.insert_all!(step_rows)
-        Ductwork::Job.insert_all!(job_rows)
-        Ductwork::Execution.insert_all!(execution_rows)
-        Ductwork::Availability.insert_all!(availability_rows)
-
-        Ductwork.logger.info(
-          msg: "Job batch enqueued",
-          count: batch.count,
-          job_klass: step_klass
-        )
+        new_step.save!
       end
     end
 
-    def conditionally_collapse_next_steps(node, edge, advancing_ids)
-      if steps.where(status: %w[pending in_progress], node: node).none?
-        collapse_next_steps(edge, advancing_ids)
-      else
-        Ductwork.logger.debug(
-          msg: "Not all expanded steps have completed; not collapsing",
-          pipeline_id: id,
-          role: :pipeline_advancer
-        )
-      end
+    def re_enqueue_failed_step(branch, new_branch, new_run, now)
+      failed_step = branch.steps.find_by(status: :failed)
+      args = JSON.parse(failed_step.job.input_args).fetch("args")
+      step = new_branch.steps.create!(
+        run: new_run,
+        source_step: failed_step,
+        node: failed_step.node,
+        klass: failed_step.klass,
+        to_transition: failed_step.to_transition,
+        status: :in_progress,
+        started_at: now
+      )
+      Ductwork::Job.enqueue(step, *args)
     end
 
-    def collapse_next_steps(edge, advancing_ids)
-      input_arg = []
+    def re_advance_latest_step(branch, new_branch, new_run, now)
+      latest_step = branch.steps.order(started_at: :desc).first
+      step = new_branch.steps.create!(
+        run: new_run,
+        source_step: latest_step,
+        node: latest_step.node,
+        klass: latest_step.klass,
+        to_transition: latest_step.to_transition,
+        status: :advancing,
+        started_at: now
+      )
+      job = latest_step.job.dup
+      job.step = step
+      job.started_at = now
+      job.completed_at = now
+      job.save!
+    end
 
-      Ductwork::Job.where(step_id: advancing_ids).find_each do |job|
-        input_arg << job.return_value
-      end
+    def conditionally_duplicate_context(last_run, new_run, now, duplicate_context)
+      if duplicate_context
+        last_run.tuples.find_each do |tuple|
+          new_tuple = tuple.dup
+          new_tuple.run = new_run
+          new_tuple.first_set_at = now
+          new_tuple.last_set_at = now
 
-      create_step_and_enqueue_job(edge:, input_arg:)
+          new_tuple.save!
+        end
+      end
     end
 
-    def log_pipeline_advanced(edges)
-      Ductwork.logger.info(
-        msg: "Pipeline advanced",
-        pipeline_id: id,
-        transitions: edges.map { |_, v| v.dig(-1, :type) },
-        role: :pipeline_advancer
-      )
+    def duplicate_all_branch_links(new_run, last_run)
+      map = Ductwork::Step
+            .where(branch_id: new_run.branches.select(:id))
+            .where.not(source_step_id: nil)
+            .joins("INNER JOIN ductwork_steps src ON src.id = ductwork_steps.source_step_id")
+            .pluck("src.branch_id", "ductwork_steps.branch_id")
+            .uniq
+            .to_h
+      old_branch_ids = last_run.branches.select(:id)
+
+      Ductwork::BranchLink
+        .where(parent_branch_id: old_branch_ids, child_branch_id: old_branch_ids)
+        .find_each do |old_link|
+          Ductwork::BranchLink.create!(
+            parent_branch_id: map.fetch(old_link.parent_branch_id),
+            child_branch_id: map.fetch(old_link.child_branch_id)
+          )
+        end
     end
   end
 end

data/lib/ductwork/models/process.rb

@@ -2,18 +2,190 @@
 
 module Ductwork
   class Process < Ductwork::Record
-    class NotFoundError < StandardError; end
+    has_many :advancements,
+             class_name: "Ductwork::Advancement",
+             foreign_key: "process_id",
+             dependent: :nullify
+    has_many :availabilities,
+             class_name: "Ductwork::Availability",
+             foreign_key: "process_id",
+             dependent: :nullify
+    has_many :executions,
+             class_name: "Ductwork::Execution",
+             foreign_key: "process_id",
+             dependent: :nullify
 
     validates :pid, uniqueness: { scope: :machine_identifier }
 
-    def self.report_heartbeat!
+    enum :role,
+         supervisor: "supervisor",
+         pipeline_advancer: "pipeline_advancer",
+         job_worker: "job_worker"
+
+    ORPHANED_CLAIM_SWEEP_MULTIPLIER = 3
+
+    def self.adopt_or_create_current!(role)
       pid = ::Process.pid
       machine_identifier = Ductwork::MachineIdentifier.fetch
+      last_heartbeat_at = Ductwork::DatabaseClock.now
+      existing = Ductwork::Process.find_by(pid:, machine_identifier:)
+
+      # NOTE: Same pid + machine_identifier can only mean the OS reused this
+      # pid, which happens after the prior process at this identity has
+      # exited -- a live process can never be replaced while still running.
+      # So its in-flight claims belong to a dead incarnation and must always
+      # be recovered here, even when the heartbeat still looks fresh because
+      # the reaper's timeout hasn't elapsed yet.
+      existing&.recover_crashed_claims!(role)
+
+      Ductwork::Process
+        .find_or_initialize_by(pid:, machine_identifier:)
+        .tap { |process| process.update!(last_heartbeat_at:, role:) }
+    end
+
+    def self.current
+      pid = ::Process.pid
+      machine_identifier = Ductwork::MachineIdentifier.fetch
+
+      find_by(pid:, machine_identifier:)
+    end
+
+    def self.reap_all!(role)
+      count = 0
+      timeout = Ductwork.configuration.supervisor_reaper_timeout
+      sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", timeout)
+
+      Ductwork.logger.debug(
+        msg: "Reaping orphaned process records",
+        role: role
+      )
+
+      where(sql).find_each do |process|
+        process.reap!(role)
+        count += 1
+      end
+
+      Ductwork.logger.debug(
+        msg: "Reaped #{count} orphaned process records",
+        count: count,
+        role: role
+      )
+    end
+
+    # NOTE: backstop for claims that lost their process_id (dependent: :nullify
+    # on a process destroy racing a fresh claim) and so are unreachable through
+    # any Process association -- `reap!`/`reap_all!` above only ever look
+    # through a Process record's own advancements/executions, and a nil
+    # process_id means there is no Process record to look through. This scans
+    # the orphaned rows directly instead.
+    #
+    # Executions are additionally scoped to availabilities with a completed_at
+    # present: an unclaimed execution legitimately has process_id: nil while
+    # it waits to be picked up (see RowLockingExecutionClaim#claim_availability,
+    # which only sets process_id at claim time), so process_id: nil alone
+    # would misidentify perfectly healthy queued work as orphaned.
+    def self.sweep_orphaned_claims!(role)
+      timeout = Ductwork.configuration.supervisor_reaper_timeout * ORPHANED_CLAIM_SWEEP_MULTIPLIER
+      count = 0
+      advancement_sql = Ductwork::DatabaseClock.ago_sql("started_at", timeout)
+
+      Ductwork::Advancement
+        .where(process_id: nil, completed_at: nil)
+        .where(advancement_sql)
+        .find_each do |advancement|
+          advancement.process_crashed!
+          count += 1
+        end
+
+      execution_sql = Ductwork::DatabaseClock.ago_sql("ductwork_availabilities.completed_at", timeout)
 
-      find_by!(pid:, machine_identifier:)
-        .update!(last_heartbeat_at: Time.current)
+      Ductwork::Execution
+        .joins(:availability)
+        .where(process_id: nil, completed_at: nil)
+        .where.not(ductwork_availabilities: { completed_at: nil })
+        .where(execution_sql)
+        .find_each do |execution|
+          execution.crashed!
+          count += 1
+        end
+
+      Ductwork.logger.debug(
+        msg: "Swept #{count} orphaned claims",
+        count: count,
+        role: role
+      )
+    end
+
+    def self.report_heartbeat!(role)
+      process = current
+
+      if process.present?
+        process.update!(last_heartbeat_at: Ductwork::DatabaseClock.now)
+        process
+      else
+        Ductwork.logger.warn(
+          msg: "Process record missing, re-adopting (likely reaped after host suspend)",
+          pid: ::Process.pid
+        )
+        adopt_or_create_current!(role)
+      end
+    end
+
+    def reap!(role, force: false)
+      timeout = Ductwork.configuration.supervisor_reaper_timeout
+      sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", timeout)
+
+      Ductwork.logger.debug(
+        msg: "Reaping orphaned process record #{id}",
+        id: id,
+        role: role
+      )
+
+      Ductwork::Record.transaction do
+        # NOTE: Callers that have already killed/stopped the process hold proof
+        # of death and pass force: true to skip the staleness guard. The row
+        # lock and existence check are kept either way to stay atomic and to
+        # avoid double-reaping a record another parent already cleaned up
+        scope = Ductwork::Process.where(id:).lock
+        scope = scope.where(sql) unless force
+
+        return unless scope.exists?
+
+        recover_crashed_claims!(role)
+        destroy
+      end
+
+      Ductwork.logger.debug(
+        msg: "Reaped orphaned process record #{id}",
+        id: id,
+        role: role
+      )
     rescue ActiveRecord::RecordNotFound
-      raise NotFoundError, "Process #{pid} not found"
+      Ductwork.logger.debug(
+        msg: "Process already reaped by another parent",
+        id: id,
+        role: role
+      )
+    end
+
+    def recover_crashed_claims!(role)
+      Ductwork.logger.debug(
+        msg: "Recovering in-flight claims on reused process record #{id}",
+        id: id,
+        role: role
+      )
+
+      Ductwork::Record.transaction do
+        advancements.where(completed_at: nil).find_each(&:process_crashed!)
+        executions.where(completed_at: nil).find_each(&:crashed!)
+      end
+    end
+
+    def healthy?
+      timeout = Ductwork.configuration.supervisor_reaper_timeout
+      sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", timeout)
+
+      self.class.where(id:).where(sql).none?
     end
   end
 end