ductwork 0.25.0 → 1.0.0

data/app/views/ductwork/pipelines/show.html.erb

@@ -5,8 +5,8 @@
     </h1>
 
     <h2>
-      <span class="status-pill big-pill <%= @pipeline.status %>">
-        <%= @pipeline.status.gsub("_", "-") %>
+      <span class="status-pill big-pill <%= @last_run.status %>">
+        <%= @last_run.status.gsub("_", "-") %>
       </span>
     </h2>
   </div>
@@ -30,34 +30,34 @@
         <div>
           <label>Runtime</label>
           <span>
-            <% if @pipeline.completed_at.nil? %>
-              <div data-started-at=<%= @pipeline.started_at.iso8601 %>>
+            <% if @last_run.completed_at.nil? %>
+              <div data-started-at=<%= @last_run.started_at.iso8601 %>>
                 <span id="elapsed-timer">0h 0m 0s</span>
               </div>
             <% else %>
-              <%= formatted_time_distance(@pipeline.started_at, @pipeline.completed_at) %>
+              <%= formatted_time_distance(@last_run.started_at, @last_run.completed_at) %>
             <% end %>
           </span>
         </div>
         <div class="metric-heading">
           <label>Triggered At</label>
           <span>
-            <%= @pipeline.triggered_at.iso8601(3) %>
+            <%= @last_run.triggered_at.iso8601(3) %>
           </span>
         </div>
       </div>
 
       <div>
         <div class="metric-heading">
-          <label>Last Advanced At</label>
+          <label>Started At</label>
           <span>
-            <%= @pipeline.last_advanced_at.iso8601(3) %>
+            <%= @last_run.started_at.iso8601(3) %>
           </span>
         </div>
         <div>
           <label>Completed At</label>
           <span>
-            <%= @pipeline.completed_at&.iso8601(3) %>
+            <%= @last_run.completed_at&.iso8601(3) %>
           </span>
         </div>
       </div>
@@ -66,12 +66,12 @@
     <div style="margin-top: 1rem;">
       <label>Definition</label>
       <code style="width: 100%;">
-        <%= @pipeline.parsed_definition %>
+        <%= @last_run.parsed_definition %>
       </code>
     </div>
   </article>
 
-  <h3>Steps (<%= @pipeline.steps.count %>)</h3>
+  <h3>Steps (<%= @last_run.steps.count %>)</h3>
 
   <div class="separate-content">
     <div class="filter-group">
@@ -245,7 +245,7 @@
                       <td><%= execution.retry_count + 1 %></td>
                       <td><%= execution.started_at.strftime("%H:%M:%S.%3N") %></td>
                       <td><%= execution.availability.completed_at&.strftime("%H:%M:%S.%3N") %></td>
-                      <td><%= execution.run&.started_at&.strftime("%H:%M:%S.%3N") %></td>
+                      <td><%= execution.attempt&.started_at&.strftime("%H:%M:%S.%3N") %></td>
                       <td><%= execution.completed_at&.strftime("%H:%M:%S.%3N") %></td>
                       <td>
                         <% if execution.result.present? && execution.result.failure? %>
@@ -269,7 +269,7 @@
                                   <%= execution.result.error_message %>
                                 </h5>
                                 <code>
-                                  <%= execution.result.error_backtrace.split("\n").each do |line| %>
+                                  <%= (execution.result.error_backtrace&.split("\n") || []).each do |line| %>
                                     <div><%= line %></div>
                                   <% end %>
                                 </code>

data/app/views/ductwork/step_errors/index.html.erb

@@ -56,7 +56,7 @@
           </td>
           <td>
             <code>
-              <%= link_to(result.execution.job.step.pipeline_id, pipeline_path(result.execution.job.step.pipeline_id)) %>
+              <%= link_to(result.execution.job.step.run.pipeline_id, pipeline_path(result.execution.job.step.run.pipeline_id)) %>
             </code>
           </td>
           <td>
@@ -85,7 +85,7 @@
                     <%= result.error_message %>
                   </h5>
                   <code>
-                    <%= result.error_backtrace.split("\n").each do |line| %>
+                    <%= (result.error_backtrace&.split("\n") || []).each do |line| %>
                       <div><%= line %></div>
                     <% end %>
                   </code>

data/lib/ductwork/branch_claim.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class BranchClaim
+    attr_reader :transition, :advancement, :token
+
+    def initialize(pipeline_klass)
+      @pipeline_klass = pipeline_klass
+      @claimed_for_advancing_at = nil
+    end
+
+    def latest
+      # NOTE: an advancement must attach to a live process record so the reaper
+      # can find and recover it if this process dies. `Process.current` is only
+      # nil when our record was reaped for a stale heartbeat (e.g. host suspend)
+      # and the runner has not re-adopted it yet — i.e. the system has already
+      # declared this process dead. Claiming as a presumed-dead zombie would
+      # create an advancement with a nil `process_id` that no reaper sweep can
+      # reach, orphaning the branch. Skip this cycle instead; the runner
+      # re-adopts within its heartbeat interval.
+      process = Ductwork::Process.current
+
+      return log_no_process if process.nil?
+
+      id = find_candidate_branch_id
+
+      return log_no_branches if id.blank?
+
+      rows_updated = claim_and_setup_records(id, process)
+
+      if rows_updated == 1
+        Ductwork::Branch.find(id)
+      else
+        log_race_condition(id)
+      end
+    rescue ActiveRecord::InvalidForeignKey => e
+      # NOTE: our own `process` record was reaped (heartbeat-stale, destroyed)
+      # between the nil-check above and `transition.advancements.create!`
+      # inside `claim_and_setup_records`. The insert's FK check blocks on the
+      # reaper's row lock and then fails once the parent is gone, rolling back
+      # the whole claim transaction (branch + transition + advancement) --
+      # nothing is left half-committed. Treat it like any other lost claim
+      # race instead of letting it propagate and kill the advancer thread.
+      log_process_reaped_mid_claim(id, e)
+    end
+
+    private
+
+    attr_reader :pipeline_klass, :claimed_for_advancing_at
+
+    def find_candidate_branch_id
+      Ductwork::Branch
+        .in_progress
+        .where(pipeline_klass:, claimed_for_advancing_at:)
+        .where(steps: Ductwork::Step.where(status: %w[advancing failed]))
+        .order(:last_advanced_at)
+        .limit(1)
+        .pluck(:id)
+        .first
+    end
+
+    def claim_and_setup_records(id, process)
+      now = Time.current
+      @token = SecureRandom.uuid
+
+      Ductwork::Record.transaction do
+        rows_updated = Ductwork::Branch
+                       .where(id:, claimed_for_advancing_at:)
+                       .update_all(
+                         claimed_for_advancing_at: now,
+                         claim_token: token,
+                         status: :advancing
+                       )
+
+        if rows_updated == 1
+          branch = Branch.find(id)
+          @transition = find_or_create_transition(branch, now)
+          Ductwork::FaultInjection.checkpoint(:before_advancement_create)
+          @advancement = transition.advancements.create!(
+            process: process,
+            started_at: now,
+            crash_count: next_crash_count(transition)
+          )
+        end
+
+        rows_updated
+      end
+    end
+
+    def find_or_create_transition(branch, now)
+      existing = branch
+                 .transitions
+                 .where(completed_at: nil)
+                 .order(started_at: :desc)
+                 .limit(1)
+                 .first
+
+      if existing
+        fail_abandoned_advancement(existing, now)
+        existing
+      else
+        branch.transitions.create!(
+          in_step: branch.latest_step,
+          started_at: now
+        )
+      end
+    end
+
+    # NOTE: each crash spawns a fresh advancement at re-claim, so the running
+    # crash total lives on the advancement and carries forward (mirroring
+    # `Ductwork::Execution#crash_count`). The prior advancement is read after
+    # `find_or_create_transition` has already marked any abandoned in-flight
+    # advancement as a crash, so both the reaper/thread-cleanup path and the
+    # `fail_abandoned_advancement` path are reflected here. A fresh transition
+    # starts at 0; a non-crash errored prior carries the total unchanged.
+    def next_crash_count(transition)
+      prior = transition.advancements.order(started_at: :desc).first
+      base = prior&.crash_count || 0
+
+      if prior&.crash?
+        base + 1
+      else
+        base
+      end
+    end
+
+    def fail_abandoned_advancement(transition, now)
+      transition
+        .advancements
+        .where(completed_at: nil)
+        .order(started_at: :desc)
+        .limit(1)
+        .first
+        &.update!(
+          completed_at: now,
+          error_klass: "Ductwork::ProcessCrash",
+          error_message: "Advancement was abandoned from a process crash"
+        )
+    end
+
+    def log_no_process
+      Ductwork.logger.debug(
+        msg: "No live process record, skipping branch claim",
+        pipeline: pipeline_klass,
+        role: :pipeline_advancer
+      )
+
+      nil
+    end
+
+    def log_no_branches
+      Ductwork.logger.debug(
+        msg: "No branches needs advancing",
+        pipeline: pipeline_klass,
+        role: :pipeline_advancer
+      )
+
+      nil
+    end
+
+    def log_race_condition(id)
+      Ductwork.logger.debug(
+        msg: "Did not claim branch, avoided race condition",
+        branch_id: id,
+        pipeline_klass: pipeline_klass,
+        role: :pipeline_advancer
+      )
+
+      nil
+    end
+
+    def log_process_reaped_mid_claim(id, error)
+      Ductwork.logger.warn(
+        msg: "Did not claim branch, our process record was reaped mid-claim",
+        branch_id: id,
+        pipeline_klass: pipeline_klass,
+        error_klass: error.class.to_s,
+        error_message: error.message,
+        role: :pipeline_advancer
+      )
+
+      nil
+    end
+  end
+end

data/lib/ductwork/cli.rb

@@ -4,49 +4,60 @@ require "optparse"
 
 module Ductwork
   class CLI
+    DEFAULT_COMMAND = "start"
+
     def self.start!(args)
       new(args).start!
     end
 
     def initialize(args)
-      @args = args
-      @options = {}
-    end
-
-    def start!
-      option_parser.parse!(args)
-      auto_configure
-      puts banner
-      launch_processes
-    end
-
-    private
-
-    attr_reader :args, :options
-
-    def option_parser
-      OptionParser.new do |op|
+      @raw_args = args.dup
+      @configuration_options = {}
+      @health_check_options = {}
+      @top_level_parser = OptionParser.new do |op|
         op.banner = "ductwork [options]"
 
         op.on("-c", "--config PATH", "path to YAML config file") do |arg|
-          options[:path] = arg
+          configuration_options[:path] = arg
         end
 
         op.on("-h", "--help", "Prints this help") do
           puts op
+          puts "\nCommands:\n  start\n  health"
           exit
         end
+      end
+    end
 
-        op.on("-v", "--version", "Prints the version") do
-          puts "Ductwork #{Ductwork::VERSION}"
-          exit
-        end
+    def start!
+      parse!
+      auto_configure
+      execute_command
+    end
+
+    private
+
+    attr_reader :raw_args, :configuration_options, :health_check_options,
+                :top_level_parser, :command
+
+    def parse!
+      top_level_parser.order(raw_args)
+      @command = raw_args.shift || DEFAULT_COMMAND
+
+      if command == "start"
+        start_command_parser.parse!(raw_args)
+      elsif command == "health"
+        health_command_parser.parse!(raw_args)
+      else
+        warn "Unknown command: #{command}"
+        puts top_level_parser
+        exit 1
       end
     end
 
     def auto_configure
-      options[:role] = ENV.fetch("DUCTWORK_ROLE", nil)
-      Ductwork.configuration = Configuration.new(**options)
+      configuration_options[:role] = ENV.fetch("DUCTWORK_ROLE", nil)
+      Ductwork.configuration = Configuration.new(**configuration_options)
       Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
                           Rails.logger
                         else
@@ -73,8 +84,46 @@ module Ductwork
       BANNER
     end
 
+    def execute_command
+      if command == "start"
+        launch_processes
+      elsif command == "health"
+        check_health
+      end
+    end
+
     def launch_processes
+      puts banner
+
       Ductwork::Processes::Launcher.start_processes!
     end
+
+    def check_health
+      Ductwork::Processes::HealthCheck.run(**health_check_options)
+    end
+
+    def start_command_parser
+      OptionParser.new do |op|
+        op.banner = "ductwork start [options]"
+
+        op.on("-c", "--config PATH") do |arg|
+          configuration_options[:path] = arg
+        end
+      end
+    end
+
+    def health_command_parser
+      OptionParser.new do |op|
+        op.banner = "ductwork health [options]"
+
+        op.on("-p", "--pid PID") do |arg|
+          health_check_options[:pid] = arg.to_i
+        end
+
+        op.on("-V", "--verbose", "Prints all the health check data") do
+          health_check_options[:verbose] = true
+        end
+      end
+    end
   end
 end

data/lib/ductwork/configuration.rb

@@ -6,18 +6,24 @@ module Ductwork
     DEFAULT_FILE_PATH = "config/ductwork.yml"
     DEFAULT_FORKING = "default" # fork pipeline advancer and job workers
     DEFAULT_JOB_WORKER_COUNT = 5 # threads
+    DEFAULT_JOB_WORKER_MAX_CRASH = 10 # attempts
     DEFAULT_JOB_WORKER_MAX_RETRY = 3 # attempts
     DEFAULT_JOB_WORKER_POLLING_TIMEOUT = 1 # second
     DEFAULT_JOB_WORKER_SHUTDOWN_TIMEOUT = 20 # seconds
     DEFAULT_LOGGER_LEVEL = ::Logger::INFO
     DEFAULT_LOGGER_SOURCE = "default" # `Logger` instance writing to STDOUT
+    DEFAULT_PIPELINE_ADVANCER_MAX_CRASH = 10 # attempts
+    DEFAULT_PIPELINE_ADVANCER_MAX_RETRY = 10 # attempts
+    DEFAULT_PIPELINE_ADVANCER_THREAD_COUNT = 3 # threads
     DEFAULT_PIPELINE_POLLING_TIMEOUT = 1 # second
     DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT = 20 # seconds
     DEFAULT_ROLE = "all" # supervisor, pipeline advancer, and job workers
     DEFAULT_STEPS_MAX_DEPTH = -1 # unlimited count
     DEFAULT_SUPERVISOR_POLLING_TIMEOUT = 1 # second
+    DEFAULT_SUPERVISOR_REAPER_TIMEOUT = 300 # seconds
     DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT = 30 # seconds
     DEFAULT_LOGGER = ::Logger.new($stdout)
+    PIPELINES_DIRECTORIES = %w[app/pipelines app/workflows].freeze
     PIPELINES_WILDCARD = "*"
     VALID_ROLES = %w[all advancer worker].freeze
 
@@ -25,10 +31,14 @@ module Ductwork
 
     attr_writer :job_worker_count, :job_worker_polling_timeout,
                 :job_worker_shutdown_timeout, :job_worker_max_retry,
+                :job_worker_max_crash,
                 :logger_level,
+                :pipeline_advancer_count, :pipeline_advancer_max_crash,
+                :pipeline_advancer_max_retry,
                 :pipeline_polling_timeout, :pipeline_shutdown_timeout,
                 :steps_max_depth,
-                :supervisor_polling_timeout, :supervisor_shutdown_timeout
+                :supervisor_polling_timeout, :supervisor_reaper_timeout,
+                :supervisor_shutdown_timeout
 
     def initialize(path: DEFAULT_FILE_PATH, role: nil)
       full_path = Pathname.new(path)
@@ -57,9 +67,10 @@ module Ductwork
       raw_pipelines = config[:pipelines] || []
 
       if raw_pipelines == PIPELINES_WILDCARD
-        Dir
-          .glob("**/*.rb", base: "app/pipelines")
+        PIPELINES_DIRECTORIES
+          .flat_map { |dir| Dir.glob("**/*.rb", base: dir) }
           .map { |path| path.delete_suffix(".rb").camelize }
+          .uniq
       else
         raw_pipelines.map(&:strip)
       end
@@ -81,6 +92,24 @@ module Ductwork
       end
     end
 
+    def job_worker_max_crash(pipeline: nil, step: nil) # rubocop:disable Metrics
+      return @job_worker_max_crash if instance_variable_defined?(:@job_worker_max_crash)
+
+      pipeline ||= :default
+      step ||= :default
+      base_config = config.dig(:job_worker, :max_crash)
+
+      if base_config.is_a?(Hash) && base_config[pipeline.to_sym].is_a?(Hash)
+        pipeline_config = config.dig(:job_worker, :max_crash, pipeline.to_sym)
+
+        pipeline_config[step.to_sym] || pipeline_config[:default] || DEFAULT_JOB_WORKER_MAX_CRASH
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || DEFAULT_JOB_WORKER_MAX_CRASH
+      else
+        base_config || DEFAULT_JOB_WORKER_MAX_CRASH
+      end
+    end
+
     def job_worker_max_retry(pipeline: nil, step: nil) # rubocop:disable Metrics
       return @job_worker_max_retry if instance_variable_defined?(:@job_worker_max_retry)
 
@@ -125,6 +154,28 @@ module Ductwork
       @logger_source ||= fetch_logger_source
     end
 
+    def pipeline_advancer_count(pipeline = nil)
+      pipeline ||= :default
+      default = DEFAULT_PIPELINE_ADVANCER_THREAD_COUNT
+      base_config = config.dig(:pipeline_advancer, :count)
+
+      if instance_variable_defined?(:@pipeline_advancer_count)
+        @pipeline_advancer_count
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || default
+      else
+        base_config || default
+      end
+    end
+
+    def pipeline_advancer_max_crash
+      @pipeline_advancer_max_crash ||= fetch_pipeline_advancer_max_crash
+    end
+
+    def pipeline_advancer_max_retry
+      @pipeline_advancer_max_retry ||= fetch_pipeline_advancer_max_retry
+    end
+
     def pipeline_polling_timeout(pipeline = nil)
       pipeline ||= :default
       default = DEFAULT_PIPELINE_POLLING_TIMEOUT
@@ -169,6 +220,10 @@ module Ductwork
       @supervisor_polling_timeout ||= fetch_supervisor_polling_timeout
     end
 
+    def supervisor_reaper_timeout
+      @supervisor_reaper_timeout ||= fetch_supervisor_reaper_timeout
+    end
+
     def supervisor_shutdown_timeout
       @supervisor_shutdown_timeout ||= fetch_supervisor_shutdown_timeout
     end
@@ -190,6 +245,16 @@ module Ductwork
       config.dig(:logger, :source) || DEFAULT_LOGGER_SOURCE
     end
 
+    def fetch_pipeline_advancer_max_crash
+      config.dig(:pipeline_advancer, :max_crash) ||
+        DEFAULT_PIPELINE_ADVANCER_MAX_CRASH
+    end
+
+    def fetch_pipeline_advancer_max_retry
+      config.dig(:pipeline_advancer, :max_retry) ||
+        DEFAULT_PIPELINE_ADVANCER_MAX_RETRY
+    end
+
     def fetch_pipeline_shutdown_timeout
       config.dig(:pipeline_advancer, :shutdown_timeout) ||
         DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT
@@ -200,6 +265,11 @@ module Ductwork
         DEFAULT_SUPERVISOR_POLLING_TIMEOUT
     end
 
+    def fetch_supervisor_reaper_timeout
+      config.dig(:supervisor, :reaper_timeout) ||
+        DEFAULT_SUPERVISOR_REAPER_TIMEOUT
+    end
+
     def fetch_supervisor_shutdown_timeout
       config.dig(:supervisor, :shutdown_timeout) ||
         DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT

data/lib/ductwork/context.rb

@@ -4,42 +4,40 @@ module Ductwork
   class Context
     class OverwriteError < StandardError; end
 
-    def initialize(pipeline_id)
-      @pipeline_id = pipeline_id
+    def initialize(run_id)
+      @run_id = run_id
     end
 
     def get(key)
       raise ArgumentError, "Key must be a string" if !key.is_a?(String)
 
-      Ductwork.wrap_with_app_executor do
-        Ductwork::Tuple
-          .select(:serialized_value)
-          .find_by(pipeline_id:, key:)
-          &.value
-      end
+      Ductwork::Tuple
+        .select(:serialized_value)
+        .find_by(run_id:, key:)
+        &.value
     end
 
     def set(key, value, overwrite: false)
       attributes = {
         id: SecureRandom.uuid_v7,
-        pipeline_id: pipeline_id,
+        run_id: run_id,
         key: key,
         serialized_value: Ductwork::Tuple.serialize(value),
         first_set_at: Time.current,
         last_set_at: Time.current,
       }
-      unique_by = %i[pipeline_id key]
+      opts = if Ductwork::Tuple.connection.adapter_name.match?(/mysql|trilogy/i)
+               {}
+             else
+               { unique_by: %i[run_id key] }
+             end
 
       if overwrite
-        Ductwork.wrap_with_app_executor do
-          Ductwork::Tuple.upsert(attributes, unique_by:)
-        end
+        Ductwork::Tuple.upsert(attributes, **opts)
       else
-        result = Ductwork.wrap_with_app_executor do
-          Ductwork::Tuple.insert(attributes, unique_by:)
-        end
-
-        if result.rows.none?
+        begin
+          Ductwork::Tuple.create!(attributes)
+        rescue ActiveRecord::RecordNotUnique
           raise Ductwork::Context::OverwriteError, "Can only set value once"
         end
       end
@@ -49,6 +47,6 @@ module Ductwork
 
     private
 
-    attr_reader :pipeline_id
+    attr_reader :run_id
   end
 end