ductwork 1.1.0 → 1.1.2

data/lib/ductwork/models/execution.rb

@@ -12,6 +12,12 @@ module Ductwork
     validates :started_at, presence: true
 
     FAILED_EXECUTION_TIMEOUT = 10.seconds
+    ERROR_MESSAGE_LIMIT = 255
+    # NOTE: `error_backtrace` is a text column, which is unbounded on Postgres
+    # and SQLite but caps at 64KB on MySQL. The frames nearest the raise are
+    # the diagnostic ones; everything below is the work loop and the framework,
+    # identical on every crash
+    ERROR_BACKTRACE_LIMIT = 10
 
     class CommitFailed < StandardError; end
 
@@ -23,7 +29,7 @@ module Ductwork
         job_klass: job.klass
       )
       args = JSON.parse(job.input_args)["args"]
-      instance = Object.const_get(job.klass).build_for_execution(job.step.run_id, *args)
+      instance = Object.const_get(job.klass).build_for_execution(job.step.run_id, job.step.id, *args)
       create_attempt!(started_at: Time.current)
       output_payload = nil
 
@@ -52,13 +58,13 @@ module Ductwork
     end
 
     def succeeded!(output_payload, owner_process_id)
-      completed_at = Time.current
+      completed_at = updated_at = Time.current
       payload = JSON.dump({ payload: output_payload })
 
       Ductwork::Record.transaction do
         rows_updated = Ductwork::Execution
                        .where(id: id, completed_at: nil, process_id: owner_process_id)
-                       .update_all(completed_at:)
+                       .update_all(completed_at:, updated_at:)
 
         if rows_updated.zero?
           raise Ductwork::Execution::CommitFailed, "Reaper clobbered claimed job execution"
@@ -71,23 +77,25 @@ module Ductwork
       end
     end
 
-    def crashed! # rubocop:todo Metrics
+    def crashed!(error) # rubocop:todo Metrics
       run = job.step.run
       max_crash = Ductwork.configuration.job_worker_max_crash(
         pipeline: run.pipeline_klass,
         step: job.klass
       )
 
+      completed_at = updated_at = Time.current
+
       Ductwork::Record.transaction do # rubocop:todo Metrics/BlockLength
         rows_updated = Ductwork::Execution
                        .where(id: id, process_id: process_id, completed_at: nil)
-                       .update_all(completed_at: Time.current)
+                       .update_all(completed_at:, updated_at:)
 
         return if rows_updated.zero?
 
         reload
         attempt&.update!(completed_at: Time.current)
-        create_result!(result_type: "process_crashed")
+        create_result!(result_type: "process_crashed", **error_attributes(error))
 
         if crash_count < max_crash
           new_crash_count = crash_count + 1
@@ -107,6 +115,8 @@ module Ductwork
 
           Ductwork.logger.error(
             msg: "Job exceeded crash limit and failed",
+            error_klass: error.class.to_s,
+            error_message: error.message,
             job_id: job.id,
             job_klass: job.klass,
             run_id: run.id,
@@ -118,7 +128,7 @@ module Ductwork
 
     def errored!(error, owner_process_id) # rubocop:todo Metrics
       run = job.step.run
-      completed_at = Time.current
+      completed_at = updated_at = Time.current
       max_retry = Ductwork.configuration.job_worker_max_retry(
         pipeline: run.pipeline_klass,
         step: job.klass
@@ -127,19 +137,14 @@ module Ductwork
       Ductwork::Record.transaction do # rubocop:todo Metrics/BlockLength
         rows_updated = Ductwork::Execution
                        .where(id: id, completed_at: nil, process_id: owner_process_id)
-                       .update_all(completed_at:)
+                       .update_all(completed_at:, updated_at:)
 
         if rows_updated.zero?
           raise Ductwork::Execution::CommitFailed, "Reaper clobbered claimed job execution"
         end
 
         attempt.update!(completed_at: Time.current)
-        create_result!(
-          result_type: "failure",
-          error_klass: error.class.to_s,
-          error_message: error.message,
-          error_backtrace: error.backtrace.join("\n")
-        )
+        create_result!(result_type: "failure", **error_attributes(error))
 
         if retry_count < max_retry
           retry_at = Ductwork::DatabaseClock.now + FAILED_EXECUTION_TIMEOUT
@@ -181,6 +186,28 @@ module Ductwork
 
     private
 
+    def error_attributes(error)
+      {
+        error_klass: error.class.to_s,
+        error_message: error.message.truncate(ERROR_MESSAGE_LIMIT),
+        error_backtrace: truncated_backtrace(error),
+      }
+    end
+
+    # NOTE: `backtrace` is nil for a Ductwork::Crash marker, which is built
+    # rather than raised -- there is no stack worth capturing for "some other
+    # process noticed this claim was dead"
+    def truncated_backtrace(error)
+      frames = error.backtrace
+      return if frames.blank?
+
+      kept = frames.first(ERROR_BACKTRACE_LIMIT)
+      dropped = frames.size - kept.size
+      kept << "... #{dropped} more frames" if dropped.positive?
+
+      kept.join("\n")
+    end
+
     # NOTE: the first third (floored) of the crash budget retries immediately
     # so a one-off reaper clobber / deploy restart recovers fast; past that,
     # delay linearly so sub-cap crash loops stop hammering the database and

data/lib/ductwork/models/pipeline.rb

@@ -56,7 +56,7 @@ module Ductwork
           raise DefinitionError, "Pipeline must be defined before triggering"
         end
 
-        now = Time.current
+        now = Ductwork::DatabaseClock.now
         node = pipeline_definition.dig(:nodes, 0)
         klass = pipeline_definition.dig(:edges, node, :klass)
         definition = JSON.dump(pipeline_definition)
@@ -118,7 +118,7 @@ module Ductwork
         raise ReviveError, "Cannot revive pipeline without previous run"
       end
 
-      now = Time.current
+      now = Ductwork::DatabaseClock.now
       new_run = last_run.dup
       new_run.triggered_at = now
       new_run.started_at = now

data/lib/ductwork/models/process.rb

@@ -60,7 +60,7 @@ module Ductwork
         role: role
       )
 
-      where(sql).find_each do |process|
+      where(sql).where.not(id: current&.id).find_each do |process|
         process.reap!(role)
         count += 1
       end
@@ -105,7 +105,9 @@ module Ductwork
         .where.not(ductwork_availabilities: { completed_at: nil })
         .where(execution_sql)
         .find_each do |execution|
-          execution.crashed!
+          execution.crashed!(
+            Ductwork::OrphanedClaim.new("Swept claim with no owning process record")
+          )
           count += 1
         end
 
@@ -177,7 +179,9 @@ module Ductwork
 
       Ductwork::Record.transaction do
         advancements.where(completed_at: nil).find_each(&:process_crashed!)
-        executions.where(completed_at: nil).find_each(&:crashed!)
+        executions.where(completed_at: nil).find_each do |execution|
+          execution.crashed!(Ductwork::ProcessCrash.new("Reaped from orphaned process"))
+        end
       end
     end
 

data/lib/ductwork/models/run.rb

@@ -84,9 +84,10 @@ module Ductwork
 
       return if klass.blank?
 
+      on_halt_dispatched_at = updated_at = Time.current
       claimed = self.class
                     .where(id: id, status: "halted", on_halt_dispatched_at: nil)
-                    .update_all(on_halt_dispatched_at: Time.current)
+                    .update_all(on_halt_dispatched_at:, updated_at:)
 
       return if claimed.zero?
 

data/lib/ductwork/models/step.rb

@@ -2,6 +2,12 @@
 
 module Ductwork
   class Step < Ductwork::Record
+    # The statuses that mean this step is done running and its branch is ready
+    # to be advanced: `advancing` (the job succeeded) or `failed` (the job
+    # exhausted its retry or crash budget). Shared by branch candidate
+    # selection, branch claiming, and the guard in `Branch#advance!`.
+    ADVANCEABLE_STATUSES = %w[advancing failed].freeze
+
     belongs_to :run, class_name: "Ductwork::Run"
     belongs_to :branch, class_name: "Ductwork::Branch"
     belongs_to :source_step, class_name: "Ductwork::Step", optional: true
@@ -34,23 +40,22 @@ module Ductwork
          converge: "converge",
          dampen: "dampen"
 
-    def self.build_for_execution(run_id, *, **)
+    def self.build_for_execution(run_id, idempotency_key, *, **)
       instance = allocate
       instance.instance_variable_set(:@run_id, run_id)
+      instance.instance_variable_set(:@idempotency_key, idempotency_key)
       instance.send(:initialize, *, **)
       instance
     end
 
-    alias_attribute :idempotency_key, :id
-
     def run_id
       @run_id || (@attributes && super)
     end
 
-    # The result_type of the most recent execution to finish for this step's
-    # job, used to distinguish *why* a step failed (e.g. `errored!` writes
-    # "failure", `crashed!` writes "process_crashed"). Returns nil when no
-    # execution has produced a result yet.
+    def idempotency_key
+      @idempotency_key || (@attributes && id)
+    end
+
     def terminal_result_type
       return if job.blank?
 

data/lib/ductwork/optimistic_locking_execution_claim.rb

@@ -24,8 +24,9 @@ module Ductwork
               availability_id: id
             )
 
+            updated_at = Time.current
             @execution = find_execution
-            execution.update_columns(process_id:)
+            execution.update_columns(process_id:, updated_at:)
 
             update_state
           else
@@ -66,9 +67,11 @@ module Ductwork
     end
 
     def claim_availability
+      completed_at = updated_at = Ductwork::DatabaseClock.now
+
       Ductwork::Availability
         .where(id: id, completed_at: nil)
-        .update_all(completed_at: Time.current, process_id: process_id)
+        .update_all(completed_at:, process_id:, updated_at:)
     end
 
     def find_execution
@@ -78,20 +81,7 @@ module Ductwork
     end
 
     def update_state
-      step = execution.job.step
-
-      Ductwork::Step
-        .where(id: step.id)
-        .where.not(status: "in_progress")
-        .update_all(status: "in_progress", updated_at: Time.current)
-      Ductwork::Run
-        .where(id: step.run_id)
-        .where.not(status: "in_progress")
-        .update_all(status: "in_progress", updated_at: Time.current)
-      Ductwork::Pipeline
-        .where(id: Ductwork::Run.where(id: step.run_id).select(:pipeline_id))
-        .where.not(status: "in_progress")
-        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::ClaimedState.mark_in_progress!(execution.job.step)
     end
   end
 end

data/lib/ductwork/orphaned_claim.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class OrphanedClaim < Ductwork::Crash; end
+end

data/lib/ductwork/polling_interval.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Ductwork
+  # NOTE: pollers start in lockstep and stay there. A runner spawns its
+  # advancer (or worker) threads in a tight loop, so they all reach their first
+  # idle `sleep` within microseconds of each other, and because every
+  # subsequent sleep is the same fixed interval the phase alignment never
+  # decays on its own. Aligned pollers wake together, run the same candidate
+  # `SELECT` against the same rows, and all but one lose the claim race -- the
+  # very contention that sampling a candidate window in `BranchClaim` is meant
+  # to spread out. Spreading the wake-ups spreads the queries.
+  #
+  # The jitter is re-rolled on every sleep rather than being a per-thread
+  # offset chosen once at startup, because a burst of work re-synchronizes the
+  # pollers no matter how they were staggered: they all stay busy while the
+  # queue drains and then all go idle together the moment it empties.
+  #
+  # The spread is symmetric so the configured polling timeout stays the average
+  # wait. Someone who sets `polling_timeout: 1` still gets a one second average
+  # poll latency instead of a quietly slower one.
+  module PollingInterval
+    JITTER_RATIO = 0.25
+
+    def self.jittered(timeout)
+      spread = timeout * JITTER_RATIO
+
+      timeout + Kernel.rand(-spread..spread)
+    end
+  end
+end

data/lib/ductwork/process_crash.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class ProcessCrash < Ductwork::Crash; end
+end

data/lib/ductwork/processes/job_worker.rb

@@ -8,17 +8,25 @@ module Ductwork
       def initialize(pipeline, id)
         @pipeline = pipeline
         @id = id
-        @running_context = Ductwork::RunningContext.new
+        @running_context = nil
         @thread = nil
         @last_heartbeat_at = Time.current
       end
 
       def start
+        return false if alive?
+
+        @running_context = Ductwork::RunningContext.new
+        @last_heartbeat_at = Time.current
         @thread = Thread.new { work_loop }
         @thread.name = name
+
+        true
       end
 
       def restart
+        return false if alive?
+
         cleanup_dead_thread!
         start
       end
@@ -32,7 +40,7 @@ module Ductwork
       end
 
       def stop
-        running_context.shutdown!
+        running_context&.shutdown!
       end
 
       def kill
@@ -90,6 +98,7 @@ module Ductwork
                 role: :job_worker,
                 pipeline: pipeline
               )
+
               sleep(polling_timeout)
             end
           rescue Ductwork::Execution::CommitFailed => e
@@ -107,7 +116,7 @@ module Ductwork
           rescue StandardError => e
             if execution.present?
               Ductwork.wrap_with_app_executor do
-                execution.crashed!
+                execution.crashed!(e)
               end
             end
 
@@ -121,9 +130,15 @@ module Ductwork
               pipeline: pipeline
             )
           ensure
+            # NOTE: the rescue above already labelled anything that raised a
+            # StandardError, so reaching here with an uncommitted claim means
+            # either that its `crashed!` itself failed or that the iteration
+            # left by a path no rescue saw
             if execution.present? && execution.reload.completed_at.nil?
               Ductwork.wrap_with_app_executor do
-                execution.crashed!
+                execution.crashed!(
+                  Ductwork::AbandonedClaim.new("Work loop iteration exited holding an uncommitted claim")
+                )
               rescue StandardError
                 nil
               end
@@ -146,7 +161,9 @@ module Ductwork
 
       def cleanup_dead_thread!
         if execution.present? && execution.reload.completed_at.nil?
-          execution.crashed!
+          execution.crashed!(
+            Ductwork::ThreadCrash.new("Worker thread died holding an uncommitted claim")
+          )
         end
       ensure
         @execution = nil
@@ -167,7 +184,9 @@ module Ductwork
       end
 
       def polling_timeout
-        Ductwork.configuration.job_worker_polling_timeout(pipeline)
+        timeout = Ductwork.configuration.job_worker_polling_timeout(pipeline)
+
+        Ductwork::PollingInterval.jittered(timeout)
       end
     end
   end

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -8,17 +8,25 @@ module Ductwork
       def initialize(klass, index = nil)
         @klass = klass
         @index = index || 0
-        @running_context = Ductwork::RunningContext.new
+        @running_context = nil
         @last_heartbeat_at = Time.current
         @thread = nil
       end
 
       def start
+        return false if alive?
+
+        @running_context = Ductwork::RunningContext.new
+        @last_heartbeat_at = Time.current
         @thread = Thread.new { work_loop }
         @thread.name = name
+
+        true
       end
 
       def restart
+        return false if alive?
+
         cleanup_dead_thread!
         start
       end
@@ -32,7 +40,7 @@ module Ductwork
       end
 
       def stop
-        running_context.shutdown!
+        running_context&.shutdown!
       end
 
       def kill
@@ -62,10 +70,10 @@ module Ductwork
         )
 
         while running_context.running?
-          claimed = false
+          outcome = :idle
 
           Ductwork.wrap_with_app_executor do
-            claimed = Branch.with_latest_claimed(klass) do |branch, transition, advancement|
+            outcome = Branch.with_latest_claimed(klass) do |branch, transition, advancement|
               @branch = branch
               @original_claim_token = branch.claim_token
 
@@ -80,7 +88,7 @@ module Ductwork
 
           @last_heartbeat_at = Time.current
 
-          if !claimed
+          if outcome == :idle
             sleep(polling_timeout)
           end
         end
@@ -126,7 +134,9 @@ module Ductwork
       end
 
       def polling_timeout
-        Ductwork.configuration.pipeline_polling_timeout(klass)
+        timeout = Ductwork.configuration.pipeline_polling_timeout(klass)
+
+        Ductwork::PollingInterval.jittered(timeout)
       end
     end
   end

data/lib/ductwork/processes/process_supervisor.rb

@@ -45,6 +45,7 @@ module Ductwork
         while running_context.running?
           sleep(Ductwork.configuration.supervisor_polling_timeout)
           check_workers
+          report_heartbeat!
           reap_process_records
         end
 
@@ -61,6 +62,7 @@ module Ductwork
         terminate_gracefully
         wait_for_workers_to_exit
         terminate_immediately
+        reap_own_process_record!
         run_hooks_for(:stop)
       end
 
@@ -111,6 +113,19 @@ module Ductwork
         )
       end
 
+      def report_heartbeat!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.report_heartbeat!(:supervisor)
+        end
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reporting heartbeat failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :process_supervisor
+        )
+      end
+
       def reap_process_records
         Ductwork.wrap_with_app_executor do
           Ductwork::Process.reap_all!(:process_supervisor)
@@ -230,6 +245,12 @@ module Ductwork
         end
       end
 
+      def reap_own_process_record!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.current&.reap!(:process_supervisor, force: true)
+        end
+      end
+
       def run_hooks_for(event)
         Ductwork.hooks[:supervisor].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/worker_health_check.rb

@@ -3,17 +3,22 @@
 module Ductwork
   module Processes
     class WorkerHealthCheck
+      KILL_BUDGET = 3
+      JOIN_TIMEOUT = 1
+
       def initialize(workers, role)
         @workers = workers
         @role = role
       end
 
       def check
+        deadline = Time.current + KILL_BUDGET
+
         workers.each do |worker|
           if !worker.alive?
             restart_dead_worker(worker)
           elsif worker.stuck?
-            restart_stuck_worker(worker)
+            restart_stuck_worker(worker, deadline)
           end
         end
       end
@@ -23,13 +28,9 @@ module Ductwork
       attr_reader :workers, :role
 
       def restart_dead_worker(worker)
-        worker.restart
+        claimed_args = claimed_args_for(worker)
 
-        claimed_args = if worker.is_a?(Ductwork::Processes::PipelineAdvancer)
-                         { branch_id: worker.branch&.id }
-                       else
-                         { job_id: worker.execution&.job_id }
-                       end
+        worker.restart
 
         Ductwork.logger.warn(
           msg: "Restarted dead thread",
@@ -39,9 +40,17 @@ module Ductwork
         )
       end
 
-      def restart_stuck_worker(worker)
-        worker.kill if worker.alive?
-        worker.join(1)
+      def restart_stuck_worker(worker, deadline)
+        if !dead_after_kill?(worker, deadline)
+          Ductwork.logger.warn(
+            msg: "Unable to confirm stuck thread died, deferring restart",
+            role: role,
+            thread: worker.name
+          )
+
+          return
+        end
+
         worker.restart
 
         Ductwork.logger.warn(
@@ -50,6 +59,25 @@ module Ductwork
           thread: worker.name
         )
       end
+
+      def dead_after_kill?(worker, deadline)
+        worker.kill
+
+        while worker.alive? && Time.current < deadline
+          worker.join(JOIN_TIMEOUT)
+          worker.kill if worker.alive?
+        end
+
+        !worker.alive?
+      end
+
+      def claimed_args_for(worker)
+        if worker.is_a?(Ductwork::Processes::PipelineAdvancer)
+          { branch_id: worker.branch&.id }
+        else
+          { job_id: worker.execution&.job_id }
+        end
+      end
     end
   end
 end

data/lib/ductwork/row_locking_execution_claim.rb

@@ -50,28 +50,15 @@ module Ductwork
 
       return unless availability
 
-      completed_at = Time.current
+      completed_at = updated_at = Ductwork::DatabaseClock.now
       @execution = availability.execution
 
-      availability.update_columns(completed_at:, process_id:)
-      execution.update_columns(process_id:)
+      availability.update_columns(completed_at:, process_id:, updated_at:)
+      execution.update_columns(process_id:, updated_at:)
     end
 
     def update_state
-      step = execution.job.step
-
-      Ductwork::Step
-        .where(id: step.id)
-        .where.not(status: "in_progress")
-        .update_all(status: "in_progress", updated_at: Time.current)
-      Ductwork::Run
-        .where(id: step.run_id)
-        .where.not(status: "in_progress")
-        .update_all(status: "in_progress", updated_at: Time.current)
-      Ductwork::Pipeline
-        .where(id: Ductwork::Run.where(id: step.run_id).select(:pipeline_id))
-        .where.not(status: "in_progress")
-        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::ClaimedState.mark_in_progress!(execution.job.step)
     end
   end
 end

data/lib/ductwork/thread_crash.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class ThreadCrash < Ductwork::Crash; end
+end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "1.1.0"
+  VERSION = "1.1.2"
 end

data/lib/ductwork.rb

@@ -11,8 +11,12 @@ require "zeitwerk"
 
 module Ductwork
   class << self
-    attr_accessor :app_executor, :configuration, :loader, :logger
-    attr_writer :defined_pipelines, :hooks
+    attr_accessor :app_executor, :loader, :logger
+    attr_writer :configuration, :defined_pipelines, :hooks
+
+    def configuration
+      @configuration ||= Ductwork::Configuration.new
+    end
 
     def eager_load
       loader.eager_load