ductwork 0.26.0 → 1.0.0

data/lib/ductwork/processes/process_supervisor.rb

@@ -2,7 +2,12 @@
 
 module Ductwork
   module Processes
-    class ProcessSupervisor
+    class ProcessSupervisor # rubocop:todo Metrics/ClassLength
+      # Upper bound on how long sig_kill_process will block the supervisor
+      # loop waiting for a KILL'd child to be reaped (see sig_kill_process).
+      KILL_REAP_TIMEOUT = 5 # seconds
+      KILL_REAP_INTERVAL = 0.1
+
       attr_reader :workers
 
       def initialize
@@ -29,6 +34,8 @@ module Ductwork
       end
 
       def run
+        adopt_or_create_process!
+
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :process_supervisor,
@@ -38,6 +45,7 @@ module Ductwork
         while running_context.running?
           sleep(Ductwork.configuration.supervisor_polling_timeout)
           check_workers
+          reap_process_records
         end
 
         shutdown
@@ -60,6 +68,12 @@ module Ductwork
 
       attr_reader :running_context
 
+      def adopt_or_create_process!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.adopt_or_create_current!(:supervisor)
+        end
+      end
+
       def check_workers
         Ductwork.logger.debug(
           msg: "Checking workers are alive",
@@ -67,8 +81,10 @@ module Ductwork
         )
 
         workers.each do |worker|
-          if process_dead?(worker[:pid])
+          if heartbeat_stale?(worker[:pid])
             old_pid = worker[:pid]
+            sig_kill_process(old_pid)
+            reap_process_record!(old_pid)
             new_pid = fork do
               worker[:block].call(worker[:metadata])
             end
@@ -86,6 +102,27 @@ module Ductwork
           msg: "All workers are alive or revived",
           role: :process_supervisor
         )
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Checking workers failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :process_supervisor
+        )
+      end
+
+      def reap_process_records
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.reap_all!(:process_supervisor)
+          Ductwork::Process.sweep_orphaned_claims!(:process_supervisor)
+        end
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reaping process records failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :process_supervisor
+        )
       end
 
       def terminate_gracefully
@@ -97,6 +134,8 @@ module Ductwork
             signal: :TERM
           )
           ::Process.kill(:TERM, worker[:pid])
+        rescue Errno::ESRCH
+          # Child already exited; nothing to terminate
         end
       end
 
@@ -121,38 +160,76 @@ module Ductwork
 
       def terminate_immediately
         workers.each_with_index do |worker, index|
-          Ductwork.logger.debug(
+          Ductwork.logger.warn(
             msg: "Sending KILL signal to process (#{worker[:pid]})",
             role: :process_supervisor,
             pid: worker[:pid],
             signal: :KILL
           )
-          ::Process.kill(:KILL, worker[:pid])
-          ::Process.wait(worker[:pid])
+          sig_kill_process(worker[:pid])
+          reap_process_record!(worker[:pid])
           workers[index] = nil
-          Ductwork.logger.debug(
+          Ductwork.logger.warn(
             msg: "Child process (#{worker[:pid]}) killed after timeout",
             role: :process_supervisor,
             pid: worker[:pid]
           )
-        rescue Errno::ESRCH, Errno::ECHILD
-          # no-op because process is already dead
         end
 
         @workers = workers.compact
       end
 
-      def process_dead?(pid)
+      def heartbeat_stale?(pid)
         machine_identifier = Ductwork::MachineIdentifier.fetch
+        threshold = Ductwork.configuration.supervisor_reaper_timeout - 10.seconds
+        sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", threshold)
 
         Ductwork.wrap_with_app_executor do
           Ductwork::Process
             .where(pid:, machine_identifier:)
-            .where("last_heartbeat_at < ?", 5.minutes.ago)
+            .where(sql)
             .exists?
         end
       end
 
+      def sig_kill_process(pid)
+        ::Process.kill(:KILL, pid)
+
+        deadline = now + KILL_REAP_TIMEOUT
+        loop do
+          return if ::Process.wait(pid, ::Process::WNOHANG)
+
+          if now >= deadline
+            # Likely wedged in uninterruptible sleep (D state): the SIGKILL is
+            # pending but undeliverable until its syscall returns. Don't block
+            # the supervisor loop on it - the kill still applies, but the OS
+            # process is left unreaped (a zombie) until the supervisor itself
+            # exits and init reaps it. The DB process record is reaped
+            # separately by the caller.
+            Ductwork.logger.warn(
+              msg: "Process (#{pid}) did not exit after KILL; deferring reap",
+              role: :process_supervisor,
+              pid: pid
+            )
+            return
+          end
+
+          sleep(KILL_REAP_INTERVAL)
+        end
+      rescue Errno::ECHILD, Errno::ESRCH
+        # Process is already dead or reaped
+      end
+
+      def reap_process_record!(pid)
+        machine_identifier = Ductwork::MachineIdentifier.fetch
+
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process
+            .find_by(pid:, machine_identifier:)
+            &.reap!(:process_supervisor, force: true)
+        end
+      end
+
       def run_hooks_for(event)
         Ductwork.hooks[:supervisor].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/thread_supervisor.rb

@@ -9,6 +9,7 @@ module Ductwork
         @running_context = Ductwork::RunningContext.new
         @workers = []
 
+        adopt_or_create_process!
         run_hooks_for(:start)
 
         Signal.trap(:INT) { @running_context.shutdown! }
@@ -48,7 +49,9 @@ module Ductwork
 
         while running_context.running?
           sleep(Ductwork.configuration.supervisor_polling_timeout)
-          check_worker_health
+          check_workers_health
+          report_heartbeat!
+          reap_process_records
         end
 
         shutdown
@@ -58,26 +61,15 @@ module Ductwork
 
       attr_reader :running_context
 
-      def check_worker_health
-        Ductwork.logger.debug(
-          msg: "Checking workers are alive",
-          role: :thread_supervisor
-        )
-
-        workers.each do |worker|
-          if !worker.alive?
-            worker.restart
-
-            Ductwork.logger.warn(
-              msg: "Restarted supervised thread",
-              role: :thread_supervisor,
-              thread: worker.name
-            )
-          end
-        end
-
-        Ductwork.logger.debug(
-          msg: "Checked workers are alive",
+      def check_workers_health
+        Ductwork::Processes::WorkerHealthCheck
+          .new(workers, :thread_supervisor)
+          .check
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Checking worker health failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
           role: :thread_supervisor
         )
       end
@@ -88,6 +80,7 @@ module Ductwork
         workers.each(&:stop)
         await_threads_graceful_shutdown
         kill_remaining_threads
+        reap_process_record!
         run_hooks_for(:stop)
       end
 
@@ -130,6 +123,47 @@ module Ductwork
         end
       end
 
+      def adopt_or_create_process!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.adopt_or_create_current!(:supervisor)
+        end
+      end
+
+      def report_heartbeat!
+        Ductwork.logger.debug(msg: "Reporting heartbeat", role: :thread_supervisor)
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.report_heartbeat!(:supervisor)
+        end
+        Ductwork.logger.debug(msg: "Reported heartbeat", role: :thread_supervisor)
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reporting heartbeat failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :thread_supervisor
+        )
+      end
+
+      def reap_process_records
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.reap_all!(:thread_supervisor)
+          Ductwork::Process.sweep_orphaned_claims!(:thread_supervisor)
+        end
+      rescue StandardError => e
+        Ductwork.logger.warn(
+          msg: "Reaping process records failed",
+          error_klass: e.class.to_s,
+          error_message: e.message,
+          role: :thread_supervisor
+        )
+      end
+
+      def reap_process_record!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.current&.reap!(:thread_supervisor, force: true)
+        end
+      end
+
       def run_hooks_for(event)
         Ductwork.hooks[:supervisor].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/thread_supervisor_runner.rb

@@ -11,9 +11,11 @@ module Ductwork
       def run
         if Ductwork.configuration.role.in?(%w[all advancer])
           pipelines.each do |pipeline|
-            log_created_pipeline_advancer(pipeline)
-            supervisor.add_worker(metadata: { pipeline: }) do
-              pipeline_advancer.new(pipeline)
+            Ductwork.configuration.pipeline_advancer_count(pipeline).times do |index|
+              log_created_pipeline_advancer(pipeline, index)
+              supervisor.add_worker(metadata: { pipeline: }) do
+                pipeline_advancer.new(pipeline, index)
+              end
             end
           end
         end
@@ -36,11 +38,12 @@ module Ductwork
 
       attr_reader :pipelines, :supervisor
 
-      def log_created_pipeline_advancer(pipeline)
+      def log_created_pipeline_advancer(pipeline, index)
         Ductwork.logger.debug(
           msg: "Created new pipeline advancer",
           role: :thread_supervisor_runner,
-          pipeline: pipeline
+          pipeline: pipeline,
+          index: index
         )
       end
 

data/lib/ductwork/processes/worker_health_check.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class WorkerHealthCheck
+      def initialize(workers, role)
+        @workers = workers
+        @role = role
+      end
+
+      def check
+        workers.each do |worker|
+          if !worker.alive?
+            restart_dead_worker(worker)
+          elsif worker.stuck?
+            restart_stuck_worker(worker)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :workers, :role
+
+      def restart_dead_worker(worker)
+        worker.restart
+
+        claimed_args = if worker.is_a?(Ductwork::Processes::PipelineAdvancer)
+                         { branch_id: worker.branch&.id }
+                       else
+                         { job_id: worker.execution&.job_id }
+                       end
+
+        Ductwork.logger.warn(
+          msg: "Restarted dead thread",
+          role: role,
+          thread: worker.name,
+          **claimed_args
+        )
+      end
+
+      def restart_stuck_worker(worker)
+        worker.kill if worker.alive?
+        worker.join(1)
+        worker.restart
+
+        Ductwork.logger.warn(
+          msg: "Killed and restarted stuck thread",
+          role: role,
+          thread: worker.name
+        )
+      end
+    end
+  end
+end

data/lib/ductwork/row_locking_execution_claim.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class RowLockingExecutionClaim
+    def initialize(klass, owner_process_id)
+      @availability = nil
+      @execution = nil
+      @klass = klass
+      @process_id = owner_process_id
+    end
+
+    def latest
+      Ductwork::Record.transaction do
+        claim_availability
+
+        if availability.present?
+          Ductwork.logger.debug(
+            msg: "Job claimed",
+            role: :job_worker,
+            process_id: process_id,
+            availability_id: availability.id
+          )
+          update_state
+        else
+          Ductwork.logger.debug(
+            msg: "No available job to claim",
+            role: :job_worker,
+            process_id: process_id,
+            pipeline: klass
+          )
+        end
+      end
+
+      execution
+    end
+
+    private
+
+    attr_reader :availability, :execution, :klass, :process_id
+
+    def claim_availability
+      sql = Ductwork::DatabaseClock.now_sql("ductwork_availabilities.started_at")
+      @availability = Ductwork::Availability
+                      .where(sql)
+                      .where(completed_at: nil, pipeline_klass: klass)
+                      .order(:started_at)
+                      .lock("FOR UPDATE SKIP LOCKED")
+                      .limit(1)
+                      .first
+
+      return unless availability
+
+      completed_at = Time.current
+      @execution = availability.execution
+
+      availability.update_columns(completed_at:, process_id:)
+      execution.update_columns(process_id:)
+    end
+
+    def update_state
+      step = execution.job.step
+
+      Ductwork::Step
+        .where(id: step.id)
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::Run
+        .where(id: step.run_id)
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::Pipeline
+        .where(id: Ductwork::Run.where(id: step.run_id).select(:pipeline_id))
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+    end
+  end
+end

data/lib/ductwork/testing/rspec.rb

@@ -66,7 +66,7 @@ RSpec::Matchers.define(:have_set_context) do |expected|
   supports_block_expectations
 
   match do |block|
-    ctx = Ductwork::Context.new(pipeline_id)
+    ctx = Ductwork::Context.new(run_id)
     before_values = expected.map { |k, _| ctx.get(k.to_s) }
 
     block.call
@@ -76,7 +76,21 @@ RSpec::Matchers.define(:have_set_context) do |expected|
     before_values.all?(&:nil?) && after_context == expected
   end
 
-  chain :for_pipeline, :pipeline_id
+  chain :for_pipeline do |pipeline|
+    @run_id = pipeline.current_run.id
+  end
+
+  chain :for_run do |given_run_id|
+    @run_id = given_run_id
+  end
+
+  def run_id
+    if @run_id.blank?
+      raise ArgumentError, "Must chain with .for_pipeline or .for_run"
+    end
+
+    @run_id
+  end
 
   failure_message do
     "Context does not match expected result"
@@ -88,25 +102,28 @@ module Ductwork
     module RSpec
       def pipeline_for(klass, **attrs)
         definition = klass.pipeline_definition.to_json
-        definition_sha1 = OpenSSL::Digest::SHA256.hexdigest(definition)
+        definition_sha1 = Digest::SHA1.hexdigest(definition)
+        pipeline_klass = klass.name.to_s
+        now = Time.current
         status = attrs[:status] || "in_progress"
-        triggered_at = attrs[:triggered_at] || Time.current
-        started_at = attrs[:started_at] || Time.current
-        last_advanced_at = attrs[:last_advanced_at] || Time.current
+        triggered_at = attrs[:triggered_at] || now
+        started_at = attrs[:started_at] || now
 
-        Ductwork::Pipeline.create!(
-          klass:,
+        pipeline = Ductwork::Pipeline.create!(klass:, status:)
+        pipeline.runs.create!(
+          pipeline_klass:,
           definition:,
           definition_sha1:,
           status:,
           triggered_at:,
-          started_at:,
-          last_advanced_at:
+          started_at:
         )
+
+        pipeline
       end
 
       def set_pipeline_context(pipeline, **key_values)
-        ctx = Ductwork::Context.new(pipeline.id)
+        ctx = Ductwork::Context.new(pipeline.current_run.id)
         key_values.each do |key, value|
           ctx.set(key.to_s, value)
         end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.26.0"
+  VERSION = "1.0.0"
 end

data/lib/ductwork.rb

@@ -65,6 +65,7 @@ module Ductwork
     def validate!
       step_directory = Rails.root.join("app/steps")
       pipeline_directory = Rails.root.join("app/pipelines")
+      workflow_directory = Rails.root.join("app/workflows")
       loader = if defined?(Rails.autoloaders)
                  Rails.autoloaders.main
                else
@@ -79,6 +80,10 @@ module Ductwork
         loader.eager_load_dir(pipeline_directory)
       end
 
+      if workflow_directory.exist?
+        loader.eager_load_dir(workflow_directory)
+      end
+
       true
     end
 

data/lib/generators/ductwork/install/install_generator.rb

@@ -17,20 +17,30 @@ module Ductwork
 
       migration_template "db/create_ductwork_pipelines.rb",
                          "db/migrate/create_ductwork_pipelines.rb"
+      migration_template "db/create_ductwork_runs.rb",
+                         "db/migrate/create_ductwork_runs.rb"
+      migration_template "db/create_ductwork_branches.rb",
+                         "db/migrate/create_ductwork_branches.rb"
+      migration_template "db/create_ductwork_branch_links.rb",
+                         "db/migrate/create_ductwork_branch_links.rb"
       migration_template "db/create_ductwork_steps.rb",
                          "db/migrate/create_ductwork_steps.rb"
       migration_template "db/create_ductwork_jobs.rb",
                          "db/migrate/create_ductwork_jobs.rb"
+      migration_template "db/create_ductwork_processes.rb",
+                         "db/migrate/create_ductwork_processes.rb"
       migration_template "db/create_ductwork_executions.rb",
                          "db/migrate/create_ductwork_executions.rb"
       migration_template "db/create_ductwork_availabilities.rb",
                          "db/migrate/create_ductwork_availabilities.rb"
-      migration_template "db/create_ductwork_runs.rb",
-                         "db/migrate/create_ductwork_runs.rb"
+      migration_template "db/create_ductwork_attempts.rb",
+                         "db/migrate/create_ductwork_attempts.rb"
       migration_template "db/create_ductwork_results.rb",
                          "db/migrate/create_ductwork_results.rb"
-      migration_template "db/create_ductwork_processes.rb",
-                         "db/migrate/create_ductwork_processes.rb"
+      migration_template "db/create_ductwork_transitions.rb",
+                         "db/migrate/create_ductwork_transitions.rb"
+      migration_template "db/create_ductwork_advancements.rb",
+                         "db/migrate/create_ductwork_advancements.rb"
       migration_template "db/create_ductwork_tuples.rb",
                          "db/migrate/create_ductwork_tuples.rb"
 

data/lib/generators/ductwork/install/templates/config/ductwork.yml

@@ -9,12 +9,16 @@ default: &default
     polling_timeout: 1
     shutdown_timeout: 20
   pipeline_advancer:
+    count: 3
+    max_crash: 10
+    max_retry: 3
     polling_timeout: 1
     shutdown_timeout: 20
     steps:
       max_depth: -1
   supervisor:
     polling_timeout: 1
+    reaper_timeout: 300
     shutdown_timeout: 30
 
 development:

data/lib/generators/ductwork/install/templates/db/create_ductwork_advancements.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class CreateDuctworkAdvancements < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_advancements do |table|
+      belongs_to(
+        table,
+        :process,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_processes }
+      )
+      belongs_to(
+        table,
+        :transition,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_transitions }
+      )
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.integer :crash_count
+      table.string :error_klass
+      table.string :error_message
+      table.text :error_backtrace
+      table.timestamps null: false
+    end
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_attempts.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class CreateDuctworkAttempts < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_attempts do |table|
+      belongs_to(
+        table,
+        :execution,
+        index: false,
+        null: false,
+        foreign_key: { to_table: :ductwork_executions }
+      )
+      table.timestamp :started_at, null: false
+      table.timestamp :completed_at
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_attempts, :execution_id, unique: true
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb

@@ -10,18 +10,31 @@ class CreateDuctworkAvailabilities < Ductwork::Migration
         null: false,
         foreign_key: { to_table: :ductwork_executions }
       )
+      belongs_to(
+        table,
+        :process,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_processes }
+      )
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
-      table.integer :process_id
       table.string :pipeline_klass, null: false
       table.timestamps null: false
     end
 
     add_index :ductwork_availabilities, :execution_id, unique: true
     add_index :ductwork_availabilities, %i[id process_id]
-    add_index :ductwork_availabilities,
-              %i[pipeline_klass started_at],
-              name: "index_ductwork_availabilities_on_claim_latest",
-              where: "completed_at IS NULL"
+
+    if mysql?
+      add_index :ductwork_availabilities,
+                %i[pipeline_klass completed_at started_at],
+                name: "index_ductwork_availabilities_on_claim_latest"
+    else
+      add_index :ductwork_availabilities,
+                %i[pipeline_klass started_at],
+                name: "index_ductwork_availabilities_on_claim_latest",
+                where: "completed_at IS NULL"
+    end
   end
 end