ductwork 0.18.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d93b3436f460ab6b872e05f460f8b4d807e8e7d5a0d5320284306331063db1e8
-  data.tar.gz: 2ebdcc63a041e048569debeda1add50123d694491f8eaab20a0bc95ba7930cfe
+  metadata.gz: 82de3534dfed89267c6a4aaca8e1337841fa81f66c6c42b494d9fddfe6ff4a0e
+  data.tar.gz: 334f632bcf121d1bfcddb94046d797ca1e65b2cd74fe19100b1d6d3560b1f05a
 SHA512:
-  metadata.gz: 146333524dc958193695149dd8be51bb68cc9f2503b4b8122a97e6c3e4de5abd784a7c3c3691a89219bf4d295c70dacc2c84c4b7601154027fcd9eeae8208ffb
-  data.tar.gz: 6c8f0f8abe30c8138896064ffd83e5bdc2784ed02e9cd52577cd38052eee6695aff2d724defbae9006a10c12a92996a24bddc02cabcc07c5ad33ea21d5e82abf
+  metadata.gz: 76ff2b157c3d85f4443178bf9a138c58dcead96c39c10c91cde45af88b7c07447c6cce73558f8f2a961d5fd2b3574fbbd8371442d1bf39bac4fbe30f80f415fa
+  data.tar.gz: c8c060ebb944a64a3eb42e086f857f042b944a5bdaf1c07ff2d518409530febd4b68e21634452c861d885dcf6ea4d4cbb27a10b0231d99149a591b7a1c85546c

data/CHANGELOG.md

@@ -1,5 +1,32 @@
 # Ductwork Changelog
 
+## [0.20.0]
+
+- feat: add "timed_out" enum value to execution result model
+- chore: change log level to warn for thread restart messages
+- chore: add thread name to log messages
+- feat: add optional index argument to pipeline advancer to be used in thread name
+- fix: use correct "role" in log messages
+- feat: support a "thread-only" mode instead of forking processes
+- chore: use more descriptive thread names for job workers
+- chore: refactor to `Ductwork::Processes::JobWorker#join`
+- chore: refactor to `Ductwork::Processes::JobWorker#kill`
+- chore: refactor to `Ductwork::Processes::JobWorker#name`
+- chore: use `RunningContext` instance instead of a boolean in the Supervisor - this running context will likely be shared to all child runners when running in "all thread" mode
+- chore!: officially drop support for ruby 3.1 - BREAKING CHANGE: while this is a breaking change, this shouldn't affect anyone since ruby 3.1 has been EOL for a bit; this is more of a formality
+- chore: update `bundler`
+- chore: change project ruby version to 4.0.1
+- feat: add "forking" configuration with default - this will be used to change the concurreny model on boot, specifically deciding if pipeline advancer and job workers will be forked or created as threads
+
+## [0.19.0]
+
+- chore: bump rails-related dependencies to v8.1.2
+- chore: remove ruby v3.2.9 from CI testing matrix - support is ending in March '26 but it's being removed now to better support edge rails
+- feat: loosen rails version constraint to allow rails edge
+- feat: respect "role" configuration when booting - ie. run main process as supervisor, pipeline advancer, or job worker
+- feat: allow job worker runner to take a collection of pipelines and create workers for all configured pipelines - this will only happen when the "role" configuration is "worker" otherwise a process will be spun up for each pipeline
+- feat: add "role" configuration - to be used to set the role of the entire ductwork running instance
+
 ## [0.18.0]
 
 - fix: show countdown for pipelines/steps scheduled in the future

data/README.md

@@ -46,7 +46,7 @@ default: &default
   pipelines: "*"
 ```
 
-See the [Configuration Guide](https://docs.getductwork.io/advanced/configuration.html) for all available options including thread counts, timeouts, and database settings.
+See the [Configuration Guide](https://docs.getductwork.io/getting-started/configuration.html) for all available options including thread counts, timeouts, and database settings.
 
 ## Usage
 

data/lib/ductwork/cli.rb

@@ -17,7 +17,7 @@ module Ductwork
       option_parser.parse!(args)
       auto_configure
       puts banner
-      supervisor_runner.start!
+      launch_processes
     end
 
     private
@@ -45,6 +45,7 @@ module Ductwork
     end
 
     def auto_configure
+      options[:role] = ENV.fetch("DUCTWORK_ROLE", nil)
       Ductwork.configuration = Configuration.new(**options)
       Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
                           Rails.logger
@@ -63,17 +64,17 @@ module Ductwork
   ██║  ██║██║   ██║██║        ██║   ██║███╗██║██║   ██║██╔══██╗██╔═██╗
   ██████╔╝╚██████╔╝╚██████╗   ██║   ╚███╔███╔╝╚██████╔╝██║  ██║██║  ██╗
   ╚═════╝  ╚═════╝  ╚═════╝   ╚═╝    ╚══╝╚══╝  ╚═════╝ ╚═╝  ╚═╝╚═╝  ╚═╝
-   ▒▒▓  ▒ ░▒▓▒ ▒ ▒ ░ ░▒ ▒  ░  ▒ ░░   ░ ▓░▒ ▒  ░ ▒░▒░▒░ ░ ▒▓ ░▒▓░▒ ▒▒ ▓▒
-    ░ ▒  ▒ ░░▒░ ░ ░   ░  ▒       ░      ▒ ░ ░    ░ ▒ ▒░   ░▒ ░ ▒░░ ░▒ ▒░
-     ░ ░  ░  ░░░ ░ ░ ░          ░        ░   ░  ░ ░ ░ ▒    ░░   ░ ░ ░░ ░
-        ░       ░     ░ ░                    ░        ░ ░     ░     ░  ░
-      ░               ░
+  ▒▒▓  ▒ ░▒▓▒ ▒ ▒ ░ ░▒ ▒  ░  ▒ ░░   ░ ▓░▒ ▒  ░ ▒░▒░▒░ ░ ▒▓ ░▒▓░▒ ▒▒ ▓▒
+   ░ ▒  ▒ ░░▒░ ░ ░   ░  ▒       ░      ▒ ░ ░    ░ ▒ ▒░   ░▒ ░ ▒░░ ░▒ ▒░
+    ░ ░  ░  ░░░ ░ ░ ░          ░        ░   ░  ░ ░ ░ ▒    ░░   ░ ░ ░░ ░
+       ░       ░     ░ ░                    ░        ░ ░     ░     ░  ░
+     ░               ░
   \e[0m
       BANNER
     end
 
-    def supervisor_runner
-      Ductwork::Processes::SupervisorRunner
+    def launch_processes
+      Ductwork::Processes::Launcher.start_processes!
     end
   end
 end

data/lib/ductwork/configuration.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 module Ductwork
-  class Configuration
+  class Configuration # rubocop:todo Metrics/ClassLength
     DEFAULT_ENV = :default
     DEFAULT_FILE_PATH = "config/ductwork.yml"
+    DEFAULT_FORKING = "default" # fork pipeline advancer and job workers
     DEFAULT_JOB_WORKER_COUNT = 5 # threads
     DEFAULT_JOB_WORKER_MAX_RETRY = 3 # attempts
     DEFAULT_JOB_WORKER_POLLING_TIMEOUT = 1 # second
@@ -12,11 +13,15 @@ module Ductwork
     DEFAULT_LOGGER_SOURCE = "default" # `Logger` instance writing to STDOUT
     DEFAULT_PIPELINE_POLLING_TIMEOUT = 1 # second
     DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT = 20 # seconds
+    DEFAULT_ROLE = "all" # supervisor, pipeline advancer, and job workers
     DEFAULT_STEPS_MAX_DEPTH = -1 # unlimited count
     DEFAULT_SUPERVISOR_POLLING_TIMEOUT = 1 # second
     DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT = 30 # seconds
     DEFAULT_LOGGER = ::Logger.new($stdout)
     PIPELINES_WILDCARD = "*"
+    VALID_ROLES = %w[all advancer worker].freeze
+
+    class InvalidRoleError < StandardError; end
 
     attr_writer :job_worker_count, :job_worker_polling_timeout,
                 :job_worker_shutdown_timeout, :job_worker_max_retry,
@@ -25,13 +30,27 @@ module Ductwork
                 :steps_max_depth,
                 :supervisor_polling_timeout, :supervisor_shutdown_timeout
 
-    def initialize(path: DEFAULT_FILE_PATH)
+    def initialize(path: DEFAULT_FILE_PATH, role: nil)
       full_path = Pathname.new(path)
       data = ActiveSupport::ConfigurationFile.parse(full_path).deep_symbolize_keys
       env = defined?(Rails) ? Rails.env.to_sym : DEFAULT_ENV
-      @config = data[env]
+      @config = if role.present?
+                  data[env].merge(role:)
+                else
+                  data[env]
+                end
     rescue Errno::ENOENT
-      @config = {}
+      @config = { role: }.compact
+    end
+
+    def role
+      r = config[:role] || DEFAULT_ROLE
+      validate_role!(r)
+      r
+    end
+
+    def forking
+      config[:forking] || DEFAULT_FORKING
     end
 
     def pipelines
@@ -185,5 +204,11 @@ module Ductwork
       config.dig(:supervisor, :shutdown_timeout) ||
         DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT
     end
+
+    def validate_role!(role)
+      if VALID_ROLES.exclude?(role)
+        raise InvalidRoleError, "Must use a valid role"
+      end
+    end
   end
 end

data/lib/ductwork/models/result.rb

@@ -8,6 +8,7 @@ module Ductwork
 
     enum :result_type,
          success: "success",
-         failure: "failure"
+         failure: "failure",
+         timed_out: "timed_out"
   end
 end

data/lib/ductwork/models/step.rb

@@ -26,10 +26,10 @@ module Ductwork
          expand: "expand",
          collapse: "collapse"
 
-    def self.build_for_execution(pipeline_id, *args, **kwargs)
+    def self.build_for_execution(pipeline_id, *, **)
       instance = allocate
       instance.instance_variable_set(:@pipeline_id, pipeline_id)
-      instance.send(:initialize, *args, **kwargs)
+      instance.send(:initialize, *, **)
       instance
     end
 

data/lib/ductwork/processes/job_worker.rb

@@ -3,7 +3,7 @@
 module Ductwork
   module Processes
     class JobWorker
-      attr_reader :thread, :last_heartbeat_at, :job
+      attr_reader :thread, :last_heartbeat_at, :job, :pipeline
 
       def initialize(pipeline, id)
         @pipeline = pipeline
@@ -15,7 +15,7 @@ module Ductwork
 
       def start
         @thread = Thread.new { work_loop }
-        @thread.name = "ductwork.job_worker.#{id}"
+        @thread.name = name
       end
 
       alias restart start
@@ -28,9 +28,22 @@ module Ductwork
         running_context.shutdown!
       end
 
+      def kill
+        stop
+        thread&.kill
+      end
+
+      def join(limit)
+        thread&.join(limit)
+      end
+
+      def name
+        "ductwork.job_worker.#{pipeline}.#{id}"
+      end
+
       private
 
-      attr_reader :pipeline, :id, :running_context
+      attr_reader :id, :running_context
 
       def work_loop
         run_hooks_for(:start)

data/lib/ductwork/processes/job_worker_runner.rb

@@ -3,8 +3,8 @@
 module Ductwork
   module Processes
     class JobWorkerRunner
-      def initialize(pipeline)
-        @pipeline = pipeline
+      def initialize(*pipelines)
+        @pipelines = pipelines
         @running_context = Ductwork::RunningContext.new
         @job_workers = []
 
@@ -30,7 +30,7 @@ module Ductwork
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker_runner,
-          pipeline: pipeline
+          pipelines: pipelines
         )
 
         while running?
@@ -45,7 +45,7 @@ module Ductwork
 
       private
 
-      attr_reader :pipeline, :running_context, :job_workers
+      attr_reader :pipelines, :running_context, :job_workers
 
       def create_process_record!
         Ductwork.wrap_with_app_executor do
@@ -58,16 +58,19 @@ module Ductwork
       end
 
       def start_job_workers
-        Ductwork.configuration.job_worker_count(pipeline).times do |i|
-          job_worker = Ductwork::Processes::JobWorker.new(pipeline, i)
-          job_workers.push(job_worker)
-          job_worker.start
-
-          Ductwork.logger.debug(
-            msg: "Created new job worker",
-            role: :job_worker_runner,
-            pipeline: pipeline
-          )
+        pipelines.each do |pipeline|
+          Ductwork.configuration.job_worker_count(pipeline).times do |i|
+            job_worker = Ductwork::Processes::JobWorker.new(pipeline, i)
+            job_workers.push(job_worker)
+            job_worker.start
+
+            Ductwork.logger.debug(
+              msg: "Created new job worker",
+              role: :job_worker_runner,
+              pipeline: pipeline,
+              thread: job_worker.name
+            )
+          end
         end
       end
 
@@ -79,23 +82,24 @@ module Ductwork
         Ductwork.logger.debug(
           msg: "Checking thread health",
           role: :job_worker_runner,
-          pipeline: pipeline
+          pipelines: pipelines
         )
         job_workers.each do |job_worker|
           if !job_worker.alive?
             job_worker.restart
 
-            Ductwork.logger.info(
-              msg: "Restarted thread",
+            Ductwork.logger.warn(
+              msg: "Restarted job worker",
               role: :job_worker_runner,
-              pipeline: pipeline
+              pipeline: job_worker.pipeline,
+              thread: job_worker.name
             )
           end
         end
         Ductwork.logger.debug(
           msg: "Checked thread health",
           role: :job_worker_runner,
-          pipeline: pipeline
+          pipelines: pipelines
         )
       end
 
@@ -130,7 +134,7 @@ module Ductwork
 
             # TODO: Maybe make this configurable. If there's a ton of workers
             # it may not even get to the "later" ones depending on the timeout
-            job_worker.thread.join(1)
+            job_worker.join(1)
           end
         end
       end
@@ -138,11 +142,11 @@ module Ductwork
       def kill_remaining_job_workers
         job_workers.each do |job_worker|
           if job_worker.alive?
-            job_worker.thread.kill
+            job_worker.kill
             Ductwork.logger.debug(
               msg: "Killed thread",
               role: :job_worker_runner,
-              thread: job_worker.thread.name
+              thread: job_worker.name
             )
           end
         end

data/lib/ductwork/processes/launcher.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class Launcher
+      def self.start_processes!
+        new.start_processes!
+      end
+
+      def initialize
+        @pipelines = Ductwork.configuration.pipelines
+        @runner_klass = if Ductwork.configuration.forking == "default"
+                          case Ductwork.configuration.role
+                          when "all"
+                            process_supervisor_runner
+                          when "advancer"
+                            pipeline_advancer_runner
+                          when "worker"
+                            job_worker_runner
+                          end
+                        else
+                          thread_supervisor_runner
+                        end
+      end
+
+      def start_processes!
+        runner_klass
+          .new(*pipelines)
+          .run
+      end
+
+      private
+
+      attr_reader :pipelines, :runner_klass
+
+      def thread_supervisor_runner
+        Ductwork::Processes::ThreadSupervisorRunner
+      end
+
+      def process_supervisor_runner
+        Ductwork::Processes::ProcessSupervisorRunner
+      end
+
+      def pipeline_advancer_runner
+        Ductwork::Processes::PipelineAdvancerRunner
+      end
+
+      def job_worker_runner
+        Ductwork::Processes::JobWorkerRunner
+      end
+    end
+  end
+end

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -3,14 +3,57 @@
 module Ductwork
   module Processes
     class PipelineAdvancer
-      def initialize(running_context, klass)
-        @running_context = running_context
+      attr_reader :thread, :last_heartbeat_at, :pipeline
+
+      def initialize(klass, index = nil)
         @klass = klass
+        @index = index || 0
+        @running_context = Ductwork::RunningContext.new
+        @last_heartbeat_at = Time.current
+        @thread = nil
+      end
+
+      def start
+        @thread = Thread.new { work_loop }
+        @thread.name = name
+      end
+
+      alias restart start
+
+      def alive?
+        thread&.alive? || false
+      end
+
+      def stop
+        running_context.shutdown!
+      end
+
+      def kill
+        stop
+        thread&.kill
+      end
+
+      def join(limit)
+        thread&.join(limit)
       end
 
-      def run # rubocop:todo Metrics/AbcSize, Metrics/MethodLength
+      def name
+        "ductwork.pipeline_advancer.#{klass}.#{index}"
+      end
+
+      private
+
+      attr_reader :klass, :index, :running_context
+
+      def work_loop # rubocop:todo Metrics
         run_hooks_for(:start)
 
+        Ductwork.logger.debug(
+          msg: "Entering main work loop",
+          role: :pipeline_advancer,
+          pipeline: klass
+        )
+
         while running_context.running?
           id = Ductwork.wrap_with_app_executor do
             Ductwork::Pipeline
@@ -42,8 +85,8 @@ module Ductwork
                 role: :pipeline_advancer
               )
 
-              pipeline = Ductwork.wrap_with_app_executor do
-                pipeline = Ductwork::Pipeline.find(id)
+              @pipeline = Ductwork.wrap_with_app_executor do
+                @pipeline = Ductwork::Pipeline.find(id)
                 pipeline.advance!
 
                 Ductwork.logger.debug(
@@ -82,16 +125,20 @@ module Ductwork
             )
           end
 
+          @last_heartbeat_at = Time.current
+
           sleep(polling_timeout)
         end
 
+        Ductwork.logger.debug(
+          msg: "Shutting down",
+          role: :pipeline_advancer,
+          pipeline: klass
+        )
+
         run_hooks_for(:stop)
       end
 
-      private
-
-      attr_reader :running_context, :klass
-
       def run_hooks_for(event)
         Ductwork.hooks[:advancer].fetch(event, []).each do |block|
           Ductwork.wrap_with_app_executor do

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -6,7 +6,7 @@ module Ductwork
       def initialize(*klasses)
         @klasses = klasses
         @running_context = Ductwork::RunningContext.new
-        @threads = create_threads
+        @advancers = []
 
         Signal.trap(:INT) { running_context.shutdown! }
         Signal.trap(:TERM) { running_context.shutdown! }
@@ -25,54 +25,65 @@ module Ductwork
 
       def run
         create_process!
+        start_pipeline_advancers
+
         Ductwork.logger.debug(
           msg: "Entering main work loop",
-          role: :pipeline_advancer_runner
+          role: :pipeline_advancer_runner,
+          pipelines: klasses
         )
 
         while running_context.running?
           # TODO: Increase or make configurable
           sleep(5)
-          attempt_synchronize_threads
+          check_thread_health
           report_heartbeat!
         end
 
-        shutdown
+        shutdown!
       end
 
       private
 
-      attr_reader :klasses, :running_context, :threads
+      attr_reader :klasses, :running_context, :advancers
 
-      def create_threads
-        klasses.map do |klass|
-          thread = Thread.new do
-            Ductwork::Processes::PipelineAdvancer
-              .new(running_context, klass)
-              .run
-          end
-          thread.name = "ductwork.pipeline_advancer.#{klass}"
+      def start_pipeline_advancers
+        klasses.each do |klass|
+          advancer = Ductwork::Processes::PipelineAdvancer.new(klass)
+          advancers.push(advancer)
+          advancer.start
 
           Ductwork.logger.debug(
-            msg: "Created new thread",
+            msg: "Created new pipeline advancer",
             role: :pipeline_advancer_runner,
-            thread: thread.name,
-            pipeline: klass
+            pipeline: klass,
+            thread: advancer.name
           )
-
-          thread
         end
       end
 
-      def attempt_synchronize_threads
+      def check_thread_health
         Ductwork.logger.debug(
-          msg: "Attempting to synchronize threads",
-          role: :pipeline_advancer_runner
+          msg: "Checking threads health",
+          role: :pipeline_advancer_runner,
+          pipelines: klasses
         )
-        threads.each { |thread| thread.join(0.1) }
+        advancers.each do |advancer|
+          if !advancer.alive?
+            advancer.restart
+
+            Ductwork.logger.warn(
+              msg: "Restarted pipeline advancer",
+              role: :pipeline_advancer_runner,
+              pipeline: advancer.pipeline,
+              thread: advancer.name
+            )
+          end
+        end
         Ductwork.logger.debug(
-          msg: "Synchronizing threads timed out",
-          role: :pipeline_advancer_runner
+          msg: "Checked thread health",
+          role: :pipeline_advancer_runner,
+          pipelines: klasses
         )
       end
 
@@ -94,9 +105,10 @@ module Ductwork
         Ductwork.logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
       end
 
-      def shutdown
+      def shutdown!
         log_shutting_down
         stop_running_context
+        advancers.each(&:stop)
         await_threads_graceful_shutdown
         kill_remaining_threads
         delete_process!
@@ -118,25 +130,25 @@ module Ductwork
           msg: "Attempting graceful shutdown",
           role: :pipeline_advancer_runner
         )
-        while Time.current < deadline && threads.any?(&:alive?)
-          threads.each do |thread|
+        while Time.current < deadline && advancers.any?(&:alive?)
+          advancers.each do |advancer|
             break if Time.current > deadline
 
             # TODO: Maybe make this configurable. If there's a ton of workers
             # it may not even get to the "later" ones depending on the timeout
-            thread.join(1)
+            advancer.join(1)
           end
         end
       end
 
       def kill_remaining_threads
-        threads.each do |thread|
-          if thread.alive?
-            thread.kill
+        advancers.each do |advancer|
+          if advancer.alive?
+            advancer.kill
             Ductwork.logger.debug(
               msg: "Killed thread",
               role: :pipeline_advancer_runner,
-              thread: thread.name
+              thread: advancer.name
             )
           end
         end

data/lib/ductwork/processes/{supervisor.rb → process_supervisor.rb}

@@ -2,17 +2,17 @@
 
 module Ductwork
   module Processes
-    class Supervisor
+    class ProcessSupervisor
       attr_reader :workers
 
       def initialize
-        @running = true
+        @running_context = Ductwork::RunningContext.new
         @workers = []
 
         run_hooks_for(:start)
 
-        Signal.trap(:INT) { @running = false }
-        Signal.trap(:TERM) { @running = false }
+        Signal.trap(:INT) { @running_context.shutdown! }
+        Signal.trap(:TERM) { @running_context.shutdown! }
         Signal.trap(:TTIN) { puts "No threads to dump" }
       end
 
@@ -29,9 +29,13 @@ module Ductwork
       end
 
       def run
-        Ductwork.logger.debug(msg: "Entering main work loop", role: :supervisor, pid: ::Process.pid)
+        Ductwork.logger.debug(
+          msg: "Entering main work loop",
+          role: :process_supervisor,
+          pid: ::Process.pid
+        )
 
-        while running
+        while running_context.running?
           sleep(Ductwork.configuration.supervisor_polling_timeout)
           check_workers
         end
@@ -40,9 +44,12 @@ module Ductwork
       end
 
       def shutdown
-        @running = false
+        running_context.shutdown!
+        Ductwork.logger.debug(
+          msg: "Beginning shutdown",
+          role: :process_supervisor
+        )
 
-        Ductwork.logger.debug(msg: "Beginning shutdown", role: :supervisor)
         terminate_gracefully
         wait_for_workers_to_exit
         terminate_immediately
@@ -51,10 +58,13 @@ module Ductwork
 
       private
 
-      attr_reader :running
+      attr_reader :running_context
 
       def check_workers
-        Ductwork.logger.debug(msg: "Checking workers are alive", role: :supervisor)
+        Ductwork.logger.debug(
+          msg: "Checking workers are alive",
+          role: :process_supervisor
+        )
 
         workers.each do |worker|
           if process_dead?(worker[:pid])
@@ -65,21 +75,24 @@ module Ductwork
             worker[:pid] = new_pid
             Ductwork.logger.debug(
               msg: "Restarted process (#{old_pid}) as (#{new_pid})",
-              role: :supervisor,
+              role: :process_supervisor,
               old_pid: old_pid,
               new_pid: new_pid
             )
           end
         end
 
-        Ductwork.logger.debug(msg: "All workers are alive or revived", role: :supervisor)
+        Ductwork.logger.debug(
+          msg: "All workers are alive or revived",
+          role: :process_supervisor
+        )
       end
 
       def terminate_gracefully
         workers.each do |worker|
           Ductwork.logger.debug(
             msg: "Sending TERM signal to process (#{worker[:pid]})",
-            role: :supervisor,
+            role: :process_supervisor,
             pid: worker[:pid],
             signal: :TERM
           )
@@ -97,7 +110,7 @@ module Ductwork
               workers[index] = nil
               Ductwork.logger.debug(
                 msg: "Child process (#{worker[:pid]}) stopped successfully",
-                role: :supervisor,
+                role: :process_supervisor,
                 pid: worker[:pid]
               )
             end
@@ -110,7 +123,7 @@ module Ductwork
         workers.each_with_index do |worker, index|
           Ductwork.logger.debug(
             msg: "Sending KILL signal to process (#{worker[:pid]})",
-            role: :supervisor,
+            role: :process_supervisor,
             pid: worker[:pid],
             signal: :KILL
           )
@@ -119,7 +132,7 @@ module Ductwork
           workers[index] = nil
           Ductwork.logger.debug(
             msg: "Child process (#{worker[:pid]}) killed after timeout",
-            role: :supervisor,
+            role: :process_supervisor,
             pid: worker[:pid]
           )
         rescue Errno::ESRCH, Errno::ECHILD

data/lib/ductwork/processes/process_supervisor_runner.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class ProcessSupervisorRunner
+      def initialize(*pipelines)
+        @pipelines = pipelines
+        @supervisor = Ductwork::Processes::ProcessSupervisor.new
+      end
+
+      def run
+        supervisor.add_worker(metadata: { pipelines: }) do
+          log_starting_pipline_advancer
+          pipline_advancer_runner.new(*pipelines).run
+        end
+
+        pipelines.each do |pipeline|
+          supervisor.add_worker(metadata: { pipeline: }) do
+            log_starting_job_worker(pipeline)
+            job_worker_runner.new(*pipeline).run
+          end
+        end
+
+        supervisor.run
+      end
+
+      private
+
+      attr_reader :pipelines, :supervisor
+
+      def log_starting_pipline_advancer
+        Ductwork.logger.debug(
+          msg: "Starting Pipeline Advancer process",
+          role: :process_supervisor_runner
+        )
+      end
+
+      def pipline_advancer_runner
+        Ductwork::Processes::PipelineAdvancerRunner
+      end
+
+      def log_starting_job_worker(pipeline)
+        Ductwork.logger.debug(
+          msg: "Starting Job Worker Runner process",
+          role: :process_supervisor_runner,
+          pipeline: pipeline
+        )
+      end
+
+      def job_worker_runner
+        Ductwork::Processes::JobWorkerRunner
+      end
+    end
+  end
+end

data/lib/ductwork/processes/thread_supervisor.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class ThreadSupervisor
+      attr_reader :workers
+
+      def initialize
+        @running_context = Ductwork::RunningContext.new
+        @workers = []
+
+        run_hooks_for(:start)
+
+        Signal.trap(:INT) { @running_context.shutdown! }
+        Signal.trap(:TERM) { @running_context.shutdown! }
+        Signal.trap(:TTIN) do
+          Thread.list.each do |thread|
+            puts thread.name
+            if thread.backtrace
+              puts thread.backtrace.join("\n")
+            else
+              puts "No backtrace to dump"
+            end
+            puts
+          end
+        end
+      end
+
+      # TODO: maybe change the whole supervisor interface because this is clunky
+      def add_worker(metadata: {}, &block)
+        worker = block.call(metadata)
+        workers << worker
+        worker.start
+
+        Ductwork.logger.debug(
+          msg: "Started supervised thread",
+          role: :thread_supervisor,
+          thread: worker.name
+        )
+      end
+
+      def run
+        Ductwork.logger.debug(
+          msg: "Entering main work loop",
+          role: :thread_supervisor,
+          pid: ::Process.pid
+        )
+
+        while running_context.running?
+          sleep(Ductwork.configuration.supervisor_polling_timeout)
+          check_worker_health
+        end
+
+        shutdown
+      end
+
+      private
+
+      attr_reader :running_context
+
+      def check_worker_health
+        Ductwork.logger.debug(
+          msg: "Checking workers are alive",
+          role: :thread_supervisor
+        )
+
+        workers.each do |worker|
+          if !worker.alive?
+            worker.restart
+
+            Ductwork.logger.warn(
+              msg: "Restarted supervised thread",
+              role: :thread_supervisor,
+              thread: worker.name
+            )
+          end
+        end
+
+        Ductwork.logger.debug(
+          msg: "Checked workers are alive",
+          role: :thread_supervisor
+        )
+      end
+
+      def shutdown
+        running_context.shutdown!
+        log_beginning_shutdown
+        workers.each(&:stop)
+        await_threads_graceful_shutdown
+        kill_remaining_threads
+        run_hooks_for(:stop)
+      end
+
+      def log_beginning_shutdown
+        Ductwork.logger.debug(
+          msg: "Beginning shutdown",
+          role: :thread_supervisor
+        )
+      end
+
+      def await_threads_graceful_shutdown
+        timeout = Ductwork.configuration.supervisor_shutdown_timeout
+        deadline = Time.current + timeout
+
+        Ductwork.logger.debug(
+          msg: "Attempting graceful shutdown",
+          role: :thread_supervisor
+        )
+        while Time.current < deadline && workers.any?(&:alive?)
+          workers.each do |worker|
+            break if Time.current > deadline
+
+            # TODO: Maybe make this configurable. If there's a ton of workers
+            # it may not even get to the "later" ones depending on the timeout
+            worker.join(1)
+          end
+        end
+      end
+
+      def kill_remaining_threads
+        workers.each do |worker|
+          if worker.alive?
+            worker.kill
+            Ductwork.logger.debug(
+              msg: "Killed supervised thread",
+              role: :thread_supervisor,
+              thread: worker.name
+            )
+          end
+        end
+      end
+
+      def run_hooks_for(event)
+        Ductwork.hooks[:supervisor].fetch(event, []).each do |block|
+          Ductwork.wrap_with_app_executor do
+            block.call(self)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ductwork/processes/thread_supervisor_runner.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class ThreadSupervisorRunner
+      def initialize(*pipelines)
+        @pipelines = pipelines
+        @supervisor = Ductwork::Processes::ThreadSupervisor.new
+      end
+
+      def run
+        if Ductwork.configuration.role.in?(%w[all advancer])
+          pipelines.each do |pipeline|
+            log_created_pipeline_advancer(pipeline)
+            supervisor.add_worker(metadata: { pipeline: }) do
+              pipeline_advancer.new(pipeline)
+            end
+          end
+        end
+
+        if Ductwork.configuration.role.in?(%w[all worker])
+          pipelines.each do |pipeline|
+            Ductwork.configuration.job_worker_count(pipeline).times do |index|
+              log_created_job_worker(pipeline, index)
+              supervisor.add_worker(metadata: { pipeline: }) do
+                job_worker.new(pipeline, index)
+              end
+            end
+          end
+        end
+
+        supervisor.run
+      end
+
+      private
+
+      attr_reader :pipelines, :supervisor
+
+      def log_created_pipeline_advancer(pipeline)
+        Ductwork.logger.debug(
+          msg: "Created new pipeline advancer",
+          role: :thread_supervisor_runner,
+          pipeline: pipeline
+        )
+      end
+
+      def pipeline_advancer
+        Ductwork::Processes::PipelineAdvancer
+      end
+
+      def log_created_job_worker(pipeline, index)
+        Ductwork.logger.debug(
+          msg: "Created new job worker",
+          role: :thread_supervisor_runner,
+          pipeline: pipeline,
+          index: index
+        )
+      end
+
+      def job_worker
+        Ductwork::Processes::JobWorker
+      end
+    end
+  end
+end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.18.0"
+  VERSION = "0.20.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.18.0
+  version: 0.20.0
 platform: ruby
 authors:
 - Tyler Ewing
@@ -18,7 +18,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,7 +28,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -48,7 +48,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -68,7 +68,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -78,7 +78,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -88,7 +88,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -153,10 +153,13 @@ files:
 - lib/ductwork/models/tuple.rb
 - lib/ductwork/processes/job_worker.rb
 - lib/ductwork/processes/job_worker_runner.rb
+- lib/ductwork/processes/launcher.rb
 - lib/ductwork/processes/pipeline_advancer.rb
 - lib/ductwork/processes/pipeline_advancer_runner.rb
-- lib/ductwork/processes/supervisor.rb
-- lib/ductwork/processes/supervisor_runner.rb
+- lib/ductwork/processes/process_supervisor.rb
+- lib/ductwork/processes/process_supervisor_runner.rb
+- lib/ductwork/processes/thread_supervisor.rb
+- lib/ductwork/processes/thread_supervisor_runner.rb
 - lib/ductwork/running_context.rb
 - lib/ductwork/testing.rb
 - lib/ductwork/testing/helpers.rb
@@ -193,7 +196,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/ductwork/processes/supervisor_runner.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Ductwork
-  module Processes
-    class SupervisorRunner
-      def self.start!
-        supervisor = Ductwork::Processes::Supervisor.new
-        pipelines_to_advance = Ductwork.configuration.pipelines
-
-        supervisor.add_worker(metadata: { pipelines: pipelines_to_advance }) do
-          Ductwork.logger.debug(
-            msg: "Starting Pipeline Advancer process",
-            role: :supervisor_runner
-          )
-          Ductwork::Processes::PipelineAdvancerRunner
-            .new(*pipelines_to_advance).run
-        end
-
-        pipelines_to_advance.each do |pipeline|
-          supervisor.add_worker(metadata: { pipeline: }) do
-            Ductwork.logger.debug(
-              msg: "Starting Job Worker Runner process",
-              role: :supervisor_runner,
-              pipeline: pipeline
-            )
-            Ductwork::Processes::JobWorkerRunner.new(pipeline).run
-          end
-        end
-
-        supervisor.run
-      end
-    end
-  end
-end