ductwork 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c1f2564c8a446cc2daf77b882acfc6abab314e1a651bf1e7a6ab9c7432a77f6
-  data.tar.gz: ff33012c7f1dce6f9297c41185e97d8bffc4054ac26738c19459db50a70f6d19
+  metadata.gz: 18ebf23ac85cb80300ecc0c35475e0a2364fa4296a6012fdc21e95b8c5943023
+  data.tar.gz: e08b2cda1074255a7b7a79c5fbe789ac373909eef57b7836be49b3f7554a4350
 SHA512:
-  metadata.gz: cdc5618e70f3b42f758ab6de842ff46732ee34435e82b2b04777aad7e203f0457fc00ce496fa23f2b64e232ecc81548bbff669fddfbe3cdb5c404b3d0036beea
-  data.tar.gz: e3697a5c616527013580003f6e0f131c4ddfabf871b01dc2f910527febdb19e2ead1e5f3f244a001242ff0876fe331f84eca116c24111b3fdeffbc05f7205289
+  metadata.gz: 9d3b28265bfdf2f90210c43e2609d4255dda6d120ab597b74e54e9c020cbdc4be0b07235bb104873a62ae309511fc841c7d236f20dd52124150760c8dfaa94c8
+  data.tar.gz: fb6db1e8385b880b942404a25aba9ef02637d0c1a2dd53b1a6ddd78a8c70e6494c6349b8d7bbf837727cee5d2ed2c2e2afd67fe506c0be193797fc2de8867619

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Ductwork Changelog
 
+## [0.4.0]
+
+- chore: change job worker thread name format
+- feat: add and respect pipeline-level `pipeline_advancer.polling_timeout` configuration in pipeline advancer
+- feat: respect `job_worker.polling_timeout` configuration in job runner
+- feat: add pipeline-level `job_worker.polling_timeout` configuration
+- feat: check pipeline and step-level max retry configurations when retrying a job
+- feat: add pipeline and step-level `job_worker.max_retry` configurations
+- feat: add ability to set `job_worker.count` config manually
+- chore: move configuration specs under their own directory
+- feat: halt pipeline instead of erroring if max step depth is exceeded
+- chore: move specs under directory
+- feat: allow setting `pipeline_advancer.steps_max_depth` configuration manually
+- feat: raise `Ductwork::Pipeline::StepDepthError` error if return payload count exceeds the configuration
+- feat: add `pipeline_advancer.steps.max_depth` configuration
+
+## [0.3.1]
+
+- chore: bump dependencies and update necessary files
+- chore: update email address in gemspec
+- chore: move `logger` out of config to top-level `Ductwork` module
+- chore: promote `Ductwork::Pipeline#parsed_definition` to a public method
+- fix: raise when trying to collapse a most recently divided pipeline and vice versa
+
 ## [0.3.0]
 
 - fix: correctly create collapsing and combining steps and jobs for complex pipelines

data/README.md

@@ -103,7 +103,7 @@ end
 
 **Important:** Return values must be JSON-serializable.
 
-See [Defining Pipelines](https://docs.getductwork.io/getting_started/defining-pipelines.html) for detailed documentation.
+See [Defining Pipelines](https://docs.getductwork.io/getting-started/defining-pipelines.html) for detailed documentation.
 
 ### 4. Run Ductwork
 

data/app/models/ductwork/job.rb

@@ -39,7 +39,7 @@ module Ductwork
         end
 
         if rows_updated == 1
-          Ductwork.configuration.logger.debug(
+          Ductwork.logger.debug(
             msg: "Job claimed",
             role: :job_worker,
             process_id: process_id,
@@ -49,7 +49,7 @@ module Ductwork
             .joins(executions: :availability)
             .find_by(ductwork_availabilities: { id:, process_id: })
         else
-          Ductwork.configuration.logger.debug(
+          Ductwork.logger.debug(
             msg: "Did not claim job, avoided race condition",
             role: :job_worker,
             process_id: process_id,
@@ -78,7 +78,7 @@ module Ductwork
         j
       end
 
-      Ductwork.configuration.logger.info(
+      Ductwork.logger.info(
         msg: "Job enqueued",
         job_id: job.id,
         job_klass: job.klass
@@ -90,7 +90,7 @@ module Ductwork
     def execute(pipeline)
       # i don't _really_ like this, but it should be fine for now...
       execution = executions.order(:created_at).last
-      logger.debug(
+      Ductwork.logger.debug(
         msg: "Executing job",
         role: :job_worker,
         pipeline: pipeline,
@@ -111,7 +111,7 @@ module Ductwork
         execution_failed!(execution, run, e)
         result = "failure"
       ensure
-        logger.info(
+        Ductwork.logger.info(
           msg: "Job executed",
           pipeline: pipeline,
           job_id: id,
@@ -130,10 +130,6 @@ module Ductwork
 
     private
 
-    def logger
-      Ductwork.configuration.logger
-    end
-
     def execution_succeeded!(execution, run, output_payload)
       payload = JSON.dump({ payload: output_payload })
 
@@ -146,9 +142,12 @@ module Ductwork
       end
     end
 
-    def execution_failed!(execution, run, error) # rubocop:disable Metrics/AbcSize
+    def execution_failed!(execution, run, error) # rubocop:todo Metrics
       halted = false
       pipeline = step.pipeline
+      max_retry = Ductwork
+                  .configuration
+                  .job_worker_max_retry(pipeline: pipeline.klass, step: klass)
 
       Ductwork::Record.transaction do
         execution.update!(completed_at: Time.current)
@@ -160,7 +159,7 @@ module Ductwork
           error_backtrace: error.backtrace
         )
 
-        if execution.retry_count < Ductwork.configuration.job_worker_max_retry
+        if execution.retry_count < max_retry
           new_execution = executions.create!(
             retry_count: execution.retry_count + 1,
             started_at: FAILED_EXECUTION_TIMEOUT.from_now
@@ -176,7 +175,7 @@ module Ductwork
         end
       end
 
-      logger.warn(
+      Ductwork.logger.warn(
         msg: "Job errored",
         error_klass: error.class.name,
         error_message: error.message,

data/app/models/ductwork/pipeline.rb

@@ -76,7 +76,7 @@ module Ductwork
           p
         end
 
-        Ductwork.configuration.logger.info(
+        Ductwork.logger.info(
           msg: "Pipeline triggered",
           pipeline_id: pipeline.id,
           role: :application
@@ -107,6 +107,10 @@ module Ductwork
       end
     end
 
+    def parsed_definition
+      @parsed_definition ||= JSON.parse(definition).with_indifferent_access
+    end
+
     private
 
     def create_step_and_enqueue_job(klass:, step_type:, input_arg:)
@@ -116,17 +120,13 @@ module Ductwork
       Ductwork::Job.enqueue(next_step, input_arg)
     end
 
-    def parsed_definition
-      @parsed_definition ||= JSON.parse(definition).with_indifferent_access
-    end
-
     def conditionally_complete_pipeline(advancing)
       advancing.update!(status: :completed, completed_at: Time.current)
 
       if steps.where(status: %w[in_progress pending]).none?
         update!(status: :completed, completed_at: Time.current)
 
-        Ductwork.configuration.logger.info(
+        Ductwork.logger.info(
           msg: "Pipeline completed",
           pipeline_id: id,
           role: :pipeline_advancer
@@ -167,7 +167,7 @@ module Ductwork
         elsif step_type == "expand"
           expand_to_next_steps(step_type, advancing, edge)
         else
-          Ductwork.configuration.logger.error(
+          Ductwork.logger.error(
             msg: "Invalid step type",
             step_type: step_type,
             pipeline_id: id,
@@ -178,14 +178,26 @@ module Ductwork
     end
 
     def advance_to_next_steps(step_type, advancing, edge)
-      edge[:to].each do |to_klass|
-        next_step = steps.create!(
-          klass: to_klass,
-          status: :in_progress,
-          step_type: step_type,
-          started_at: Time.current
-        )
-        Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
+      too_many = edge[:to].tally.any? do |to_klass, count|
+        depth = Ductwork
+                .configuration
+                .steps_max_depth(pipeline: klass, step: to_klass)
+
+        depth != -1 && count > depth
+      end
+
+      if too_many
+        halted!
+      else
+        edge[:to].each do |to_klass|
+          next_step = steps.create!(
+            klass: to_klass,
+            status: :in_progress,
+            step_type: step_type,
+            started_at: Time.current
+          )
+          Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
+        end
       end
     end
 
@@ -193,7 +205,7 @@ module Ductwork
       if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
         combine_next_steps(edges, advancing)
       else
-        Ductwork.configuration.logger.debug(
+        Ductwork.logger.debug(
           msg: "Not all divided steps have completed; not combining",
           pipeline_id: id,
           role: :pipeline_advancer
@@ -219,12 +231,20 @@ module Ductwork
     end
 
     def expand_to_next_steps(step_type, advancing, edge)
-      Array(advancing.take.job.return_value).each do |input_arg|
-        create_step_and_enqueue_job(
-          klass: edge[:to].sole,
-          step_type: step_type,
-          input_arg: input_arg
-        )
+      next_klass = edge[:to].sole
+      return_value = advancing.take.job.return_value
+      max_depth = Ductwork.configuration.steps_max_depth(pipeline: klass, step: next_klass)
+
+      if max_depth != -1 && return_value.count > max_depth
+        halted!
+      else
+        Array(return_value).each do |input_arg|
+          create_step_and_enqueue_job(
+            klass: next_klass,
+            step_type: step_type,
+            input_arg: input_arg
+          )
+        end
       end
     end
 
@@ -232,7 +252,7 @@ module Ductwork
       if steps.where(status: %w[pending in_progress], klass: step_klass).none?
         collapse_next_steps(edge[:to].sole, advancing)
       else
-        Ductwork.configuration.logger.debug(
+        Ductwork.logger.debug(
           msg: "Not all expanded steps have completed; not collapsing",
           pipeline_id: id,
           role: :pipeline_advancer
@@ -255,7 +275,7 @@ module Ductwork
     end
 
     def log_pipeline_advanced(edges)
-      Ductwork.configuration.logger.info(
+      Ductwork.logger.info(
         msg: "Pipeline advanced",
         pipeline_id: id,
         transitions: edges.map { |_, v| v.dig(-1, :type) },

data/lib/ductwork/cli.rb

@@ -8,12 +8,12 @@ module Ductwork
       def start!(args)
         options = parse_options(args)
         Ductwork.configuration = Configuration.new(**options)
-        Ductwork.configuration.logger = if Ductwork.configuration.logger_source == "rails"
-                                          Rails.logger
-                                        else
-                                          Ductwork::Configuration::DEFAULT_LOGGER
-                                        end
-        Ductwork.configuration.logger.level = Ductwork.configuration.logger_level
+        Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
+                            Rails.logger
+                          else
+                            Ductwork::Configuration::DEFAULT_LOGGER
+                          end
+        Ductwork.logger.level = Ductwork.configuration.logger_level
 
         Ductwork::Processes::SupervisorRunner.start!
       end

data/lib/ductwork/configuration.rb

@@ -12,16 +12,17 @@ module Ductwork
     DEFAULT_LOGGER_SOURCE = "default" # `Logger` instance writing to STDOUT
     DEFAULT_PIPELINE_POLLING_TIMEOUT = 1 # second
     DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT = 20 # seconds
+    DEFAULT_STEPS_MAX_DEPTH = -1 # unlimited count
     DEFAULT_SUPERVISOR_POLLING_TIMEOUT = 1 # second
     DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT = 30 # seconds
     DEFAULT_LOGGER = ::Logger.new($stdout)
     PIPELINES_WILDCARD = "*"
 
-    attr_accessor :logger
-    attr_writer :job_worker_polling_timeout, :job_worker_shutdown_timeout,
-                :job_worker_max_retry,
+    attr_writer :job_worker_count, :job_worker_polling_timeout,
+                :job_worker_shutdown_timeout, :job_worker_max_retry,
                 :logger_level,
                 :pipeline_polling_timeout, :pipeline_shutdown_timeout,
+                :steps_max_depth,
                 :supervisor_polling_timeout, :supervisor_shutdown_timeout
 
     def initialize(path: DEFAULT_FILE_PATH)
@@ -50,6 +51,8 @@ module Ductwork
     end
 
     def job_worker_count(pipeline)
+      return @job_worker_count if instance_variable_defined?(:@job_worker_count)
+
       raw_count = config.dig(:job_worker, :count) || DEFAULT_JOB_WORKER_COUNT
 
       if raw_count.is_a?(Hash)
@@ -59,12 +62,36 @@ module Ductwork
       end
     end
 
-    def job_worker_max_retry
-      @job_worker_max_retry ||= fetch_job_worker_max_retry
+    def job_worker_max_retry(pipeline: nil, step: nil) # rubocop:disable Metrics
+      return @job_worker_max_retry if instance_variable_defined?(:@job_worker_max_retry)
+
+      pipeline ||= :default
+      step ||= :default
+      base_config = config.dig(:job_worker, :max_retry)
+
+      if base_config.is_a?(Hash) && base_config[pipeline.to_sym].is_a?(Hash)
+        pipeline_config = config.dig(:job_worker, :max_retry, pipeline.to_sym)
+
+        pipeline_config[step.to_sym] || pipeline_config[:default] || DEFAULT_JOB_WORKER_MAX_RETRY
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || DEFAULT_JOB_WORKER_MAX_RETRY
+      else
+        base_config || DEFAULT_JOB_WORKER_MAX_RETRY
+      end
     end
 
-    def job_worker_polling_timeout
-      @job_worker_polling_timeout ||= fetch_job_worker_polling_timeout
+    def job_worker_polling_timeout(pipeline = nil)
+      pipeline ||= :default
+      default = DEFAULT_JOB_WORKER_POLLING_TIMEOUT
+      base_config = config.dig(:job_worker, :polling_timeout)
+
+      if instance_variable_defined?(:@job_worker_polling_timeout)
+        @job_worker_polling_timeout
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || default
+      else
+        base_config || default
+      end
     end
 
     def job_worker_shutdown_timeout
@@ -79,14 +106,46 @@ module Ductwork
       @logger_source ||= fetch_logger_source
     end
 
-    def pipeline_polling_timeout
-      @pipeline_polling_timeout ||= fetch_pipeline_polling_timeout
+    def pipeline_polling_timeout(pipeline = nil)
+      pipeline ||= :default
+      default = DEFAULT_PIPELINE_POLLING_TIMEOUT
+      base_config = config.dig(:pipeline_advancer, :polling_timeout)
+
+      if instance_variable_defined?(:@pipeline_polling_timeout)
+        @pipeline_polling_timeout
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || default
+      else
+        base_config || default
+      end
     end
 
     def pipeline_shutdown_timeout
       @pipeline_shutdown_timeout ||= fetch_pipeline_shutdown_timeout
     end
 
+    def steps_max_depth(pipeline: nil, step: nil) # rubocop:disable Metrics
+      return @steps_max_depth if instance_variable_defined?(:@steps_max_depth)
+
+      pipeline ||= :default
+      step ||= :default
+      base_config = config.dig(:pipeline_advancer, :steps, :max_depth)
+
+      if base_config.is_a?(Hash) && base_config[pipeline.to_sym].is_a?(Hash)
+        pipeline_config = config.dig(:pipeline_advancer, :steps, :max_depth, pipeline.to_sym)
+
+        pipeline_config[step.to_sym] ||
+          pipeline_config[:default] ||
+          DEFAULT_STEPS_MAX_DEPTH
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] ||
+          base_config[:default] ||
+          DEFAULT_STEPS_MAX_DEPTH
+      else
+        base_config || DEFAULT_STEPS_MAX_DEPTH
+      end
+    end
+
     def supervisor_polling_timeout
       @supervisor_polling_timeout ||= fetch_supervisor_polling_timeout
     end
@@ -99,16 +158,6 @@ module Ductwork
 
     attr_reader :config
 
-    def fetch_job_worker_max_retry
-      config.dig(:job_worker, :max_retry) ||
-        DEFAULT_JOB_WORKER_MAX_RETRY
-    end
-
-    def fetch_job_worker_polling_timeout
-      config.dig(:job_worker, :polling_timeout) ||
-        DEFAULT_JOB_WORKER_POLLING_TIMEOUT
-    end
-
     def fetch_job_worker_shutdown_timeout
       config.dig(:job_worker, :shutdown_timeout) ||
         DEFAULT_JOB_WORKER_SHUTDOWN_TIMEOUT
@@ -122,11 +171,6 @@ module Ductwork
       config.dig(:logger, :source) || DEFAULT_LOGGER_SOURCE
     end
 
-    def fetch_pipeline_polling_timeout
-      config.dig(:pipeline_advancer, :polling_timeout) ||
-        DEFAULT_PIPELINE_POLLING_TIMEOUT
-    end
-
     def fetch_pipeline_shutdown_timeout
       config.dig(:pipeline_advancer, :shutdown_timeout) ||
         DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT

data/lib/ductwork/dsl/definition_builder.rb

@@ -12,8 +12,7 @@ module Ductwork
           nodes: [],
           edges: {},
         }
-        @divisions = 0
-        @expansions = 0
+        @divergences = []
         @last_nodes = []
       end
 
@@ -42,7 +41,7 @@ module Ductwork
         add_edge_to_last_nodes(*to, type: :divide)
         add_new_nodes(*to)
 
-        @divisions += 1
+        divergences.push(:divide)
 
         if block_given?
           branches = to.map do |klass|
@@ -59,9 +58,9 @@ module Ductwork
       def combine(into:)
         validate_classes!(into)
         validate_definition_started!(action: "combining steps")
-        validate_definition_divided!
+        validate_can_combine!
 
-        @divisions -= 1
+        divergences.pop
 
         last_nodes = definition[:nodes].reverse.select do |node|
           definition[:edges][node].empty?
@@ -83,7 +82,7 @@ module Ductwork
         add_edge_to_last_nodes(to, type: :expand)
         add_new_nodes(to)
 
-        @expansions += 1
+        divergences.push(:expand)
 
         self
       end
@@ -91,11 +90,11 @@ module Ductwork
       def collapse(into:)
         validate_classes!(into)
         validate_definition_started!(action: "collapsing steps")
-        validate_definition_expanded!
+        validate_can_collapse!
         add_edge_to_last_nodes(into, type: :collapse)
         add_new_nodes(into)
 
-        @expansions -= 1
+        divergences.pop
 
         self
       end
@@ -118,7 +117,7 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :divisions, :expansions, :last_nodes
+      attr_reader :definition, :last_nodes, :divergences
 
       def validate_classes!(klasses)
         valid = Array(klasses).all? do |klass|
@@ -150,15 +149,19 @@ module Ductwork
         end
       end
 
-      def validate_definition_divided!
-        if divisions.zero?
+      def validate_can_combine!
+        if divergences.empty?
           raise CombineError, "Must divide pipeline definition before combining steps"
+        elsif divergences[-1] != :divide
+          raise CombineError, "Ambiguous combine on most recently expanded definition"
         end
       end
 
-      def validate_definition_expanded!
-        if expansions.zero?
+      def validate_can_collapse!
+        if divergences.empty?
           raise CollapseError, "Must expand pipeline definition before collapsing steps"
+        elsif divergences[-1] != :expand
+          raise CollapseError, "Ambiguous collapse on most recently divided definition"
         end
       end
 

data/lib/ductwork/engine.rb

@@ -8,7 +8,7 @@ module Ductwork
 
     initializer "ductwork.configure" do
       Ductwork.configuration ||= Ductwork::Configuration.new
-      Ductwork.configuration.logger ||= Rails.logger
+      Ductwork.logger ||= Ductwork::Configuration::DEFAULT_LOGGER
     end
 
     initializer "ductwork.validate_definitions", after: :load_config_initializers do

data/lib/ductwork/processes/job_worker.rb

@@ -10,13 +10,13 @@ module Ductwork
 
       def run
         run_hooks_for(:start)
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker,
           pipeline: pipeline
         )
         while running_context.running?
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Attempting to claim job",
             role: :job_worker,
             pipeline: pipeline
@@ -30,12 +30,12 @@ module Ductwork
               job.execute(pipeline)
             end
           else
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "No job to claim, looping",
               role: :job_worker,
               pipeline: pipeline
             )
-            sleep(Ductwork.configuration.job_worker_polling_timeout)
+            sleep(polling_timeout)
           end
         end
 
@@ -47,7 +47,7 @@ module Ductwork
       attr_reader :pipeline, :running_context
 
       def shutdown
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Shutting down",
           role: :job_worker,
           pipeline: pipeline
@@ -63,8 +63,8 @@ module Ductwork
         end
       end
 
-      def logger
-        Ductwork.configuration.logger
+      def polling_timeout
+        Ductwork.configuration.job_worker_polling_timeout(pipeline)
       end
     end
   end

data/lib/ductwork/processes/job_worker_runner.rb

@@ -25,7 +25,7 @@ module Ductwork
 
       def run
         create_process!
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker_runner,
           pipeline: pipeline
@@ -55,7 +55,7 @@ module Ductwork
             pipeline,
             running_context
           )
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Creating new thread",
             role: :job_worker_runner,
             pipeline: pipeline
@@ -63,9 +63,9 @@ module Ductwork
           thread = Thread.new do
             job_worker.run
           end
-          thread.name = "ductwork.job_worker_#{i}"
+          thread.name = "ductwork.job_worker.#{i}"
 
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Created new thread",
             role: :job_worker_runner,
             pipeline: pipeline
@@ -90,13 +90,13 @@ module Ductwork
       end
 
       def attempt_synchronize_threads
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Attempting to synchronize threads",
           role: :job_worker_runner,
           pipeline: pipeline
         )
         threads.each { |thread| thread.join(0.1) }
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Synchronizing threads timed out",
           role: :job_worker_runner,
           pipeline: pipeline
@@ -104,11 +104,11 @@ module Ductwork
       end
 
       def report_heartbeat!
-        logger.debug(msg: "Reporting heartbeat", role: :job_worker_runner)
+        Ductwork.logger.debug(msg: "Reporting heartbeat", role: :job_worker_runner)
         Ductwork.wrap_with_app_executor do
           Ductwork::Process.report_heartbeat!
         end
-        logger.debug(msg: "Reported heartbeat", role: :job_worker_runner)
+        Ductwork.logger.debug(msg: "Reported heartbeat", role: :job_worker_runner)
       end
 
       def shutdown!
@@ -122,7 +122,7 @@ module Ductwork
         timeout = Ductwork.configuration.job_worker_shutdown_timeout
         deadline = Time.current + timeout
 
-        logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
+        Ductwork.logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
         while Time.current < deadline && threads.any?(&:alive?)
           threads.each do |thread|
             break if Time.current < deadline
@@ -138,7 +138,7 @@ module Ductwork
         threads.each do |thread|
           if thread.alive?
             thread.kill
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "Killed thread",
               role: :job_worker_runner,
               thread: thread.name
@@ -155,10 +155,6 @@ module Ductwork
           ).delete
         end
       end
-
-      def logger
-        Ductwork.configuration.logger
-      end
     end
   end
 end

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -28,7 +28,7 @@ module Ductwork
                            .update_all(claimed_for_advancing_at: Time.current)
 
             if rows_updated == 1
-              logger.debug(
+              Ductwork.logger.debug(
                 msg: "Pipeline claimed",
                 pipeline: klass,
                 role: :pipeline_advancer
@@ -37,13 +37,13 @@ module Ductwork
               pipeline = Ductwork::Pipeline.find(id)
               pipeline.advance!
 
-              logger.debug(
+              Ductwork.logger.debug(
                 msg: "Pipeline advanced",
                 pipeline: klass,
                 role: :pipeline_advancer
               )
             else
-              logger.debug(
+              Ductwork.logger.debug(
                 msg: "Did not claim pipeline, avoided race condition",
                 pipeline: klass,
                 role: :pipeline_advancer
@@ -57,7 +57,7 @@ module Ductwork
               last_advanced_at: Time.current
             )
           else
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "No pipeline needs advancing",
               pipeline: klass,
               id: id,
@@ -65,7 +65,7 @@ module Ductwork
             )
           end
 
-          sleep(Ductwork.configuration.pipeline_polling_timeout)
+          sleep(polling_timeout)
         end
 
         run_hooks_for(:stop)
@@ -83,8 +83,8 @@ module Ductwork
         end
       end
 
-      def logger
-        Ductwork.configuration.logger
+      def polling_timeout
+        Ductwork.configuration.pipeline_polling_timeout(klass)
       end
     end
   end

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -25,7 +25,7 @@ module Ductwork
 
       def run
         create_process!
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :pipeline_advancer_runner
         )
@@ -51,7 +51,7 @@ module Ductwork
             klass
           )
 
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Creating new thread",
             role: :pipeline_advancer_runner,
             pipeline: klass
@@ -63,7 +63,7 @@ module Ductwork
           end
           thread.name = "ductwork.pipeline_advancer.#{klass}"
 
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Created new thread",
             role: :pipeline_advancer_runner,
             thread: thread.name,
@@ -75,12 +75,12 @@ module Ductwork
       end
 
       def attempt_synchronize_threads
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Attempting to synchronize threads",
           role: :pipeline_advancer_runner
         )
         threads.each { |thread| thread.join(0.1) }
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Synchronizing threads timed out",
           role: :pipeline_advancer_runner
         )
@@ -97,11 +97,11 @@ module Ductwork
       end
 
       def report_heartbeat!
-        logger.debug(msg: "Reporting heartbeat", role: :pipeline_advancer_runner)
+        Ductwork.logger.debug(msg: "Reporting heartbeat", role: :pipeline_advancer_runner)
         Ductwork.wrap_with_app_executor do
           Ductwork::Process.report_heartbeat!
         end
-        logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
+        Ductwork.logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
       end
 
       def shutdown
@@ -113,7 +113,7 @@ module Ductwork
       end
 
       def log_shutting_down
-        logger.debug(msg: "Shutting down", role: :pipeline_advancer_runner)
+        Ductwork.logger.debug(msg: "Shutting down", role: :pipeline_advancer_runner)
       end
 
       def stop_running_context
@@ -124,7 +124,7 @@ module Ductwork
         timeout = Ductwork.configuration.pipeline_shutdown_timeout
         deadline = Time.current + timeout
 
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Attempting graceful shutdown",
           role: :pipeline_advancer_runner
         )
@@ -143,7 +143,7 @@ module Ductwork
         threads.each do |thread|
           if thread.alive?
             thread.kill
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "Killed thread",
               role: :pipeline_advancer_runner,
               thread: thread.name
@@ -160,10 +160,6 @@ module Ductwork
           ).delete
         end
       end
-
-      def logger
-        Ductwork.configuration.logger
-      end
     end
   end
 end

data/lib/ductwork/processes/supervisor.rb

@@ -22,14 +22,14 @@ module Ductwork
         end
 
         workers << { metadata:, pid:, block: }
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Started child process (#{pid}) with metadata #{metadata}",
           pid: pid
         )
       end
 
       def run
-        logger.debug(msg: "Entering main work loop", role: :supervisor, pid: ::Process.pid)
+        Ductwork.logger.debug(msg: "Entering main work loop", role: :supervisor, pid: ::Process.pid)
 
         while running
           sleep(Ductwork.configuration.supervisor_polling_timeout)
@@ -42,7 +42,7 @@ module Ductwork
       def shutdown
         @running = false
 
-        logger.debug(msg: "Beginning shutdown", role: :supervisor)
+        Ductwork.logger.debug(msg: "Beginning shutdown", role: :supervisor)
         terminate_gracefully
         wait_for_workers_to_exit
         terminate_immediately
@@ -54,7 +54,7 @@ module Ductwork
       attr_reader :running
 
       def check_workers
-        logger.debug(msg: "Checking workers are alive", role: :supervisor)
+        Ductwork.logger.debug(msg: "Checking workers are alive", role: :supervisor)
 
         workers.each do |worker|
           if process_dead?(worker[:pid])
@@ -63,7 +63,7 @@ module Ductwork
               worker[:block].call(worker[:metadata])
             end
             worker[:pid] = new_pid
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "Restarted process (#{old_pid}) as (#{new_pid})",
               role: :supervisor,
               old_pid: old_pid,
@@ -72,12 +72,12 @@ module Ductwork
           end
         end
 
-        logger.debug(msg: "All workers are alive or revived", role: :supervisor)
+        Ductwork.logger.debug(msg: "All workers are alive or revived", role: :supervisor)
       end
 
       def terminate_gracefully
         workers.each do |worker|
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Sending TERM signal to process (#{worker[:pid]})",
             role: :supervisor,
             pid: worker[:pid],
@@ -95,7 +95,7 @@ module Ductwork
           workers.each_with_index do |worker, index|
             if ::Process.wait(worker[:pid], ::Process::WNOHANG)
               workers[index] = nil
-              logger.debug(
+              Ductwork.logger.debug(
                 msg: "Child process (#{worker[:pid]}) stopped successfully",
                 role: :supervisor,
                 pid: worker[:pid]
@@ -108,7 +108,7 @@ module Ductwork
 
       def terminate_immediately
         workers.each_with_index do |worker, index|
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Sending KILL signal to process (#{worker[:pid]})",
             role: :supervisor,
             pid: worker[:pid],
@@ -117,7 +117,7 @@ module Ductwork
           ::Process.kill(:KILL, worker[:pid])
           ::Process.wait(worker[:pid])
           workers[index] = nil
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Child process (#{worker[:pid]}) killed after timeout",
             role: :supervisor,
             pid: worker[:pid]
@@ -151,10 +151,6 @@ module Ductwork
       def now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
-
-      def logger
-        Ductwork.configuration.logger
-      end
     end
   end
 end

data/lib/ductwork/processes/supervisor_runner.rb

@@ -6,10 +6,9 @@ module Ductwork
       def self.start!
         supervisor = Ductwork::Processes::Supervisor.new
         pipelines_to_advance = Ductwork.configuration.pipelines
-        logger = Ductwork.configuration.logger
 
         supervisor.add_worker(metadata: { pipelines: pipelines_to_advance }) do
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Starting Pipeline Advancer process",
             role: :supervisor_runner
           )
@@ -19,7 +18,7 @@ module Ductwork
 
         pipelines_to_advance.each do |pipeline|
           supervisor.add_worker(metadata: { pipeline: }) do
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "Starting Job Worker Runner process",
               role: :supervisor_runner,
               pipeline: pipeline

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/ductwork.rb

@@ -19,7 +19,7 @@ loader.setup
 
 module Ductwork
   class << self
-    attr_accessor :app_executor, :configuration
+    attr_accessor :app_executor, :configuration, :logger
     attr_writer :defined_pipelines, :hooks
 
     def wrap_with_app_executor(&block)

data/lib/generators/ductwork/install/templates/config/ductwork.yml

@@ -11,6 +11,8 @@ default: &default
   pipeline_advancer:
     polling_timeout: 1
     shutdown_timeout: 20
+    steps:
+      max_depth: -1
   supervisor:
     polling_timeout: 1
     shutdown_timeout: 30

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Tyler Ewing
@@ -86,7 +86,7 @@ dependencies:
 description: Ductwork lets you build complex pipelines quickly and easily using intuitive
   Ruby tooling and a natural DSL.
 email:
-- tewing10@gmail.com
+- contact@getductwork.io
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -140,7 +140,7 @@ licenses:
 - LGPL-3.0-or-later
 metadata:
   homepage_uri: https://github.com/ductwork/ductwork
-  changelog_uri: https://github.com/ductwork/ductwork/blob/main/README.md
+  changelog_uri: https://github.com/ductwork/ductwork/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
   documentation_uri: https://docs.getductwork.io/
 rdoc_options: []