ductwork 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab38ab3c5ce61b7fd6d48eac93d93bcaa33b3476a7b8b6e126403da7b4c2ea04
-  data.tar.gz: 7f3c09913cf33f5f2d6b42f2f3b786b09b35d343d1277262f77b0f9ff666b653
+  metadata.gz: 18ebf23ac85cb80300ecc0c35475e0a2364fa4296a6012fdc21e95b8c5943023
+  data.tar.gz: e08b2cda1074255a7b7a79c5fbe789ac373909eef57b7836be49b3f7554a4350
 SHA512:
-  metadata.gz: cd1bbacdd428ed77064e526a7a2bfe639aad474c19d002417b5d3893856cef916baaba805124e1f4042c90ce972328f3de8c9747c88a96d8b3a60d88a26b183a
-  data.tar.gz: cef8abdee38bea2530ba058fde0062699fc53f190e3e67deb8c369f7c1badae7a7e9538f11ce6fc8f1bce488e15a9ab97271bf6fbc0c59f38f8eea1187f376e8
+  metadata.gz: 9d3b28265bfdf2f90210c43e2609d4255dda6d120ab597b74e54e9c020cbdc4be0b07235bb104873a62ae309511fc841c7d236f20dd52124150760c8dfaa94c8
+  data.tar.gz: fb6db1e8385b880b942404a25aba9ef02637d0c1a2dd53b1a6ddd78a8c70e6494c6349b8d7bbf837727cee5d2ed2c2e2afd67fe506c0be193797fc2de8867619

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Ductwork Changelog
 
+## [0.4.0]
+
+- chore: change job worker thread name format
+- feat: add and respect pipeline-level `pipeline_advancer.polling_timeout` configuration in pipeline advancer
+- feat: respect `job_worker.polling_timeout` configuration in job runner
+- feat: add pipeline-level `job_worker.polling_timeout` configuration
+- feat: check pipeline and step-level max retry configurations when retrying a job
+- feat: add pipeline and step-level `job_worker.max_retry` configurations
+- feat: add ability to set `job_worker.count` config manually
+- chore: move configuration specs under their own directory
+- feat: halt pipeline instead of erroring if max step depth is exceeded
+- chore: move specs under directory
+- feat: allow setting `pipeline_advancer.steps_max_depth` configuration manually
+- feat: raise `Ductwork::Pipeline::StepDepthError` error if return payload count exceeds the configuration
+- feat: add `pipeline_advancer.steps.max_depth` configuration
+
 ## [0.3.1]
 
 - chore: bump dependencies and update necessary files

data/app/models/ductwork/job.rb

@@ -142,9 +142,12 @@ module Ductwork
       end
     end
 
-    def execution_failed!(execution, run, error) # rubocop:disable Metrics/AbcSize
+    def execution_failed!(execution, run, error) # rubocop:todo Metrics
       halted = false
       pipeline = step.pipeline
+      max_retry = Ductwork
+                  .configuration
+                  .job_worker_max_retry(pipeline: pipeline.klass, step: klass)
 
       Ductwork::Record.transaction do
         execution.update!(completed_at: Time.current)
@@ -156,7 +159,7 @@ module Ductwork
           error_backtrace: error.backtrace
         )
 
-        if execution.retry_count < Ductwork.configuration.job_worker_max_retry
+        if execution.retry_count < max_retry
           new_execution = executions.create!(
             retry_count: execution.retry_count + 1,
             started_at: FAILED_EXECUTION_TIMEOUT.from_now

data/app/models/ductwork/pipeline.rb

@@ -178,14 +178,26 @@ module Ductwork
     end
 
     def advance_to_next_steps(step_type, advancing, edge)
-      edge[:to].each do |to_klass|
-        next_step = steps.create!(
-          klass: to_klass,
-          status: :in_progress,
-          step_type: step_type,
-          started_at: Time.current
-        )
-        Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
+      too_many = edge[:to].tally.any? do |to_klass, count|
+        depth = Ductwork
+                .configuration
+                .steps_max_depth(pipeline: klass, step: to_klass)
+
+        depth != -1 && count > depth
+      end
+
+      if too_many
+        halted!
+      else
+        edge[:to].each do |to_klass|
+          next_step = steps.create!(
+            klass: to_klass,
+            status: :in_progress,
+            step_type: step_type,
+            started_at: Time.current
+          )
+          Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
+        end
       end
     end
 
@@ -219,12 +231,20 @@ module Ductwork
     end
 
     def expand_to_next_steps(step_type, advancing, edge)
-      Array(advancing.take.job.return_value).each do |input_arg|
-        create_step_and_enqueue_job(
-          klass: edge[:to].sole,
-          step_type: step_type,
-          input_arg: input_arg
-        )
+      next_klass = edge[:to].sole
+      return_value = advancing.take.job.return_value
+      max_depth = Ductwork.configuration.steps_max_depth(pipeline: klass, step: next_klass)
+
+      if max_depth != -1 && return_value.count > max_depth
+        halted!
+      else
+        Array(return_value).each do |input_arg|
+          create_step_and_enqueue_job(
+            klass: next_klass,
+            step_type: step_type,
+            input_arg: input_arg
+          )
+        end
       end
     end
 

data/lib/ductwork/configuration.rb

@@ -12,15 +12,17 @@ module Ductwork
     DEFAULT_LOGGER_SOURCE = "default" # `Logger` instance writing to STDOUT
     DEFAULT_PIPELINE_POLLING_TIMEOUT = 1 # second
     DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT = 20 # seconds
+    DEFAULT_STEPS_MAX_DEPTH = -1 # unlimited count
     DEFAULT_SUPERVISOR_POLLING_TIMEOUT = 1 # second
     DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT = 30 # seconds
     DEFAULT_LOGGER = ::Logger.new($stdout)
     PIPELINES_WILDCARD = "*"
 
-    attr_writer :job_worker_polling_timeout, :job_worker_shutdown_timeout,
-                :job_worker_max_retry,
+    attr_writer :job_worker_count, :job_worker_polling_timeout,
+                :job_worker_shutdown_timeout, :job_worker_max_retry,
                 :logger_level,
                 :pipeline_polling_timeout, :pipeline_shutdown_timeout,
+                :steps_max_depth,
                 :supervisor_polling_timeout, :supervisor_shutdown_timeout
 
     def initialize(path: DEFAULT_FILE_PATH)
@@ -49,6 +51,8 @@ module Ductwork
     end
 
     def job_worker_count(pipeline)
+      return @job_worker_count if instance_variable_defined?(:@job_worker_count)
+
       raw_count = config.dig(:job_worker, :count) || DEFAULT_JOB_WORKER_COUNT
 
       if raw_count.is_a?(Hash)
@@ -58,12 +62,36 @@ module Ductwork
       end
     end
 
-    def job_worker_max_retry
-      @job_worker_max_retry ||= fetch_job_worker_max_retry
+    def job_worker_max_retry(pipeline: nil, step: nil) # rubocop:disable Metrics
+      return @job_worker_max_retry if instance_variable_defined?(:@job_worker_max_retry)
+
+      pipeline ||= :default
+      step ||= :default
+      base_config = config.dig(:job_worker, :max_retry)
+
+      if base_config.is_a?(Hash) && base_config[pipeline.to_sym].is_a?(Hash)
+        pipeline_config = config.dig(:job_worker, :max_retry, pipeline.to_sym)
+
+        pipeline_config[step.to_sym] || pipeline_config[:default] || DEFAULT_JOB_WORKER_MAX_RETRY
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || DEFAULT_JOB_WORKER_MAX_RETRY
+      else
+        base_config || DEFAULT_JOB_WORKER_MAX_RETRY
+      end
     end
 
-    def job_worker_polling_timeout
-      @job_worker_polling_timeout ||= fetch_job_worker_polling_timeout
+    def job_worker_polling_timeout(pipeline = nil)
+      pipeline ||= :default
+      default = DEFAULT_JOB_WORKER_POLLING_TIMEOUT
+      base_config = config.dig(:job_worker, :polling_timeout)
+
+      if instance_variable_defined?(:@job_worker_polling_timeout)
+        @job_worker_polling_timeout
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || default
+      else
+        base_config || default
+      end
     end
 
     def job_worker_shutdown_timeout
@@ -78,14 +106,46 @@ module Ductwork
       @logger_source ||= fetch_logger_source
     end
 
-    def pipeline_polling_timeout
-      @pipeline_polling_timeout ||= fetch_pipeline_polling_timeout
+    def pipeline_polling_timeout(pipeline = nil)
+      pipeline ||= :default
+      default = DEFAULT_PIPELINE_POLLING_TIMEOUT
+      base_config = config.dig(:pipeline_advancer, :polling_timeout)
+
+      if instance_variable_defined?(:@pipeline_polling_timeout)
+        @pipeline_polling_timeout
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || default
+      else
+        base_config || default
+      end
     end
 
     def pipeline_shutdown_timeout
       @pipeline_shutdown_timeout ||= fetch_pipeline_shutdown_timeout
     end
 
+    def steps_max_depth(pipeline: nil, step: nil) # rubocop:disable Metrics
+      return @steps_max_depth if instance_variable_defined?(:@steps_max_depth)
+
+      pipeline ||= :default
+      step ||= :default
+      base_config = config.dig(:pipeline_advancer, :steps, :max_depth)
+
+      if base_config.is_a?(Hash) && base_config[pipeline.to_sym].is_a?(Hash)
+        pipeline_config = config.dig(:pipeline_advancer, :steps, :max_depth, pipeline.to_sym)
+
+        pipeline_config[step.to_sym] ||
+          pipeline_config[:default] ||
+          DEFAULT_STEPS_MAX_DEPTH
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] ||
+          base_config[:default] ||
+          DEFAULT_STEPS_MAX_DEPTH
+      else
+        base_config || DEFAULT_STEPS_MAX_DEPTH
+      end
+    end
+
     def supervisor_polling_timeout
       @supervisor_polling_timeout ||= fetch_supervisor_polling_timeout
     end
@@ -98,16 +158,6 @@ module Ductwork
 
     attr_reader :config
 
-    def fetch_job_worker_max_retry
-      config.dig(:job_worker, :max_retry) ||
-        DEFAULT_JOB_WORKER_MAX_RETRY
-    end
-
-    def fetch_job_worker_polling_timeout
-      config.dig(:job_worker, :polling_timeout) ||
-        DEFAULT_JOB_WORKER_POLLING_TIMEOUT
-    end
-
     def fetch_job_worker_shutdown_timeout
       config.dig(:job_worker, :shutdown_timeout) ||
         DEFAULT_JOB_WORKER_SHUTDOWN_TIMEOUT
@@ -121,11 +171,6 @@ module Ductwork
       config.dig(:logger, :source) || DEFAULT_LOGGER_SOURCE
     end
 
-    def fetch_pipeline_polling_timeout
-      config.dig(:pipeline_advancer, :polling_timeout) ||
-        DEFAULT_PIPELINE_POLLING_TIMEOUT
-    end
-
     def fetch_pipeline_shutdown_timeout
       config.dig(:pipeline_advancer, :shutdown_timeout) ||
         DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT

data/lib/ductwork/processes/job_worker.rb

@@ -35,7 +35,7 @@ module Ductwork
               role: :job_worker,
               pipeline: pipeline
             )
-            sleep(Ductwork.configuration.job_worker_polling_timeout)
+            sleep(polling_timeout)
           end
         end
 
@@ -62,6 +62,10 @@ module Ductwork
           end
         end
       end
+
+      def polling_timeout
+        Ductwork.configuration.job_worker_polling_timeout(pipeline)
+      end
     end
   end
 end

data/lib/ductwork/processes/job_worker_runner.rb

@@ -63,7 +63,7 @@ module Ductwork
           thread = Thread.new do
             job_worker.run
           end
-          thread.name = "ductwork.job_worker_#{i}"
+          thread.name = "ductwork.job_worker.#{i}"
 
           Ductwork.logger.debug(
             msg: "Created new thread",

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -65,7 +65,7 @@ module Ductwork
             )
           end
 
-          sleep(Ductwork.configuration.pipeline_polling_timeout)
+          sleep(polling_timeout)
         end
 
         run_hooks_for(:stop)
@@ -82,6 +82,10 @@ module Ductwork
           end
         end
       end
+
+      def polling_timeout
+        Ductwork.configuration.pipeline_polling_timeout(klass)
+      end
     end
   end
 end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/lib/generators/ductwork/install/templates/config/ductwork.yml

@@ -11,6 +11,8 @@ default: &default
   pipeline_advancer:
     polling_timeout: 1
     shutdown_timeout: 20
+    steps:
+      max_depth: -1
   supervisor:
     polling_timeout: 1
     shutdown_timeout: 30

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Tyler Ewing