ductwork 0.2.1 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ab7c6e79c26352d990680a3913a26ea8bd4bc9999136b4ee1a1846c06e74523
-  data.tar.gz: 7adb0603119c80d5edc54e94de44828bba2c927edad418e5944d594ba21f734a
+  metadata.gz: ab38ab3c5ce61b7fd6d48eac93d93bcaa33b3476a7b8b6e126403da7b4c2ea04
+  data.tar.gz: 7f3c09913cf33f5f2d6b42f2f3b786b09b35d343d1277262f77b0f9ff666b653
 SHA512:
-  metadata.gz: 83be2952cf26ebbee28521df07b68ee29e87ade164a70a87ede7c7bcfc1ef5525097b1346757d76e34d4fc25535c5489f56900fa763006d923bd8491dcba520f
-  data.tar.gz: 53cabb6cdfa4f96f847ddc68818a3f0a87d59ca443282700eca5f690a8a01805e46dbd925990565630bb5c45dc728183de251629488d055f8062d136f06dff7e
+  metadata.gz: cd1bbacdd428ed77064e526a7a2bfe639aad474c19d002417b5d3893856cef916baaba805124e1f4042c90ce972328f3de8c9747c88a96d8b3a60d88a26b183a
+  data.tar.gz: cef8abdee38bea2530ba058fde0062699fc53f190e3e67deb8c369f7c1badae7a7e9538f11ce6fc8f1bce488e15a9ab97271bf6fbc0c59f38f8eea1187f376e8

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Ductwork Changelog
 
+## [0.3.1]
+
+- chore: bump dependencies and update necessary files
+- chore: update email address in gemspec
+- chore: move `logger` out of config to top-level `Ductwork` module
+- chore: promote `Ductwork::Pipeline#parsed_definition` to a public method
+- fix: raise when trying to collapse a most recently divided pipeline and vice versa
+
+## [0.3.0]
+
+- fix: correctly create collapsing and combining steps and jobs for complex pipelines
+- fix: add a new step and job for each active branch in a running pipeline
+- fix: add a new node and edge for each active branch of the definition
+- feat: add info-level logging for job events
+- feat: add info-level logging for pipeline events
+
 ## [0.2.1]
 
 - fix: do not splat arguments when executing a job nor triggering a pipeline

data/README.md

@@ -103,7 +103,7 @@ end
 
 **Important:** Return values must be JSON-serializable.
 
-See [Defining Pipelines](https://docs.getductwork.io/getting_started/defining-pipelines.html) for detailed documentation.
+See [Defining Pipelines](https://docs.getductwork.io/getting-started/defining-pipelines.html) for detailed documentation.
 
 ### 4. Run Ductwork
 

data/app/models/ductwork/job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Ductwork
-  class Job < Ductwork::Record
+  class Job < Ductwork::Record # rubocop:todo Metrics/ClassLength
     belongs_to :step, class_name: "Ductwork::Step"
     has_many :executions, class_name: "Ductwork::Execution", foreign_key: "job_id", dependent: :destroy
 
@@ -39,7 +39,7 @@ module Ductwork
         end
 
         if rows_updated == 1
-          Ductwork.configuration.logger.debug(
+          Ductwork.logger.debug(
             msg: "Job claimed",
             role: :job_worker,
             process_id: process_id,
@@ -49,7 +49,7 @@ module Ductwork
             .joins(executions: :availability)
             .find_by(ductwork_availabilities: { id:, process_id: })
         else
-          Ductwork.configuration.logger.debug(
+          Ductwork.logger.debug(
             msg: "Did not claim job, avoided race condition",
             role: :job_worker,
             process_id: process_id,
@@ -61,13 +61,13 @@ module Ductwork
     end
 
     def self.enqueue(step, args)
-      Ductwork::Record.transaction do
-        job = step.create_job!(
+      job = Ductwork::Record.transaction do
+        j = step.create_job!(
           klass: step.klass,
           started_at: Time.current,
           input_args: JSON.dump({ args: })
         )
-        execution = job.executions.create!(
+        execution = j.executions.create!(
           started_at: Time.current,
           retry_count: 0
         )
@@ -75,14 +75,22 @@ module Ductwork
           started_at: Time.current
         )
 
-        job
+        j
       end
+
+      Ductwork.logger.info(
+        msg: "Job enqueued",
+        job_id: job.id,
+        job_klass: job.klass
+      )
+
+      job
     end
 
     def execute(pipeline)
       # i don't _really_ like this, but it should be fine for now...
       execution = executions.order(:created_at).last
-      logger.debug(
+      Ductwork.logger.debug(
         msg: "Executing job",
         role: :job_worker,
         pipeline: pipeline,
@@ -103,12 +111,13 @@ module Ductwork
         execution_failed!(execution, run, e)
         result = "failure"
       ensure
-        logger.debug(
-          msg: "Executed job",
-          role: :job_worker,
+        Ductwork.logger.info(
+          msg: "Job executed",
           pipeline: pipeline,
+          job_id: id,
           job_klass: klass,
-          result: result
+          result: result,
+          role: :job_worker
         )
       end
     end
@@ -121,10 +130,6 @@ module Ductwork
 
     private
 
-    def logger
-      Ductwork.configuration.logger
-    end
-
     def execution_succeeded!(execution, run, output_payload)
       payload = JSON.dump({ payload: output_payload })
 
@@ -167,6 +172,16 @@ module Ductwork
         end
       end
 
+      Ductwork.logger.warn(
+        msg: "Job errored",
+        error_klass: error.class.name,
+        error_message: error.message,
+        job_id: id,
+        job_klass: klass,
+        pipeline_id: pipeline.id,
+        role: :job_worker
+      )
+
       # NOTE: perform lifecycle hook execution outside of the transaction as
       # to not unnecessarily hold it open
       if halted

data/app/models/ductwork/pipeline.rb

@@ -56,8 +56,8 @@ module Ductwork
         step_klass = pipeline_definition.dig(:nodes, 0)
         definition = JSON.dump(pipeline_definition)
 
-        Record.transaction do
-          pipeline = create!(
+        pipeline = Record.transaction do
+          p = create!(
             klass: name.to_s,
             status: :in_progress,
             definition: definition,
@@ -65,7 +65,7 @@ module Ductwork
             triggered_at: Time.current,
             last_advanced_at: Time.current
           )
-          step = pipeline.steps.create!(
+          step = p.steps.create!(
             klass: step_klass,
             status: :in_progress,
             step_type: :start,
@@ -73,28 +73,44 @@ module Ductwork
           )
           Ductwork::Job.enqueue(step, args)
 
-          pipeline
+          p
         end
+
+        Ductwork.logger.info(
+          msg: "Pipeline triggered",
+          pipeline_id: pipeline.id,
+          role: :application
+        )
+
+        pipeline
       end
     end
 
     def advance!
-      step = steps.advancing.take
-      edge = if step.present?
-               parsed_definition.dig(:edges, step.klass, 0)
-             end
+      # NOTE: there could be A LOT of steps advancing for a single pipeline
+      # so instead of loading everything into memory and using ruby collection
+      # methods we make multiple queries. may need to revisist this once
+      # we do extensive load testing
+      advancing = steps.advancing
+      edges = if advancing.exists?
+                parsed_definition
+                  .fetch(:edges, {})
+                  .select { |k| k.in?(advancing.pluck(:klass)) }
+              end
 
       Ductwork::Record.transaction do
-        steps.advancing.update!(status: :completed, completed_at: Time.current)
-
-        if edge.nil?
-          conditionally_complete_pipeline
+        if edges.nil? || edges.values.all?(&:empty?)
+          conditionally_complete_pipeline(advancing)
         else
-          advance_to_next_step_by_type(edge, step)
+          advance_to_next_steps_by_type(edges, advancing)
         end
       end
     end
 
+    def parsed_definition
+      @parsed_definition ||= JSON.parse(definition).with_indifferent_access
+    end
+
     private
 
     def create_step_and_enqueue_job(klass:, step_type:, input_arg:)
@@ -104,38 +120,64 @@ module Ductwork
       Ductwork::Job.enqueue(next_step, input_arg)
     end
 
-    def parsed_definition
-      @parsed_definition ||= JSON.parse(definition).with_indifferent_access
-    end
+    def conditionally_complete_pipeline(advancing)
+      advancing.update!(status: :completed, completed_at: Time.current)
 
-    def conditionally_complete_pipeline
       if steps.where(status: %w[in_progress pending]).none?
         update!(status: :completed, completed_at: Time.current)
+
+        Ductwork.logger.info(
+          msg: "Pipeline completed",
+          pipeline_id: id,
+          role: :pipeline_advancer
+        )
       end
     end
 
-    def advance_to_next_step_by_type(edge, step)
-      # NOTE: "chain" is used by ActiveRecord so we have to call
-      # this enum value "default" :sad:
-      step_type = edge[:type] == "chain" ? "default" : edge[:type]
-
-      if step_type.in?(%w[default divide])
-        advance_to_next_steps(step_type, step, edge)
-      elsif step_type == "combine"
-        combine_next_steps(step_type, edge)
-      elsif step_type == "expand"
-        expand_to_next_steps(step_type, step, edge)
-      elsif step_type == "collapse"
-        collapse_next_steps(step_type, step, edge)
+    def advance_to_next_steps_by_type(edges, advancing)
+      if edges.all? { |_, v| v.dig(-1, :type) == "combine" }
+        conditionally_combine_next_steps(edges, advancing)
       else
-        Ductwork.configuration.logger.error(
-          msg: "Invalid step type",
-          role: :pipeline_advancer
-        )
+        edges.each do |step_klass, step_edges|
+          edge = step_edges[-1]
+          # NOTE: "chain" is used by ActiveRecord so we have to call
+          # this enum value "default" :sad:
+          step_type = edge[:type] == "chain" ? "default" : edge[:type]
+
+          if step_type == "collapse"
+            conditionally_collapse_next_steps(step_klass, edge, advancing)
+          else
+            advance_non_merging_steps(step_klass, edges, advancing)
+          end
+        end
+      end
+      advancing.update!(status: :completed, completed_at: Time.current)
+      log_pipeline_advanced(edges)
+    end
+
+    def advance_non_merging_steps(step_klass, edges, advancing)
+      advancing.where(klass: step_klass).find_each do |step|
+        edge = edges.dig(step.klass, -1)
+        # NOTE: "chain" is used by ActiveRecord so we have to call
+        # this enum value "default" :sad:
+        step_type = edge[:type] == "chain" ? "default" : edge[:type]
+
+        if step_type.in?(%w[default divide])
+          advance_to_next_steps(step_type, advancing, edge)
+        elsif step_type == "expand"
+          expand_to_next_steps(step_type, advancing, edge)
+        else
+          Ductwork.logger.error(
+            msg: "Invalid step type",
+            step_type: step_type,
+            pipeline_id: id,
+            role: :pipeline_advancer
+          )
+        end
       end
     end
 
-    def advance_to_next_steps(step_type, step, edge)
+    def advance_to_next_steps(step_type, advancing, edge)
       edge[:to].each do |to_klass|
         next_step = steps.create!(
           klass: to_klass,
@@ -143,53 +185,82 @@ module Ductwork
           step_type: step_type,
           started_at: Time.current
         )
-        Ductwork::Job.enqueue(next_step, step.job.return_value)
+        Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
       end
     end
 
-    def combine_next_steps(step_type, edge)
-      previous_klasses = parsed_definition[:edges].select do |_, v|
-        v.dig(0, :to, 0) == edge[:to].sole && v.dig(0, :type) == "combine"
-      end.keys
-
-      if steps.not_completed.where(klass: previous_klasses).none?
-        input_arg = Job.where(
-          step: steps.completed.where(klass: previous_klasses)
-        ).map(&:return_value)
-        create_step_and_enqueue_job(
-          klass: edge[:to].sole,
-          step_type: step_type,
-          input_arg: input_arg
+    def conditionally_combine_next_steps(edges, advancing)
+      if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
+        combine_next_steps(edges, advancing)
+      else
+        Ductwork.logger.debug(
+          msg: "Not all divided steps have completed; not combining",
+          pipeline_id: id,
+          role: :pipeline_advancer
         )
       end
     end
 
-    def expand_to_next_steps(step_type, step, edge)
-      step.job.return_value.each do |input_arg|
-        create_step_and_enqueue_job(
-          klass: edge[:to].sole,
-          step_type: step_type,
-          input_arg: input_arg
-        )
+    def combine_next_steps(edges, advancing)
+      klass = edges.values.sample.dig(-1, :to).sole
+      step_type = "combine"
+      groups = advancing
+               .group(:klass)
+               .count
+               .keys
+               .map { |k| advancing.where(klass: k) }
+
+      groups.first.zip(*groups[1..]).each do |group|
+        input_arg = Ductwork::Job
+                    .where(step_id: group.map(&:id))
+                    .map(&:return_value)
+        create_step_and_enqueue_job(klass:, step_type:, input_arg:)
       end
     end
 
-    def collapse_next_steps(step_type, step, edge)
-      if steps.not_completed.where(klass: step.klass).none?
-        input_arg = Job.where(
-          step: steps.completed.where(klass: step.klass)
-        ).map(&:return_value)
+    def expand_to_next_steps(step_type, advancing, edge)
+      Array(advancing.take.job.return_value).each do |input_arg|
         create_step_and_enqueue_job(
           klass: edge[:to].sole,
           step_type: step_type,
           input_arg: input_arg
         )
+      end
+    end
+
+    def conditionally_collapse_next_steps(step_klass, edge, advancing)
+      if steps.where(status: %w[pending in_progress], klass: step_klass).none?
+        collapse_next_steps(edge[:to].sole, advancing)
       else
-        Ductwork.configuration.logger.debug(
-          msg: "Not all expanded steps have completed",
+        Ductwork.logger.debug(
+          msg: "Not all expanded steps have completed; not collapsing",
+          pipeline_id: id,
           role: :pipeline_advancer
         )
       end
     end
+
+    def collapse_next_steps(klass, advancing)
+      step_type = "collapse"
+      input_arg = []
+
+      # NOTE: because of expanding based on return values, there
+      # could be A LOT of jobs so we want to use batch methods
+      # to avoid creating too many in-memory objects
+      Ductwork::Job.where(step_id: advancing.ids).find_each do |job|
+        input_arg << job.return_value
+      end
+
+      create_step_and_enqueue_job(klass:, step_type:, input_arg:)
+    end
+
+    def log_pipeline_advanced(edges)
+      Ductwork.logger.info(
+        msg: "Pipeline advanced",
+        pipeline_id: id,
+        transitions: edges.map { |_, v| v.dig(-1, :type) },
+        role: :pipeline_advancer
+      )
+    end
   end
 end

data/lib/ductwork/cli.rb

@@ -8,12 +8,12 @@ module Ductwork
       def start!(args)
         options = parse_options(args)
         Ductwork.configuration = Configuration.new(**options)
-        Ductwork.configuration.logger = if Ductwork.configuration.logger_source == "rails"
-                                          Rails.logger
-                                        else
-                                          Ductwork::Configuration::DEFAULT_LOGGER
-                                        end
-        Ductwork.configuration.logger.level = Ductwork.configuration.logger_level
+        Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
+                            Rails.logger
+                          else
+                            Ductwork::Configuration::DEFAULT_LOGGER
+                          end
+        Ductwork.logger.level = Ductwork.configuration.logger_level
 
         Ductwork::Processes::SupervisorRunner.start!
       end

data/lib/ductwork/configuration.rb

@@ -17,7 +17,6 @@ module Ductwork
     DEFAULT_LOGGER = ::Logger.new($stdout)
     PIPELINES_WILDCARD = "*"
 
-    attr_accessor :logger
     attr_writer :job_worker_polling_timeout, :job_worker_shutdown_timeout,
                 :job_worker_max_retry,
                 :logger_level,

data/lib/ductwork/dsl/definition_builder.rb

@@ -12,8 +12,8 @@ module Ductwork
           nodes: [],
           edges: {},
         }
-        @divisions = 0
-        @expansions = 0
+        @divergences = []
+        @last_nodes = []
       end
 
       def start(klass)
@@ -29,7 +29,7 @@ module Ductwork
       def chain(klass)
         validate_classes!(klass)
         validate_definition_started!(action: "chaining")
-        add_edge_to_last_node(klass, type: :chain)
+        add_edge_to_last_nodes(klass, type: :chain)
         add_new_nodes(klass)
 
         self
@@ -38,10 +38,10 @@ module Ductwork
       def divide(to:)
         validate_classes!(to)
         validate_definition_started!(action: "dividing chain")
-        add_edge_to_last_node(*to, type: :divide)
+        add_edge_to_last_nodes(*to, type: :divide)
         add_new_nodes(*to)
 
-        @divisions += 1
+        divergences.push(:divide)
 
         if block_given?
           branches = to.map do |klass|
@@ -58,9 +58,9 @@ module Ductwork
       def combine(into:)
         validate_classes!(into)
         validate_definition_started!(action: "combining steps")
-        validate_definition_divided!
+        validate_can_combine!
 
-        @divisions -= 1
+        divergences.pop
 
         last_nodes = definition[:nodes].reverse.select do |node|
           definition[:edges][node].empty?
@@ -79,10 +79,10 @@ module Ductwork
       def expand(to:)
         validate_classes!(to)
         validate_definition_started!(action: "expanding chain")
-        add_edge_to_last_node(to, type: :expand)
+        add_edge_to_last_nodes(to, type: :expand)
         add_new_nodes(to)
 
-        @expansions += 1
+        divergences.push(:expand)
 
         self
       end
@@ -90,11 +90,11 @@ module Ductwork
       def collapse(into:)
         validate_classes!(into)
         validate_definition_started!(action: "collapsing steps")
-        validate_definition_expanded!
-        add_edge_to_last_node(into, type: :collapse)
+        validate_can_collapse!
+        add_edge_to_last_nodes(into, type: :collapse)
         add_new_nodes(into)
 
-        @expansions -= 1
+        divergences.pop
 
         self
       end
@@ -117,7 +117,7 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :divisions, :expansions
+      attr_reader :definition, :last_nodes, :divergences
 
       def validate_classes!(klasses)
         valid = Array(klasses).all? do |klass|
@@ -149,32 +149,39 @@ module Ductwork
         end
       end
 
-      def validate_definition_divided!
-        if divisions.zero?
+      def validate_can_combine!
+        if divergences.empty?
           raise CombineError, "Must divide pipeline definition before combining steps"
+        elsif divergences[-1] != :divide
+          raise CombineError, "Ambiguous combine on most recently expanded definition"
         end
       end
 
-      def validate_definition_expanded!
-        if expansions.zero?
+      def validate_can_collapse!
+        if divergences.empty?
           raise CollapseError, "Must expand pipeline definition before collapsing steps"
+        elsif divergences[-1] != :expand
+          raise CollapseError, "Ambiguous collapse on most recently divided definition"
         end
       end
 
       def add_new_nodes(*klasses)
-        definition[:nodes].push(*klasses.map(&:name))
+        nodes = klasses.map(&:name)
+        @last_nodes = Array(nodes)
+
+        definition[:nodes].push(*nodes)
         klasses.each do |klass|
           definition[:edges][klass.name] ||= []
         end
       end
 
-      def add_edge_to_last_node(*klasses, type:)
-        last_node = definition.dig(:nodes, -1)
-
-        definition[:edges][last_node] << {
-          to: klasses.map(&:name),
-          type: type,
-        }
+      def add_edge_to_last_nodes(*klasses, type:)
+        last_nodes.each do |last_node|
+          definition[:edges][last_node] << {
+            to: klasses.map(&:name),
+            type: type,
+          }
+        end
       end
     end
   end

data/lib/ductwork/engine.rb

@@ -8,7 +8,7 @@ module Ductwork
 
     initializer "ductwork.configure" do
       Ductwork.configuration ||= Ductwork::Configuration.new
-      Ductwork.configuration.logger ||= Rails.logger
+      Ductwork.logger ||= Ductwork::Configuration::DEFAULT_LOGGER
     end
 
     initializer "ductwork.validate_definitions", after: :load_config_initializers do

data/lib/ductwork/processes/job_worker.rb

@@ -10,13 +10,13 @@ module Ductwork
 
       def run
         run_hooks_for(:start)
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker,
           pipeline: pipeline
         )
         while running_context.running?
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Attempting to claim job",
             role: :job_worker,
             pipeline: pipeline
@@ -30,7 +30,7 @@ module Ductwork
               job.execute(pipeline)
             end
           else
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "No job to claim, looping",
               role: :job_worker,
               pipeline: pipeline
@@ -47,7 +47,7 @@ module Ductwork
       attr_reader :pipeline, :running_context
 
       def shutdown
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Shutting down",
           role: :job_worker,
           pipeline: pipeline
@@ -62,10 +62,6 @@ module Ductwork
           end
         end
       end
-
-      def logger
-        Ductwork.configuration.logger
-      end
     end
   end
 end

data/lib/ductwork/processes/job_worker_runner.rb

@@ -25,7 +25,7 @@ module Ductwork
 
       def run
         create_process!
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker_runner,
           pipeline: pipeline
@@ -55,7 +55,7 @@ module Ductwork
             pipeline,
             running_context
           )
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Creating new thread",
             role: :job_worker_runner,
             pipeline: pipeline
@@ -65,7 +65,7 @@ module Ductwork
           end
           thread.name = "ductwork.job_worker_#{i}"
 
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Created new thread",
             role: :job_worker_runner,
             pipeline: pipeline
@@ -90,13 +90,13 @@ module Ductwork
       end
 
       def attempt_synchronize_threads
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Attempting to synchronize threads",
           role: :job_worker_runner,
           pipeline: pipeline
         )
         threads.each { |thread| thread.join(0.1) }
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Synchronizing threads timed out",
           role: :job_worker_runner,
           pipeline: pipeline
@@ -104,11 +104,11 @@ module Ductwork
       end
 
       def report_heartbeat!
-        logger.debug(msg: "Reporting heartbeat", role: :job_worker_runner)
+        Ductwork.logger.debug(msg: "Reporting heartbeat", role: :job_worker_runner)
         Ductwork.wrap_with_app_executor do
           Ductwork::Process.report_heartbeat!
         end
-        logger.debug(msg: "Reported heartbeat", role: :job_worker_runner)
+        Ductwork.logger.debug(msg: "Reported heartbeat", role: :job_worker_runner)
       end
 
       def shutdown!
@@ -122,7 +122,7 @@ module Ductwork
         timeout = Ductwork.configuration.job_worker_shutdown_timeout
         deadline = Time.current + timeout
 
-        logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
+        Ductwork.logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
         while Time.current < deadline && threads.any?(&:alive?)
           threads.each do |thread|
             break if Time.current < deadline
@@ -138,7 +138,7 @@ module Ductwork
         threads.each do |thread|
           if thread.alive?
             thread.kill
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "Killed thread",
               role: :job_worker_runner,
               thread: thread.name
@@ -155,10 +155,6 @@ module Ductwork
           ).delete
         end
       end
-
-      def logger
-        Ductwork.configuration.logger
-      end
     end
   end
 end

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -28,7 +28,7 @@ module Ductwork
                            .update_all(claimed_for_advancing_at: Time.current)
 
             if rows_updated == 1
-              logger.debug(
+              Ductwork.logger.debug(
                 msg: "Pipeline claimed",
                 pipeline: klass,
                 role: :pipeline_advancer
@@ -37,13 +37,13 @@ module Ductwork
               pipeline = Ductwork::Pipeline.find(id)
               pipeline.advance!
 
-              logger.debug(
+              Ductwork.logger.debug(
                 msg: "Pipeline advanced",
                 pipeline: klass,
                 role: :pipeline_advancer
               )
             else
-              logger.debug(
+              Ductwork.logger.debug(
                 msg: "Did not claim pipeline, avoided race condition",
                 pipeline: klass,
                 role: :pipeline_advancer
@@ -57,7 +57,7 @@ module Ductwork
               last_advanced_at: Time.current
             )
           else
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "No pipeline needs advancing",
               pipeline: klass,
               id: id,
@@ -82,10 +82,6 @@ module Ductwork
           end
         end
       end
-
-      def logger
-        Ductwork.configuration.logger
-      end
     end
   end
 end

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -25,7 +25,7 @@ module Ductwork
 
       def run
         create_process!
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :pipeline_advancer_runner
         )
@@ -51,7 +51,7 @@ module Ductwork
             klass
           )
 
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Creating new thread",
             role: :pipeline_advancer_runner,
             pipeline: klass
@@ -63,7 +63,7 @@ module Ductwork
           end
           thread.name = "ductwork.pipeline_advancer.#{klass}"
 
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Created new thread",
             role: :pipeline_advancer_runner,
             thread: thread.name,
@@ -75,12 +75,12 @@ module Ductwork
       end
 
       def attempt_synchronize_threads
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Attempting to synchronize threads",
           role: :pipeline_advancer_runner
         )
         threads.each { |thread| thread.join(0.1) }
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Synchronizing threads timed out",
           role: :pipeline_advancer_runner
         )
@@ -97,11 +97,11 @@ module Ductwork
       end
 
       def report_heartbeat!
-        logger.debug(msg: "Reporting heartbeat", role: :pipeline_advancer_runner)
+        Ductwork.logger.debug(msg: "Reporting heartbeat", role: :pipeline_advancer_runner)
         Ductwork.wrap_with_app_executor do
           Ductwork::Process.report_heartbeat!
         end
-        logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
+        Ductwork.logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
       end
 
       def shutdown
@@ -113,7 +113,7 @@ module Ductwork
       end
 
       def log_shutting_down
-        logger.debug(msg: "Shutting down", role: :pipeline_advancer_runner)
+        Ductwork.logger.debug(msg: "Shutting down", role: :pipeline_advancer_runner)
       end
 
       def stop_running_context
@@ -124,7 +124,7 @@ module Ductwork
         timeout = Ductwork.configuration.pipeline_shutdown_timeout
         deadline = Time.current + timeout
 
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Attempting graceful shutdown",
           role: :pipeline_advancer_runner
         )
@@ -143,7 +143,7 @@ module Ductwork
         threads.each do |thread|
           if thread.alive?
             thread.kill
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "Killed thread",
               role: :pipeline_advancer_runner,
               thread: thread.name
@@ -160,10 +160,6 @@ module Ductwork
           ).delete
         end
       end
-
-      def logger
-        Ductwork.configuration.logger
-      end
     end
   end
 end

data/lib/ductwork/processes/supervisor.rb

@@ -22,14 +22,14 @@ module Ductwork
         end
 
         workers << { metadata:, pid:, block: }
-        logger.debug(
+        Ductwork.logger.debug(
           msg: "Started child process (#{pid}) with metadata #{metadata}",
           pid: pid
         )
       end
 
       def run
-        logger.debug(msg: "Entering main work loop", role: :supervisor, pid: ::Process.pid)
+        Ductwork.logger.debug(msg: "Entering main work loop", role: :supervisor, pid: ::Process.pid)
 
         while running
           sleep(Ductwork.configuration.supervisor_polling_timeout)
@@ -42,7 +42,7 @@ module Ductwork
       def shutdown
         @running = false
 
-        logger.debug(msg: "Beginning shutdown", role: :supervisor)
+        Ductwork.logger.debug(msg: "Beginning shutdown", role: :supervisor)
         terminate_gracefully
         wait_for_workers_to_exit
         terminate_immediately
@@ -54,7 +54,7 @@ module Ductwork
       attr_reader :running
 
       def check_workers
-        logger.debug(msg: "Checking workers are alive", role: :supervisor)
+        Ductwork.logger.debug(msg: "Checking workers are alive", role: :supervisor)
 
         workers.each do |worker|
           if process_dead?(worker[:pid])
@@ -63,7 +63,7 @@ module Ductwork
               worker[:block].call(worker[:metadata])
             end
             worker[:pid] = new_pid
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "Restarted process (#{old_pid}) as (#{new_pid})",
               role: :supervisor,
               old_pid: old_pid,
@@ -72,12 +72,12 @@ module Ductwork
           end
         end
 
-        logger.debug(msg: "All workers are alive or revived", role: :supervisor)
+        Ductwork.logger.debug(msg: "All workers are alive or revived", role: :supervisor)
       end
 
       def terminate_gracefully
         workers.each do |worker|
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Sending TERM signal to process (#{worker[:pid]})",
             role: :supervisor,
             pid: worker[:pid],
@@ -95,7 +95,7 @@ module Ductwork
           workers.each_with_index do |worker, index|
             if ::Process.wait(worker[:pid], ::Process::WNOHANG)
               workers[index] = nil
-              logger.debug(
+              Ductwork.logger.debug(
                 msg: "Child process (#{worker[:pid]}) stopped successfully",
                 role: :supervisor,
                 pid: worker[:pid]
@@ -108,7 +108,7 @@ module Ductwork
 
       def terminate_immediately
         workers.each_with_index do |worker, index|
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Sending KILL signal to process (#{worker[:pid]})",
             role: :supervisor,
             pid: worker[:pid],
@@ -117,7 +117,7 @@ module Ductwork
           ::Process.kill(:KILL, worker[:pid])
           ::Process.wait(worker[:pid])
           workers[index] = nil
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Child process (#{worker[:pid]}) killed after timeout",
             role: :supervisor,
             pid: worker[:pid]
@@ -151,10 +151,6 @@ module Ductwork
       def now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
-
-      def logger
-        Ductwork.configuration.logger
-      end
     end
   end
 end

data/lib/ductwork/processes/supervisor_runner.rb

@@ -6,10 +6,9 @@ module Ductwork
       def self.start!
         supervisor = Ductwork::Processes::Supervisor.new
         pipelines_to_advance = Ductwork.configuration.pipelines
-        logger = Ductwork.configuration.logger
 
         supervisor.add_worker(metadata: { pipelines: pipelines_to_advance }) do
-          logger.debug(
+          Ductwork.logger.debug(
             msg: "Starting Pipeline Advancer process",
             role: :supervisor_runner
           )
@@ -19,7 +18,7 @@ module Ductwork
 
         pipelines_to_advance.each do |pipeline|
           supervisor.add_worker(metadata: { pipeline: }) do
-            logger.debug(
+            Ductwork.logger.debug(
               msg: "Starting Job Worker Runner process",
               role: :supervisor_runner,
               pipeline: pipeline

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.2.1"
+  VERSION = "0.3.1"
 end

data/lib/ductwork.rb

@@ -19,7 +19,7 @@ loader.setup
 
 module Ductwork
   class << self
-    attr_accessor :app_executor, :configuration
+    attr_accessor :app_executor, :configuration, :logger
     attr_writer :defined_pipelines, :hooks
 
     def wrap_with_app_executor(&block)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.1
 platform: ruby
 authors:
 - Tyler Ewing
@@ -86,7 +86,7 @@ dependencies:
 description: Ductwork lets you build complex pipelines quickly and easily using intuitive
   Ruby tooling and a natural DSL.
 email:
-- tewing10@gmail.com
+- contact@getductwork.io
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -140,7 +140,7 @@ licenses:
 - LGPL-3.0-or-later
 metadata:
   homepage_uri: https://github.com/ductwork/ductwork
-  changelog_uri: https://github.com/ductwork/ductwork/blob/main/README.md
+  changelog_uri: https://github.com/ductwork/ductwork/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
   documentation_uri: https://docs.getductwork.io/
 rdoc_options: []