ductwork 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ab7c6e79c26352d990680a3913a26ea8bd4bc9999136b4ee1a1846c06e74523
-  data.tar.gz: 7adb0603119c80d5edc54e94de44828bba2c927edad418e5944d594ba21f734a
+  metadata.gz: 4c1f2564c8a446cc2daf77b882acfc6abab314e1a651bf1e7a6ab9c7432a77f6
+  data.tar.gz: ff33012c7f1dce6f9297c41185e97d8bffc4054ac26738c19459db50a70f6d19
 SHA512:
-  metadata.gz: 83be2952cf26ebbee28521df07b68ee29e87ade164a70a87ede7c7bcfc1ef5525097b1346757d76e34d4fc25535c5489f56900fa763006d923bd8491dcba520f
-  data.tar.gz: 53cabb6cdfa4f96f847ddc68818a3f0a87d59ca443282700eca5f690a8a01805e46dbd925990565630bb5c45dc728183de251629488d055f8062d136f06dff7e
+  metadata.gz: cdc5618e70f3b42f758ab6de842ff46732ee34435e82b2b04777aad7e203f0457fc00ce496fa23f2b64e232ecc81548bbff669fddfbe3cdb5c404b3d0036beea
+  data.tar.gz: e3697a5c616527013580003f6e0f131c4ddfabf871b01dc2f910527febdb19e2ead1e5f3f244a001242ff0876fe331f84eca116c24111b3fdeffbc05f7205289

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Ductwork Changelog
 
+## [0.3.0]
+
+- fix: correctly create collapsing and combining steps and jobs for complex pipelines
+- fix: add a new step and job for each active branch in a running pipeline
+- fix: add a new node and edge for each active branch of the definition
+- feat: add info-level logging for job events
+- feat: add info-level logging for pipeline events
+
 ## [0.2.1]
 
 - fix: do not splat arguments when executing a job nor triggering a pipeline

data/app/models/ductwork/job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Ductwork
-  class Job < Ductwork::Record
+  class Job < Ductwork::Record # rubocop:todo Metrics/ClassLength
     belongs_to :step, class_name: "Ductwork::Step"
     has_many :executions, class_name: "Ductwork::Execution", foreign_key: "job_id", dependent: :destroy
 
@@ -61,13 +61,13 @@ module Ductwork
     end
 
     def self.enqueue(step, args)
-      Ductwork::Record.transaction do
-        job = step.create_job!(
+      job = Ductwork::Record.transaction do
+        j = step.create_job!(
           klass: step.klass,
           started_at: Time.current,
           input_args: JSON.dump({ args: })
         )
-        execution = job.executions.create!(
+        execution = j.executions.create!(
           started_at: Time.current,
           retry_count: 0
         )
@@ -75,8 +75,16 @@ module Ductwork
           started_at: Time.current
         )
 
-        job
+        j
       end
+
+      Ductwork.configuration.logger.info(
+        msg: "Job enqueued",
+        job_id: job.id,
+        job_klass: job.klass
+      )
+
+      job
     end
 
     def execute(pipeline)
@@ -103,12 +111,13 @@ module Ductwork
         execution_failed!(execution, run, e)
         result = "failure"
       ensure
-        logger.debug(
-          msg: "Executed job",
-          role: :job_worker,
+        logger.info(
+          msg: "Job executed",
           pipeline: pipeline,
+          job_id: id,
           job_klass: klass,
-          result: result
+          result: result,
+          role: :job_worker
         )
       end
     end
@@ -167,6 +176,16 @@ module Ductwork
         end
       end
 
+      logger.warn(
+        msg: "Job errored",
+        error_klass: error.class.name,
+        error_message: error.message,
+        job_id: id,
+        job_klass: klass,
+        pipeline_id: pipeline.id,
+        role: :job_worker
+      )
+
       # NOTE: perform lifecycle hook execution outside of the transaction as
       # to not unnecessarily hold it open
       if halted

data/app/models/ductwork/pipeline.rb

@@ -56,8 +56,8 @@ module Ductwork
         step_klass = pipeline_definition.dig(:nodes, 0)
         definition = JSON.dump(pipeline_definition)
 
-        Record.transaction do
-          pipeline = create!(
+        pipeline = Record.transaction do
+          p = create!(
             klass: name.to_s,
             status: :in_progress,
             definition: definition,
@@ -65,7 +65,7 @@ module Ductwork
             triggered_at: Time.current,
             last_advanced_at: Time.current
           )
-          step = pipeline.steps.create!(
+          step = p.steps.create!(
             klass: step_klass,
             status: :in_progress,
             step_type: :start,
@@ -73,24 +73,36 @@ module Ductwork
           )
           Ductwork::Job.enqueue(step, args)
 
-          pipeline
+          p
         end
+
+        Ductwork.configuration.logger.info(
+          msg: "Pipeline triggered",
+          pipeline_id: pipeline.id,
+          role: :application
+        )
+
+        pipeline
       end
     end
 
     def advance!
-      step = steps.advancing.take
-      edge = if step.present?
-               parsed_definition.dig(:edges, step.klass, 0)
-             end
+      # NOTE: there could be A LOT of steps advancing for a single pipeline
+      # so instead of loading everything into memory and using ruby collection
+      # methods we make multiple queries. may need to revisist this once
+      # we do extensive load testing
+      advancing = steps.advancing
+      edges = if advancing.exists?
+                parsed_definition
+                  .fetch(:edges, {})
+                  .select { |k| k.in?(advancing.pluck(:klass)) }
+              end
 
       Ductwork::Record.transaction do
-        steps.advancing.update!(status: :completed, completed_at: Time.current)
-
-        if edge.nil?
-          conditionally_complete_pipeline
+        if edges.nil? || edges.values.all?(&:empty?)
+          conditionally_complete_pipeline(advancing)
         else
-          advance_to_next_step_by_type(edge, step)
+          advance_to_next_steps_by_type(edges, advancing)
         end
       end
     end
@@ -108,34 +120,64 @@ module Ductwork
       @parsed_definition ||= JSON.parse(definition).with_indifferent_access
     end
 
-    def conditionally_complete_pipeline
+    def conditionally_complete_pipeline(advancing)
+      advancing.update!(status: :completed, completed_at: Time.current)
+
       if steps.where(status: %w[in_progress pending]).none?
         update!(status: :completed, completed_at: Time.current)
+
+        Ductwork.configuration.logger.info(
+          msg: "Pipeline completed",
+          pipeline_id: id,
+          role: :pipeline_advancer
+        )
       end
     end
 
-    def advance_to_next_step_by_type(edge, step)
-      # NOTE: "chain" is used by ActiveRecord so we have to call
-      # this enum value "default" :sad:
-      step_type = edge[:type] == "chain" ? "default" : edge[:type]
-
-      if step_type.in?(%w[default divide])
-        advance_to_next_steps(step_type, step, edge)
-      elsif step_type == "combine"
-        combine_next_steps(step_type, edge)
-      elsif step_type == "expand"
-        expand_to_next_steps(step_type, step, edge)
-      elsif step_type == "collapse"
-        collapse_next_steps(step_type, step, edge)
+    def advance_to_next_steps_by_type(edges, advancing)
+      if edges.all? { |_, v| v.dig(-1, :type) == "combine" }
+        conditionally_combine_next_steps(edges, advancing)
       else
-        Ductwork.configuration.logger.error(
-          msg: "Invalid step type",
-          role: :pipeline_advancer
-        )
+        edges.each do |step_klass, step_edges|
+          edge = step_edges[-1]
+          # NOTE: "chain" is used by ActiveRecord so we have to call
+          # this enum value "default" :sad:
+          step_type = edge[:type] == "chain" ? "default" : edge[:type]
+
+          if step_type == "collapse"
+            conditionally_collapse_next_steps(step_klass, edge, advancing)
+          else
+            advance_non_merging_steps(step_klass, edges, advancing)
+          end
+        end
       end
+      advancing.update!(status: :completed, completed_at: Time.current)
+      log_pipeline_advanced(edges)
     end
 
-    def advance_to_next_steps(step_type, step, edge)
+    def advance_non_merging_steps(step_klass, edges, advancing)
+      advancing.where(klass: step_klass).find_each do |step|
+        edge = edges.dig(step.klass, -1)
+        # NOTE: "chain" is used by ActiveRecord so we have to call
+        # this enum value "default" :sad:
+        step_type = edge[:type] == "chain" ? "default" : edge[:type]
+
+        if step_type.in?(%w[default divide])
+          advance_to_next_steps(step_type, advancing, edge)
+        elsif step_type == "expand"
+          expand_to_next_steps(step_type, advancing, edge)
+        else
+          Ductwork.configuration.logger.error(
+            msg: "Invalid step type",
+            step_type: step_type,
+            pipeline_id: id,
+            role: :pipeline_advancer
+          )
+        end
+      end
+    end
+
+    def advance_to_next_steps(step_type, advancing, edge)
       edge[:to].each do |to_klass|
         next_step = steps.create!(
           klass: to_klass,
@@ -143,53 +185,82 @@ module Ductwork
           step_type: step_type,
           started_at: Time.current
         )
-        Ductwork::Job.enqueue(next_step, step.job.return_value)
+        Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
       end
     end
 
-    def combine_next_steps(step_type, edge)
-      previous_klasses = parsed_definition[:edges].select do |_, v|
-        v.dig(0, :to, 0) == edge[:to].sole && v.dig(0, :type) == "combine"
-      end.keys
-
-      if steps.not_completed.where(klass: previous_klasses).none?
-        input_arg = Job.where(
-          step: steps.completed.where(klass: previous_klasses)
-        ).map(&:return_value)
-        create_step_and_enqueue_job(
-          klass: edge[:to].sole,
-          step_type: step_type,
-          input_arg: input_arg
+    def conditionally_combine_next_steps(edges, advancing)
+      if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
+        combine_next_steps(edges, advancing)
+      else
+        Ductwork.configuration.logger.debug(
+          msg: "Not all divided steps have completed; not combining",
+          pipeline_id: id,
+          role: :pipeline_advancer
         )
       end
     end
 
-    def expand_to_next_steps(step_type, step, edge)
-      step.job.return_value.each do |input_arg|
-        create_step_and_enqueue_job(
-          klass: edge[:to].sole,
-          step_type: step_type,
-          input_arg: input_arg
-        )
+    def combine_next_steps(edges, advancing)
+      klass = edges.values.sample.dig(-1, :to).sole
+      step_type = "combine"
+      groups = advancing
+               .group(:klass)
+               .count
+               .keys
+               .map { |k| advancing.where(klass: k) }
+
+      groups.first.zip(*groups[1..]).each do |group|
+        input_arg = Ductwork::Job
+                    .where(step_id: group.map(&:id))
+                    .map(&:return_value)
+        create_step_and_enqueue_job(klass:, step_type:, input_arg:)
       end
     end
 
-    def collapse_next_steps(step_type, step, edge)
-      if steps.not_completed.where(klass: step.klass).none?
-        input_arg = Job.where(
-          step: steps.completed.where(klass: step.klass)
-        ).map(&:return_value)
+    def expand_to_next_steps(step_type, advancing, edge)
+      Array(advancing.take.job.return_value).each do |input_arg|
         create_step_and_enqueue_job(
           klass: edge[:to].sole,
           step_type: step_type,
           input_arg: input_arg
         )
+      end
+    end
+
+    def conditionally_collapse_next_steps(step_klass, edge, advancing)
+      if steps.where(status: %w[pending in_progress], klass: step_klass).none?
+        collapse_next_steps(edge[:to].sole, advancing)
       else
         Ductwork.configuration.logger.debug(
-          msg: "Not all expanded steps have completed",
+          msg: "Not all expanded steps have completed; not collapsing",
+          pipeline_id: id,
           role: :pipeline_advancer
         )
       end
     end
+
+    def collapse_next_steps(klass, advancing)
+      step_type = "collapse"
+      input_arg = []
+
+      # NOTE: because of expanding based on return values, there
+      # could be A LOT of jobs so we want to use batch methods
+      # to avoid creating too many in-memory objects
+      Ductwork::Job.where(step_id: advancing.ids).find_each do |job|
+        input_arg << job.return_value
+      end
+
+      create_step_and_enqueue_job(klass:, step_type:, input_arg:)
+    end
+
+    def log_pipeline_advanced(edges)
+      Ductwork.configuration.logger.info(
+        msg: "Pipeline advanced",
+        pipeline_id: id,
+        transitions: edges.map { |_, v| v.dig(-1, :type) },
+        role: :pipeline_advancer
+      )
+    end
   end
 end

data/lib/ductwork/dsl/definition_builder.rb

@@ -14,6 +14,7 @@ module Ductwork
         }
         @divisions = 0
         @expansions = 0
+        @last_nodes = []
       end
 
       def start(klass)
@@ -29,7 +30,7 @@ module Ductwork
       def chain(klass)
         validate_classes!(klass)
         validate_definition_started!(action: "chaining")
-        add_edge_to_last_node(klass, type: :chain)
+        add_edge_to_last_nodes(klass, type: :chain)
         add_new_nodes(klass)
 
         self
@@ -38,7 +39,7 @@ module Ductwork
       def divide(to:)
         validate_classes!(to)
         validate_definition_started!(action: "dividing chain")
-        add_edge_to_last_node(*to, type: :divide)
+        add_edge_to_last_nodes(*to, type: :divide)
         add_new_nodes(*to)
 
         @divisions += 1
@@ -79,7 +80,7 @@ module Ductwork
       def expand(to:)
         validate_classes!(to)
         validate_definition_started!(action: "expanding chain")
-        add_edge_to_last_node(to, type: :expand)
+        add_edge_to_last_nodes(to, type: :expand)
         add_new_nodes(to)
 
         @expansions += 1
@@ -91,7 +92,7 @@ module Ductwork
         validate_classes!(into)
         validate_definition_started!(action: "collapsing steps")
         validate_definition_expanded!
-        add_edge_to_last_node(into, type: :collapse)
+        add_edge_to_last_nodes(into, type: :collapse)
         add_new_nodes(into)
 
         @expansions -= 1
@@ -117,7 +118,7 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :divisions, :expansions
+      attr_reader :definition, :divisions, :expansions, :last_nodes
 
       def validate_classes!(klasses)
         valid = Array(klasses).all? do |klass|
@@ -162,19 +163,22 @@ module Ductwork
       end
 
       def add_new_nodes(*klasses)
-        definition[:nodes].push(*klasses.map(&:name))
+        nodes = klasses.map(&:name)
+        @last_nodes = Array(nodes)
+
+        definition[:nodes].push(*nodes)
         klasses.each do |klass|
           definition[:edges][klass.name] ||= []
         end
       end
 
-      def add_edge_to_last_node(*klasses, type:)
-        last_node = definition.dig(:nodes, -1)
-
-        definition[:edges][last_node] << {
-          to: klasses.map(&:name),
-          type: type,
-        }
+      def add_edge_to_last_nodes(*klasses, type:)
+        last_nodes.each do |last_node|
+          definition[:edges][last_node] << {
+            to: klasses.map(&:name),
+            type: type,
+          }
+        end
       end
     end
   end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Tyler Ewing