ductwork 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 10cf03c4f8237368de710c2ee604a9d20cf40c2dcebfda776fe490a38ba26aad
-  data.tar.gz: fd44065015e06a493905b493b847df2de01d779f7451662b186a458cb1cc9977
+  metadata.gz: 4c1f2564c8a446cc2daf77b882acfc6abab314e1a651bf1e7a6ab9c7432a77f6
+  data.tar.gz: ff33012c7f1dce6f9297c41185e97d8bffc4054ac26738c19459db50a70f6d19
 SHA512:
-  metadata.gz: 712b5f00ad1a1a1c736dd186f04dce4a85f2ba3d57df234a5d1182d91483530f0722c9a153e1a4f8517142657ffc8f486e39e6bcd51c28b66f4df5a5958e5455
-  data.tar.gz: 18c7d56494dc4251310e73bffe47c56e80820634dce555ac2e8d2d46b65973242adc621004dada1efa94cb353063ab650bcb833cec2c7d0cffd635727b808fa1
+  metadata.gz: cdc5618e70f3b42f758ab6de842ff46732ee34435e82b2b04777aad7e203f0457fc00ce496fa23f2b64e232ecc81548bbff669fddfbe3cdb5c404b3d0036beea
+  data.tar.gz: e3697a5c616527013580003f6e0f131c4ddfabf871b01dc2f910527febdb19e2ead1e5f3f244a001242ff0876fe331f84eca116c24111b3fdeffbc05f7205289

data/CHANGELOG.md

@@ -1,10 +1,24 @@
 # Ductwork Changelog
 
+## [0.3.0]
+
+- fix: correctly create collapsing and combining steps and jobs for complex pipelines
+- fix: add a new step and job for each active branch in a running pipeline
+- fix: add a new node and edge for each active branch of the definition
+- feat: add info-level logging for job events
+- feat: add info-level logging for pipeline events
+
+## [0.2.1]
+
+- fix: do not splat arguments when executing a job nor triggering a pipeline
+- fix: do not splat arguments when enqueuing a job and fix related spec
+- fix: add missing `dependent: :destroy` on certain associations
+
 ## [0.2.0]
 
-- fix: allow steps to be chained while pipeline is expanded or divided (before collapsing or combining) - before this incorrectly raised a `CollapseError` or `CombineError`
-- feat: validate argument(s) passed to step transition DSL methods to be valid step class
 - feat: validate all pipeline definitions on rails boot
+- feat: validate argument(s) passed to step transition DSL methods to be valid step class
+- fix: allow steps to be chained while pipeline is expanded or divided (before collapsing or combining) - before this incorrectly raised a `CollapseError` or `CombineError`
 
 ## [0.1.0]
 

data/README.md

@@ -75,7 +75,7 @@ class UsersRequiringEnrichment
 
   def execute
     ids = User.where("data_last_refreshed_at < ?", @days_outdated.days.ago).ids
-    Rails.logger.info("Enriching #{ids.length} users' data")
+    Ductwork.logger.info("Enriching #{ids.length} users' data")
 
     # Return value becomes input to the next step
     ids

data/app/models/ductwork/execution.rb

@@ -3,9 +3,9 @@
 module Ductwork
   class Execution < Ductwork::Record
     belongs_to :job, class_name: "Ductwork::Job"
-    has_one :availability, class_name: "Ductwork::Availability", foreign_key: "execution_id"
-    has_one :run, class_name: "Ductwork::Run", foreign_key: "execution_id"
-    has_one :result, class_name: "Ductwork::Result", foreign_key: "execution_id"
+    has_one :availability, class_name: "Ductwork::Availability", foreign_key: "execution_id", dependent: :destroy
+    has_one :run, class_name: "Ductwork::Run", foreign_key: "execution_id", dependent: :destroy
+    has_one :result, class_name: "Ductwork::Result", foreign_key: "execution_id", dependent: :destroy
 
     validates :retry_count, presence: true
     validates :started_at, presence: true

data/app/models/ductwork/job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Ductwork
-  class Job < Ductwork::Record
+  class Job < Ductwork::Record # rubocop:todo Metrics/ClassLength
     belongs_to :step, class_name: "Ductwork::Step"
     has_many :executions, class_name: "Ductwork::Execution", foreign_key: "job_id", dependent: :destroy
 
@@ -60,14 +60,14 @@ module Ductwork
       end
     end
 
-    def self.enqueue(step, *args)
-      Ductwork::Record.transaction do
-        job = step.create_job!(
+    def self.enqueue(step, args)
+      job = Ductwork::Record.transaction do
+        j = step.create_job!(
           klass: step.klass,
           started_at: Time.current,
           input_args: JSON.dump({ args: })
         )
-        execution = job.executions.create!(
+        execution = j.executions.create!(
           started_at: Time.current,
           retry_count: 0
         )
@@ -75,8 +75,16 @@ module Ductwork
           started_at: Time.current
         )
 
-        job
+        j
       end
+
+      Ductwork.configuration.logger.info(
+        msg: "Job enqueued",
+        job_id: job.id,
+        job_klass: job.klass
+      )
+
+      job
     end
 
     def execute(pipeline)
@@ -89,7 +97,7 @@ module Ductwork
         job_klass: klass
       )
       args = JSON.parse(input_args)["args"]
-      instance = Object.const_get(klass).new(*args)
+      instance = Object.const_get(klass).new(args)
       run = execution.create_run!(
         started_at: Time.current
       )
@@ -103,12 +111,13 @@ module Ductwork
         execution_failed!(execution, run, e)
         result = "failure"
       ensure
-        logger.debug(
-          msg: "Executed job",
-          role: :job_worker,
+        logger.info(
+          msg: "Job executed",
           pipeline: pipeline,
+          job_id: id,
           job_klass: klass,
-          result: result
+          result: result,
+          role: :job_worker
         )
       end
     end
@@ -167,6 +176,16 @@ module Ductwork
         end
       end
 
+      logger.warn(
+        msg: "Job errored",
+        error_klass: error.class.name,
+        error_message: error.message,
+        job_id: id,
+        job_klass: klass,
+        pipeline_id: pipeline.id,
+        role: :job_worker
+      )
+
       # NOTE: perform lifecycle hook execution outside of the transaction as
       # to not unnecessarily hold it open
       if halted

data/app/models/ductwork/pipeline.rb

@@ -48,7 +48,7 @@ module Ductwork
         Ductwork.defined_pipelines << name.to_s
       end
 
-      def trigger(*args)
+      def trigger(args)
         if pipeline_definition.nil?
           raise DefinitionError, "Pipeline must be defined before triggering"
         end
@@ -56,8 +56,8 @@ module Ductwork
         step_klass = pipeline_definition.dig(:nodes, 0)
         definition = JSON.dump(pipeline_definition)
 
-        Record.transaction do
-          pipeline = create!(
+        pipeline = Record.transaction do
+          p = create!(
             klass: name.to_s,
             status: :in_progress,
             definition: definition,
@@ -65,32 +65,44 @@ module Ductwork
             triggered_at: Time.current,
             last_advanced_at: Time.current
           )
-          step = pipeline.steps.create!(
+          step = p.steps.create!(
             klass: step_klass,
             status: :in_progress,
             step_type: :start,
             started_at: Time.current
           )
-          Ductwork::Job.enqueue(step, *args)
+          Ductwork::Job.enqueue(step, args)
 
-          pipeline
+          p
         end
+
+        Ductwork.configuration.logger.info(
+          msg: "Pipeline triggered",
+          pipeline_id: pipeline.id,
+          role: :application
+        )
+
+        pipeline
       end
     end
 
     def advance!
-      step = steps.advancing.take
-      edge = if step.present?
-               parsed_definition.dig(:edges, step.klass, 0)
-             end
+      # NOTE: there could be A LOT of steps advancing for a single pipeline
+      # so instead of loading everything into memory and using ruby collection
+      # methods we make multiple queries. may need to revisist this once
+      # we do extensive load testing
+      advancing = steps.advancing
+      edges = if advancing.exists?
+                parsed_definition
+                  .fetch(:edges, {})
+                  .select { |k| k.in?(advancing.pluck(:klass)) }
+              end
 
       Ductwork::Record.transaction do
-        steps.advancing.update!(status: :completed, completed_at: Time.current)
-
-        if edge.nil?
-          conditionally_complete_pipeline
+        if edges.nil? || edges.values.all?(&:empty?)
+          conditionally_complete_pipeline(advancing)
         else
-          advance_to_next_step_by_type(edge, step)
+          advance_to_next_steps_by_type(edges, advancing)
         end
       end
     end
@@ -108,34 +120,64 @@ module Ductwork
       @parsed_definition ||= JSON.parse(definition).with_indifferent_access
     end
 
-    def conditionally_complete_pipeline
+    def conditionally_complete_pipeline(advancing)
+      advancing.update!(status: :completed, completed_at: Time.current)
+
       if steps.where(status: %w[in_progress pending]).none?
         update!(status: :completed, completed_at: Time.current)
+
+        Ductwork.configuration.logger.info(
+          msg: "Pipeline completed",
+          pipeline_id: id,
+          role: :pipeline_advancer
+        )
       end
     end
 
-    def advance_to_next_step_by_type(edge, step)
-      # NOTE: "chain" is used by ActiveRecord so we have to call
-      # this enum value "default" :sad:
-      step_type = edge[:type] == "chain" ? "default" : edge[:type]
-
-      if step_type.in?(%w[default divide])
-        advance_to_next_steps(step_type, step, edge)
-      elsif step_type == "combine"
-        combine_next_steps(step_type, edge)
-      elsif step_type == "expand"
-        expand_to_next_steps(step_type, step, edge)
-      elsif step_type == "collapse"
-        collapse_next_steps(step_type, step, edge)
+    def advance_to_next_steps_by_type(edges, advancing)
+      if edges.all? { |_, v| v.dig(-1, :type) == "combine" }
+        conditionally_combine_next_steps(edges, advancing)
       else
-        Ductwork.configuration.logger.error(
-          msg: "Invalid step type",
-          role: :pipeline_advancer
-        )
+        edges.each do |step_klass, step_edges|
+          edge = step_edges[-1]
+          # NOTE: "chain" is used by ActiveRecord so we have to call
+          # this enum value "default" :sad:
+          step_type = edge[:type] == "chain" ? "default" : edge[:type]
+
+          if step_type == "collapse"
+            conditionally_collapse_next_steps(step_klass, edge, advancing)
+          else
+            advance_non_merging_steps(step_klass, edges, advancing)
+          end
+        end
       end
+      advancing.update!(status: :completed, completed_at: Time.current)
+      log_pipeline_advanced(edges)
     end
 
-    def advance_to_next_steps(step_type, step, edge)
+    def advance_non_merging_steps(step_klass, edges, advancing)
+      advancing.where(klass: step_klass).find_each do |step|
+        edge = edges.dig(step.klass, -1)
+        # NOTE: "chain" is used by ActiveRecord so we have to call
+        # this enum value "default" :sad:
+        step_type = edge[:type] == "chain" ? "default" : edge[:type]
+
+        if step_type.in?(%w[default divide])
+          advance_to_next_steps(step_type, advancing, edge)
+        elsif step_type == "expand"
+          expand_to_next_steps(step_type, advancing, edge)
+        else
+          Ductwork.configuration.logger.error(
+            msg: "Invalid step type",
+            step_type: step_type,
+            pipeline_id: id,
+            role: :pipeline_advancer
+          )
+        end
+      end
+    end
+
+    def advance_to_next_steps(step_type, advancing, edge)
       edge[:to].each do |to_klass|
         next_step = steps.create!(
           klass: to_klass,
@@ -143,53 +185,82 @@ module Ductwork
           step_type: step_type,
           started_at: Time.current
         )
-        Ductwork::Job.enqueue(next_step, step.job.return_value)
+        Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
       end
     end
 
-    def combine_next_steps(step_type, edge)
-      previous_klasses = parsed_definition[:edges].select do |_, v|
-        v.dig(0, :to, 0) == edge[:to].sole && v.dig(0, :type) == "combine"
-      end.keys
-
-      if steps.not_completed.where(klass: previous_klasses).none?
-        input_arg = Job.where(
-          step: steps.completed.where(klass: previous_klasses)
-        ).map(&:return_value)
-        create_step_and_enqueue_job(
-          klass: edge[:to].sole,
-          step_type: step_type,
-          input_arg: input_arg
+    def conditionally_combine_next_steps(edges, advancing)
+      if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
+        combine_next_steps(edges, advancing)
+      else
+        Ductwork.configuration.logger.debug(
+          msg: "Not all divided steps have completed; not combining",
+          pipeline_id: id,
+          role: :pipeline_advancer
         )
       end
     end
 
-    def expand_to_next_steps(step_type, step, edge)
-      step.job.return_value.each do |input_arg|
-        create_step_and_enqueue_job(
-          klass: edge[:to].sole,
-          step_type: step_type,
-          input_arg: input_arg
-        )
+    def combine_next_steps(edges, advancing)
+      klass = edges.values.sample.dig(-1, :to).sole
+      step_type = "combine"
+      groups = advancing
+               .group(:klass)
+               .count
+               .keys
+               .map { |k| advancing.where(klass: k) }
+
+      groups.first.zip(*groups[1..]).each do |group|
+        input_arg = Ductwork::Job
+                    .where(step_id: group.map(&:id))
+                    .map(&:return_value)
+        create_step_and_enqueue_job(klass:, step_type:, input_arg:)
       end
     end
 
-    def collapse_next_steps(step_type, step, edge)
-      if steps.not_completed.where(klass: step.klass).none?
-        input_arg = Job.where(
-          step: steps.completed.where(klass: step.klass)
-        ).map(&:return_value)
+    def expand_to_next_steps(step_type, advancing, edge)
+      Array(advancing.take.job.return_value).each do |input_arg|
         create_step_and_enqueue_job(
           klass: edge[:to].sole,
           step_type: step_type,
           input_arg: input_arg
         )
+      end
+    end
+
+    def conditionally_collapse_next_steps(step_klass, edge, advancing)
+      if steps.where(status: %w[pending in_progress], klass: step_klass).none?
+        collapse_next_steps(edge[:to].sole, advancing)
       else
         Ductwork.configuration.logger.debug(
-          msg: "Not all expanded steps have completed",
+          msg: "Not all expanded steps have completed; not collapsing",
+          pipeline_id: id,
           role: :pipeline_advancer
         )
       end
     end
+
+    def collapse_next_steps(klass, advancing)
+      step_type = "collapse"
+      input_arg = []
+
+      # NOTE: because of expanding based on return values, there
+      # could be A LOT of jobs so we want to use batch methods
+      # to avoid creating too many in-memory objects
+      Ductwork::Job.where(step_id: advancing.ids).find_each do |job|
+        input_arg << job.return_value
+      end
+
+      create_step_and_enqueue_job(klass:, step_type:, input_arg:)
+    end
+
+    def log_pipeline_advanced(edges)
+      Ductwork.configuration.logger.info(
+        msg: "Pipeline advanced",
+        pipeline_id: id,
+        transitions: edges.map { |_, v| v.dig(-1, :type) },
+        role: :pipeline_advancer
+      )
+    end
   end
 end

data/app/models/ductwork/step.rb

@@ -3,7 +3,7 @@
 module Ductwork
   class Step < Ductwork::Record
     belongs_to :pipeline, class_name: "Ductwork::Pipeline"
-    has_one :job, class_name: "Ductwork::Job", foreign_key: "step_id"
+    has_one :job, class_name: "Ductwork::Job", foreign_key: "step_id", dependent: :destroy
 
     validates :klass, presence: true
     validates :status, presence: true

data/lib/ductwork/dsl/definition_builder.rb

@@ -14,6 +14,7 @@ module Ductwork
         }
         @divisions = 0
         @expansions = 0
+        @last_nodes = []
       end
 
       def start(klass)
@@ -29,7 +30,7 @@ module Ductwork
       def chain(klass)
         validate_classes!(klass)
         validate_definition_started!(action: "chaining")
-        add_edge_to_last_node(klass, type: :chain)
+        add_edge_to_last_nodes(klass, type: :chain)
         add_new_nodes(klass)
 
         self
@@ -38,7 +39,7 @@ module Ductwork
       def divide(to:)
         validate_classes!(to)
         validate_definition_started!(action: "dividing chain")
-        add_edge_to_last_node(*to, type: :divide)
+        add_edge_to_last_nodes(*to, type: :divide)
         add_new_nodes(*to)
 
         @divisions += 1
@@ -79,7 +80,7 @@ module Ductwork
       def expand(to:)
         validate_classes!(to)
         validate_definition_started!(action: "expanding chain")
-        add_edge_to_last_node(to, type: :expand)
+        add_edge_to_last_nodes(to, type: :expand)
         add_new_nodes(to)
 
         @expansions += 1
@@ -91,7 +92,7 @@ module Ductwork
         validate_classes!(into)
         validate_definition_started!(action: "collapsing steps")
         validate_definition_expanded!
-        add_edge_to_last_node(into, type: :collapse)
+        add_edge_to_last_nodes(into, type: :collapse)
         add_new_nodes(into)
 
         @expansions -= 1
@@ -117,7 +118,7 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :divisions, :expansions
+      attr_reader :definition, :divisions, :expansions, :last_nodes
 
       def validate_classes!(klasses)
         valid = Array(klasses).all? do |klass|
@@ -162,19 +163,22 @@ module Ductwork
       end
 
       def add_new_nodes(*klasses)
-        definition[:nodes].push(*klasses.map(&:name))
+        nodes = klasses.map(&:name)
+        @last_nodes = Array(nodes)
+
+        definition[:nodes].push(*nodes)
         klasses.each do |klass|
           definition[:edges][klass.name] ||= []
         end
       end
 
-      def add_edge_to_last_node(*klasses, type:)
-        last_node = definition.dig(:nodes, -1)
-
-        definition[:edges][last_node] << {
-          to: klasses.map(&:name),
-          type: type,
-        }
+      def add_edge_to_last_nodes(*klasses, type:)
+        last_nodes.each do |last_node|
+          definition[:edges][last_node] << {
+            to: klasses.map(&:name),
+            type: type,
+          }
+        end
       end
     end
   end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Tyler Ewing