ductwork 0.9.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9edf0e9d07be90e39d099d4f2f9a65677b4b37144c8fa5c7d3d0a88ab81ed1a3
-  data.tar.gz: 91b2177670ee8333062ed6182512f4c5b2cd3ecfc3db9c36a77fcdf54397f65a
+  metadata.gz: 816c6bd3f6f983b777f6826e0bd9365be8b5fae5dc3512c2249d1f728208f5d2
+  data.tar.gz: 9deca63fadc27613ba93ef45d68ac1e8629b6f2b06bfb931842d59116f8d002c
 SHA512:
-  metadata.gz: 7583ac23a38875a63ac31b333348ddc9ec4e35b431ec90b335eb2580a241c508acbfcfdc462957a2ce39f5c7cb6094d146f60b8b4e49b753bfc0ead13967bcc4
-  data.tar.gz: 466d021eb4635d0d7ba284569100905069598f5c58edbdc3480e9db3d2de2ba09485b72935280598ac8223374cefebfa16b5d3c60a17709df9d2455a8c1273c5
+  metadata.gz: 746d2583354152bd51dc2acdd1958ec23234d27abd2b4ded448b69e82c164ae8f7c24ae1c91c536d9f8b60a72af826aa8dad0ff5c9b9301b34e6eeae14bf758c
+  data.tar.gz: a1114da01143e85ac0b58e5127f1c60cda5a625067e2c36389eee3ee61f30364aad7bf6dd4995ce86c38c05fe0a44a176a1bdd23f1943f5503ea9c6c80d3b0bb

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Ductwork Changelog
 
+## [0.9.1]
+
+- fix!: assign unique node names (klass name and stage) in pipeline definition DAG - this is a BREAKING CHANGE because the pipeline definition structure has changed
+
 ## [0.9.0]
 
 - feat: add health check to job worker runner process - this is a basic check if a thread is healthy via `Thread#alive?` and restarts the thread if it is dead

data/lib/ductwork/dsl/branch_builder.rb

@@ -7,36 +7,37 @@ module Ductwork
 
       attr_reader :last_node
 
-      def initialize(klass:, definition:)
-        @last_node = klass.name
+      def initialize(klass:, definition:, stages:)
+        @last_node = "#{klass.name}.#{stages.length - 1}"
         @definition = definition
+        @stages = stages
         @expansions = 0
       end
 
       def chain(next_klass)
-        definition[:edges][last_node] << {
-          to: [next_klass.name],
-          type: :chain,
-        }
-
-        definition[:nodes].push(next_klass.name)
-        definition[:edges][next_klass.name] ||= []
-        @last_node = next_klass.name
+        next_klass_name = "#{next_klass.name}.#{stages.length}"
+        definition[:edges][last_node][:to] = [next_klass_name]
+        definition[:edges][last_node][:type] = :chain
+        definition[:nodes].push(next_klass_name)
+        definition[:edges][next_klass_name] ||= { klass: next_klass.name }
+        stages.push(1)
+        @last_node = next_klass_name
 
         self
       end
 
-      def divide(to:)
-        definition[:edges][last_node] << {
-          to: to.map(&:name),
-          type: :divide,
-        }
+      def divide(to:) # rubocop:todo Metrics/AbcSize
+        next_klass_names = to.map { |klass| "#{klass.name}.#{stages.length}" }
+        definition[:edges][last_node][:to] = next_klass_names
+        definition[:edges][last_node][:type] = :divide
+        definition[:nodes].push(*next_klass_names)
+        stages.push(1)
 
-        definition[:nodes].push(*to.map(&:name))
         sub_branches = to.map do |klass|
-          definition[:edges][klass.name] ||= []
+          next_klass_name = "#{klass.name}.#{stages.length - 1}"
+          definition[:edges][next_klass_name] ||= { klass: klass.name }
 
-          Ductwork::DSL::BranchBuilder.new(klass:, definition:)
+          Ductwork::DSL::BranchBuilder.new(klass:, definition:, stages:)
         end
 
         yield sub_branches
@@ -44,32 +45,30 @@ module Ductwork
         self
       end
 
-      def combine(*branch_builders, into:)
-        definition[:edges][last_node] << {
-          to: [into.name],
-          type: :combine,
-        }
+      def combine(*branch_builders, into:) # rubocop:todo Metrics/AbcSize
+        next_klass_name = "#{into.name}.#{stages.length}"
+        definition[:edges][last_node][:to] = [next_klass_name]
+        definition[:edges][last_node][:type] = :combine
+
         branch_builders.each do |branch|
-          definition[:edges][branch.last_node] << {
-            to: [into.name],
-            type: :combine,
-          }
+          definition[:edges][branch.last_node][:to] = [next_klass_name]
+          definition[:edges][branch.last_node][:type] = :combine
         end
-        definition[:nodes].push(into.name)
-        definition[:edges][into.name] ||= []
+        definition[:nodes].push(next_klass_name)
+        definition[:edges][next_klass_name] ||= { klass: into.name }
+        stages.push(1)
 
         self
       end
 
       def expand(to:)
-        definition[:edges][last_node] << {
-          to: [to.name],
-          type: :expand,
-        }
-
-        definition[:nodes].push(to.name)
-        definition[:edges][to.name] ||= []
-        @last_node = to.name
+        next_klass_name = "#{to.name}.#{stages.length}"
+        definition[:edges][last_node][:to] = [next_klass_name]
+        definition[:edges][last_node][:type] = :expand
+        definition[:nodes].push(next_klass_name)
+        definition[:edges][next_klass_name] ||= { klass: to.name }
+        stages.push(1)
+        @last_node = next_klass_name
         @expansions += 1
 
         self
@@ -81,13 +80,13 @@ module Ductwork
                 "Must expand pipeline definition before collapsing steps"
         end
 
-        definition[:edges][last_node] << {
-          to: [into.name],
-          type: :collapse,
-        }
+        next_klass_name = "#{into.name}.#{stages.length}"
+        definition[:edges][last_node][:to] = [next_klass_name]
+        definition[:edges][last_node][:type] = :collapse
 
-        definition[:nodes].push(into.name)
-        definition[:edges][into.name] ||= []
+        definition[:nodes].push(next_klass_name)
+        definition[:edges][next_klass_name] ||= { klass: into.name }
+        stages.push(1)
         @last_node = into.name
         @expansions -= 1
 
@@ -96,7 +95,7 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :expansions
+      attr_reader :definition, :expansions, :stages
     end
   end
 end

data/lib/ductwork/dsl/definition_builder.rb

@@ -15,23 +15,24 @@ module Ductwork
         }
         @divergences = []
         @last_nodes = []
+        @stages = []
       end
 
       def start(klass)
         validate_classes!(klass)
         validate_start_once!
         add_new_nodes(klass)
+        increment_position
 
         self
       end
 
-      # NOTE: there is a bug here that does not allow the user to reuse step
-      # classes in the same pipeline. i'll fix this later
       def chain(klass)
         validate_classes!(klass)
         validate_definition_started!(action: "chaining")
         add_edge_to_last_nodes(klass, type: :chain)
         add_new_nodes(klass)
+        increment_position
 
         self
       end
@@ -41,13 +42,13 @@ module Ductwork
         validate_definition_started!(action: "dividing chain")
         add_edge_to_last_nodes(*to, type: :divide)
         add_new_nodes(*to)
-
+        increment_position
         divergences.push(:divide)
 
         if block_given?
           branches = to.map do |klass|
             Ductwork::DSL::BranchBuilder
-              .new(klass:, definition:)
+              .new(klass:, definition:, stages:)
           end
 
           yield branches
@@ -64,15 +65,14 @@ module Ductwork
         divergences.pop
 
         last_nodes = definition[:nodes].reverse.select do |node|
-          definition[:edges][node].empty?
+          definition.dig(:edges, node, :to).blank?
         end
         last_nodes.each do |node|
-          definition[:edges][node] << {
-            to: [into.name],
-            type: :combine,
-          }
+          definition[:edges][node][:to] = ["#{into.name}.#{stages.length}"]
+          definition[:edges][node][:type] = :combine
         end
         add_new_nodes(into)
+        increment_position
 
         self
       end
@@ -82,7 +82,7 @@ module Ductwork
         validate_definition_started!(action: "expanding chain")
         add_edge_to_last_nodes(to, type: :expand)
         add_new_nodes(to)
-
+        increment_position
         divergences.push(:expand)
 
         self
@@ -94,7 +94,7 @@ module Ductwork
         validate_can_collapse!
         add_edge_to_last_nodes(into, type: :collapse)
         add_new_nodes(into)
-
+        increment_position
         divergences.pop
 
         self
@@ -116,7 +116,7 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :last_nodes, :divergences
+      attr_reader :definition, :last_nodes, :divergences, :stages
 
       def validate_classes!(klasses)
         valid = Array(klasses).all? do |klass|
@@ -165,23 +165,28 @@ module Ductwork
       end
 
       def add_new_nodes(*klasses)
-        nodes = klasses.map(&:name)
+        nodes = klasses.map { |klass| "#{klass.name}.#{stages.length}" }
         @last_nodes = Array(nodes)
 
         definition[:nodes].push(*nodes)
         klasses.each do |klass|
-          definition[:edges][klass.name] ||= []
+          node = "#{klass.name}.#{stages.length}"
+          definition[:edges][node] ||= { klass: klass.name }
         end
       end
 
       def add_edge_to_last_nodes(*klasses, type:)
         last_nodes.each do |last_node|
-          definition[:edges][last_node] << {
-            to: klasses.map(&:name),
-            type: type,
-          }
+          to = klasses.map { |klass| "#{klass.name}.#{stages.length}" }
+          definition[:edges][last_node][:to] = to
+          definition[:edges][last_node][:type] = type
+          definition[:edges][last_node][:klass] ||= klass
         end
       end
+
+      def increment_position
+        stages.push(1)
+      end
     end
   end
 end

data/lib/ductwork/models/pipeline.rb

@@ -56,7 +56,8 @@ module Ductwork
           raise DefinitionError, "Pipeline must be defined before triggering"
         end
 
-        step_klass = pipeline_definition.dig(:nodes, 0)
+        node = pipeline_definition.dig(:nodes, 0)
+        klass = pipeline_definition.dig(:edges, node, :klass)
         definition = JSON.dump(pipeline_definition)
 
         pipeline = Record.transaction do
@@ -70,7 +71,8 @@ module Ductwork
             last_advanced_at: Time.current
           )
           step = p.steps.create!(
-            klass: step_klass,
+            node: node,
+            klass: klass,
             status: :in_progress,
             to_transition: :start,
             started_at: Time.current
@@ -95,12 +97,12 @@ module Ductwork
       # advancing records which may cause memory issues. something to
       # watch out for here and maybe add in config to use AR relation
       # at certain counts or even memory limits.
-      advancing_steps = steps.advancing.pluck(:id, :klass)
+      advancing_steps = steps.advancing.pluck(:id, :node, :klass)
       advancing_ids = advancing_steps.map(&:first)
       edges = find_edges(advancing_steps)
 
       Ductwork::Record.transaction do
-        if edges.nil? || edges.values.all?(&:empty?)
+        if edges.nil? || edges.all? { |_, attrs| attrs[:to].blank? }
           conditionally_complete_pipeline(advancing_ids)
         else
           advance_to_next_steps_by_type(edges, advancing_ids)
@@ -114,23 +116,24 @@ module Ductwork
 
     private
 
-    def create_step_and_enqueue_job(edge:, input_arg:, klass: nil)
+    def create_step_and_enqueue_job(edge:, input_arg:, node: nil)
       status = :in_progress
       started_at = Time.current
       # NOTE: "chain" is used by ActiveRecord so we have to call
       # this enum value "default" :sad:
       to_transition = edge[:type] == "chain" ? "default" : edge[:type]
-      klass ||= edge.fetch(:to).sole
+      node ||= edge[:to].sole
+      klass = parsed_definition.dig(:edges, node, :klass)
 
-      next_step = steps.create!(klass:, status:, to_transition:, started_at:)
+      next_step = steps.create!(node:, klass:, status:, to_transition:, started_at:)
       Ductwork::Job.enqueue(next_step, input_arg)
     end
 
     def find_edges(advancing_steps)
       if advancing_steps.any?
-        klasses = advancing_steps.map(&:last)
+        nodes = advancing_steps.map(&:second)
 
-        parsed_definition.fetch(:edges, {}).select { |k| k.in?(klasses) }
+        parsed_definition.fetch(:edges, {}).select { |k| k.in?(nodes) }
       end
     end
 
@@ -158,26 +161,25 @@ module Ductwork
     def advance_to_next_steps_by_type(edges, advancing_ids)
       steps.where(id: advancing_ids).update_all(status: :completed, completed_at: Time.current)
 
-      if edges.all? { |_, v| v.dig(-1, :type) == "combine" }
+      if edges.all? { |_, attrs| attrs[:type] == "combine" }
         conditionally_combine_next_steps(edges, advancing_ids)
       else
-        edges.each do |step_klass, step_edges|
-          edge = step_edges[-1]
-
-          if edge[:type] == "collapse"
-            conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
+        edges.each do |node, attrs|
+          if attrs[:type] == "collapse"
+            conditionally_collapse_next_steps(node, attrs, advancing_ids)
           else
-            advance_non_merging_steps(step_klass, edge, advancing_ids)
+            advance_non_merging_steps(attrs, advancing_ids)
           end
         end
       end
       log_pipeline_advanced(edges)
     end
 
-    def advance_non_merging_steps(step_klass, edge, advancing_ids)
+    def advance_non_merging_steps(edge, advancing_ids)
       to_transition = edge[:type]
+      klass = edge[:klass]
 
-      steps.where(id: advancing_ids, klass: step_klass).find_each do |step|
+      steps.where(id: advancing_ids, klass: klass).find_each do |step|
         if to_transition.in?(%w[chain divide])
           advance_to_next_steps(step.id, edge)
         elsif to_transition == "expand"
@@ -205,15 +207,15 @@ module Ductwork
       if too_many
         halted!
       else
-        edge[:to].each do |klass|
+        edge[:to].each do |node|
           input_arg = Ductwork::Job.find_by(step_id:).return_value
-          create_step_and_enqueue_job(edge:, input_arg:, klass:)
+          create_step_and_enqueue_job(edge:, input_arg:, node:)
         end
       end
     end
 
     def conditionally_combine_next_steps(edges, advancing_ids)
-      if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
+      if steps.where(status: %w[pending in_progress], node: edges.keys).none?
         combine_next_steps(edges, advancing_ids)
       else
         Ductwork.logger.debug(
@@ -225,13 +227,13 @@ module Ductwork
     end
 
     def combine_next_steps(edges, advancing_ids)
-      edge = edges.values.sample[-1]
+      edge = edges.values.sample
       groups = steps
                .where(id: advancing_ids)
-               .group(:klass)
+               .group(:node)
                .count
                .keys
-               .map { |k| steps.where(id: advancing_ids).where(klass: k) }
+               .map { |node| steps.where(id: advancing_ids).where(node:) }
 
       groups.first.zip(*groups[1..]).each do |group|
         input_arg = Ductwork::Job
@@ -257,8 +259,8 @@ module Ductwork
       end
     end
 
-    def conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
-      if steps.where(status: %w[pending in_progress], klass: step_klass).none?
+    def conditionally_collapse_next_steps(node, edge, advancing_ids)
+      if steps.where(status: %w[pending in_progress], node: node).none?
         collapse_next_steps(edge, advancing_ids)
       else
         Ductwork.logger.debug(

data/lib/ductwork/models/step.rb

@@ -5,6 +5,7 @@ module Ductwork
     belongs_to :pipeline, class_name: "Ductwork::Pipeline"
     has_one :job, class_name: "Ductwork::Job", foreign_key: "step_id", dependent: :destroy
 
+    validates :node, presence: true
     validates :klass, presence: true
     validates :status, presence: true
     validates :to_transition, presence: true

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.9.0"
+  VERSION = "0.9.1"
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -4,6 +4,7 @@ class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>
   def change
     create_table :ductwork_steps do |table|
       table.belongs_to :pipeline, index: true, null: false, foreign_key: { to_table: :ductwork_pipelines }
+      table.string :node, null: false
       table.string :klass, null: false
       table.string :to_transition, null: false
       table.timestamp :started_at
@@ -14,9 +15,9 @@ class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>
       table.timestamps null: false
     end
 
-    add_index :ductwork_steps, %i[pipeline_id status klass]
-    add_index :ductwork_steps, %i[pipeline_id klass status]
-    add_index :ductwork_steps, %i[status klass]
+    add_index :ductwork_steps, %i[pipeline_id status node]
+    add_index :ductwork_steps, %i[pipeline_id node status]
+    add_index :ductwork_steps, %i[status node]
     add_index :ductwork_steps, %i[pipeline_id status]
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Tyler Ewing