ductwork 0.8.1 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6fde482283602dcf363baed46b5d8468757ddb8f6cc6fabb9b9d5e37497e10f4
-  data.tar.gz: 3aa0f1a9aa6f77b47997b64b246ed3dc7371e63554943110de8e06f98135f9d3
+  metadata.gz: 816c6bd3f6f983b777f6826e0bd9365be8b5fae5dc3512c2249d1f728208f5d2
+  data.tar.gz: 9deca63fadc27613ba93ef45d68ac1e8629b6f2b06bfb931842d59116f8d002c
 SHA512:
-  metadata.gz: bc475be1df97dada6099f8210b5694cbc4d90e5466d565a50674bac796fc252923f19a18e6beae3cf673db8d25fccf4bfb8178622d8574fd0e2b911e27dc8a0f
-  data.tar.gz: e1caa07d54fe23a2f2ea1b90809371a53bfd791eba6751bab9b65dac254c7f7935cf54f4b217adb68e084abd478192000090c91005d644188f64e2d7930264fd
+  metadata.gz: 746d2583354152bd51dc2acdd1958ec23234d27abd2b4ded448b69e82c164ae8f7c24ae1c91c536d9f8b60a72af826aa8dad0ff5c9b9301b34e6eeae14bf758c
+  data.tar.gz: a1114da01143e85ac0b58e5127f1c60cda5a625067e2c36389eee3ee61f30364aad7bf6dd4995ce86c38c05fe0a44a176a1bdd23f1943f5503ea9c6c80d3b0bb

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Ductwork Changelog
 
+## [0.9.1]
+
+- fix!: assign unique node names (klass name and stage) in pipeline definition DAG - this is a BREAKING CHANGE because the pipeline definition structure has changed
+
+## [0.9.0]
+
+- feat: add health check to job worker runner process - this is a basic check if a thread is healthy via `Thread#alive?` and restarts the thread if it is dead
+
 ## [0.8.1]
 
 - fix: properly wrap "units of work" in rails application executor in pipeline advancer

data/lib/ductwork/dsl/branch_builder.rb

@@ -7,36 +7,37 @@ module Ductwork
 
       attr_reader :last_node
 
-      def initialize(klass:, definition:)
-        @last_node = klass.name
+      def initialize(klass:, definition:, stages:)
+        @last_node = "#{klass.name}.#{stages.length - 1}"
         @definition = definition
+        @stages = stages
         @expansions = 0
       end
 
       def chain(next_klass)
-        definition[:edges][last_node] << {
-          to: [next_klass.name],
-          type: :chain,
-        }
-
-        definition[:nodes].push(next_klass.name)
-        definition[:edges][next_klass.name] ||= []
-        @last_node = next_klass.name
+        next_klass_name = "#{next_klass.name}.#{stages.length}"
+        definition[:edges][last_node][:to] = [next_klass_name]
+        definition[:edges][last_node][:type] = :chain
+        definition[:nodes].push(next_klass_name)
+        definition[:edges][next_klass_name] ||= { klass: next_klass.name }
+        stages.push(1)
+        @last_node = next_klass_name
 
         self
       end
 
-      def divide(to:)
-        definition[:edges][last_node] << {
-          to: to.map(&:name),
-          type: :divide,
-        }
+      def divide(to:) # rubocop:todo Metrics/AbcSize
+        next_klass_names = to.map { |klass| "#{klass.name}.#{stages.length}" }
+        definition[:edges][last_node][:to] = next_klass_names
+        definition[:edges][last_node][:type] = :divide
+        definition[:nodes].push(*next_klass_names)
+        stages.push(1)
 
-        definition[:nodes].push(*to.map(&:name))
         sub_branches = to.map do |klass|
-          definition[:edges][klass.name] ||= []
+          next_klass_name = "#{klass.name}.#{stages.length - 1}"
+          definition[:edges][next_klass_name] ||= { klass: klass.name }
 
-          Ductwork::DSL::BranchBuilder.new(klass:, definition:)
+          Ductwork::DSL::BranchBuilder.new(klass:, definition:, stages:)
         end
 
         yield sub_branches
@@ -44,32 +45,30 @@ module Ductwork
         self
       end
 
-      def combine(*branch_builders, into:)
-        definition[:edges][last_node] << {
-          to: [into.name],
-          type: :combine,
-        }
+      def combine(*branch_builders, into:) # rubocop:todo Metrics/AbcSize
+        next_klass_name = "#{into.name}.#{stages.length}"
+        definition[:edges][last_node][:to] = [next_klass_name]
+        definition[:edges][last_node][:type] = :combine
+
         branch_builders.each do |branch|
-          definition[:edges][branch.last_node] << {
-            to: [into.name],
-            type: :combine,
-          }
+          definition[:edges][branch.last_node][:to] = [next_klass_name]
+          definition[:edges][branch.last_node][:type] = :combine
         end
-        definition[:nodes].push(into.name)
-        definition[:edges][into.name] ||= []
+        definition[:nodes].push(next_klass_name)
+        definition[:edges][next_klass_name] ||= { klass: into.name }
+        stages.push(1)
 
         self
       end
 
       def expand(to:)
-        definition[:edges][last_node] << {
-          to: [to.name],
-          type: :expand,
-        }
-
-        definition[:nodes].push(to.name)
-        definition[:edges][to.name] ||= []
-        @last_node = to.name
+        next_klass_name = "#{to.name}.#{stages.length}"
+        definition[:edges][last_node][:to] = [next_klass_name]
+        definition[:edges][last_node][:type] = :expand
+        definition[:nodes].push(next_klass_name)
+        definition[:edges][next_klass_name] ||= { klass: to.name }
+        stages.push(1)
+        @last_node = next_klass_name
         @expansions += 1
 
         self
@@ -81,13 +80,13 @@ module Ductwork
                 "Must expand pipeline definition before collapsing steps"
         end
 
-        definition[:edges][last_node] << {
-          to: [into.name],
-          type: :collapse,
-        }
+        next_klass_name = "#{into.name}.#{stages.length}"
+        definition[:edges][last_node][:to] = [next_klass_name]
+        definition[:edges][last_node][:type] = :collapse
 
-        definition[:nodes].push(into.name)
-        definition[:edges][into.name] ||= []
+        definition[:nodes].push(next_klass_name)
+        definition[:edges][next_klass_name] ||= { klass: into.name }
+        stages.push(1)
         @last_node = into.name
         @expansions -= 1
 
@@ -96,7 +95,7 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :expansions
+      attr_reader :definition, :expansions, :stages
     end
   end
 end

data/lib/ductwork/dsl/definition_builder.rb

@@ -15,23 +15,24 @@ module Ductwork
         }
         @divergences = []
         @last_nodes = []
+        @stages = []
       end
 
       def start(klass)
         validate_classes!(klass)
         validate_start_once!
         add_new_nodes(klass)
+        increment_position
 
         self
       end
 
-      # NOTE: there is a bug here that does not allow the user to reuse step
-      # classes in the same pipeline. i'll fix this later
       def chain(klass)
         validate_classes!(klass)
         validate_definition_started!(action: "chaining")
         add_edge_to_last_nodes(klass, type: :chain)
         add_new_nodes(klass)
+        increment_position
 
         self
       end
@@ -41,13 +42,13 @@ module Ductwork
         validate_definition_started!(action: "dividing chain")
         add_edge_to_last_nodes(*to, type: :divide)
         add_new_nodes(*to)
-
+        increment_position
         divergences.push(:divide)
 
         if block_given?
           branches = to.map do |klass|
             Ductwork::DSL::BranchBuilder
-              .new(klass:, definition:)
+              .new(klass:, definition:, stages:)
           end
 
           yield branches
@@ -64,15 +65,14 @@ module Ductwork
         divergences.pop
 
         last_nodes = definition[:nodes].reverse.select do |node|
-          definition[:edges][node].empty?
+          definition.dig(:edges, node, :to).blank?
         end
         last_nodes.each do |node|
-          definition[:edges][node] << {
-            to: [into.name],
-            type: :combine,
-          }
+          definition[:edges][node][:to] = ["#{into.name}.#{stages.length}"]
+          definition[:edges][node][:type] = :combine
         end
         add_new_nodes(into)
+        increment_position
 
         self
       end
@@ -82,7 +82,7 @@ module Ductwork
         validate_definition_started!(action: "expanding chain")
         add_edge_to_last_nodes(to, type: :expand)
         add_new_nodes(to)
-
+        increment_position
         divergences.push(:expand)
 
         self
@@ -94,7 +94,7 @@ module Ductwork
         validate_can_collapse!
         add_edge_to_last_nodes(into, type: :collapse)
         add_new_nodes(into)
-
+        increment_position
         divergences.pop
 
         self
@@ -116,7 +116,7 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :last_nodes, :divergences
+      attr_reader :definition, :last_nodes, :divergences, :stages
 
       def validate_classes!(klasses)
         valid = Array(klasses).all? do |klass|
@@ -165,23 +165,28 @@ module Ductwork
       end
 
       def add_new_nodes(*klasses)
-        nodes = klasses.map(&:name)
+        nodes = klasses.map { |klass| "#{klass.name}.#{stages.length}" }
         @last_nodes = Array(nodes)
 
         definition[:nodes].push(*nodes)
         klasses.each do |klass|
-          definition[:edges][klass.name] ||= []
+          node = "#{klass.name}.#{stages.length}"
+          definition[:edges][node] ||= { klass: klass.name }
         end
       end
 
       def add_edge_to_last_nodes(*klasses, type:)
         last_nodes.each do |last_node|
-          definition[:edges][last_node] << {
-            to: klasses.map(&:name),
-            type: type,
-          }
+          to = klasses.map { |klass| "#{klass.name}.#{stages.length}" }
+          definition[:edges][last_node][:to] = to
+          definition[:edges][last_node][:type] = type
+          definition[:edges][last_node][:klass] ||= klass
         end
       end
+
+      def increment_position
+        stages.push(1)
+      end
     end
   end
 end

data/lib/ductwork/models/pipeline.rb

@@ -56,7 +56,8 @@ module Ductwork
           raise DefinitionError, "Pipeline must be defined before triggering"
         end
 
-        step_klass = pipeline_definition.dig(:nodes, 0)
+        node = pipeline_definition.dig(:nodes, 0)
+        klass = pipeline_definition.dig(:edges, node, :klass)
         definition = JSON.dump(pipeline_definition)
 
         pipeline = Record.transaction do
@@ -70,7 +71,8 @@ module Ductwork
             last_advanced_at: Time.current
           )
           step = p.steps.create!(
-            klass: step_klass,
+            node: node,
+            klass: klass,
             status: :in_progress,
             to_transition: :start,
             started_at: Time.current
@@ -95,12 +97,12 @@ module Ductwork
       # advancing records which may cause memory issues. something to
       # watch out for here and maybe add in config to use AR relation
       # at certain counts or even memory limits.
-      advancing_steps = steps.advancing.pluck(:id, :klass)
+      advancing_steps = steps.advancing.pluck(:id, :node, :klass)
       advancing_ids = advancing_steps.map(&:first)
       edges = find_edges(advancing_steps)
 
       Ductwork::Record.transaction do
-        if edges.nil? || edges.values.all?(&:empty?)
+        if edges.nil? || edges.all? { |_, attrs| attrs[:to].blank? }
           conditionally_complete_pipeline(advancing_ids)
         else
           advance_to_next_steps_by_type(edges, advancing_ids)
@@ -114,23 +116,24 @@ module Ductwork
 
     private
 
-    def create_step_and_enqueue_job(edge:, input_arg:, klass: nil)
+    def create_step_and_enqueue_job(edge:, input_arg:, node: nil)
       status = :in_progress
       started_at = Time.current
       # NOTE: "chain" is used by ActiveRecord so we have to call
       # this enum value "default" :sad:
       to_transition = edge[:type] == "chain" ? "default" : edge[:type]
-      klass ||= edge.fetch(:to).sole
+      node ||= edge[:to].sole
+      klass = parsed_definition.dig(:edges, node, :klass)
 
-      next_step = steps.create!(klass:, status:, to_transition:, started_at:)
+      next_step = steps.create!(node:, klass:, status:, to_transition:, started_at:)
       Ductwork::Job.enqueue(next_step, input_arg)
     end
 
     def find_edges(advancing_steps)
       if advancing_steps.any?
-        klasses = advancing_steps.map(&:last)
+        nodes = advancing_steps.map(&:second)
 
-        parsed_definition.fetch(:edges, {}).select { |k| k.in?(klasses) }
+        parsed_definition.fetch(:edges, {}).select { |k| k.in?(nodes) }
       end
     end
 
@@ -158,26 +161,25 @@ module Ductwork
     def advance_to_next_steps_by_type(edges, advancing_ids)
       steps.where(id: advancing_ids).update_all(status: :completed, completed_at: Time.current)
 
-      if edges.all? { |_, v| v.dig(-1, :type) == "combine" }
+      if edges.all? { |_, attrs| attrs[:type] == "combine" }
         conditionally_combine_next_steps(edges, advancing_ids)
       else
-        edges.each do |step_klass, step_edges|
-          edge = step_edges[-1]
-
-          if edge[:type] == "collapse"
-            conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
+        edges.each do |node, attrs|
+          if attrs[:type] == "collapse"
+            conditionally_collapse_next_steps(node, attrs, advancing_ids)
           else
-            advance_non_merging_steps(step_klass, edge, advancing_ids)
+            advance_non_merging_steps(attrs, advancing_ids)
           end
         end
       end
       log_pipeline_advanced(edges)
     end
 
-    def advance_non_merging_steps(step_klass, edge, advancing_ids)
+    def advance_non_merging_steps(edge, advancing_ids)
       to_transition = edge[:type]
+      klass = edge[:klass]
 
-      steps.where(id: advancing_ids, klass: step_klass).find_each do |step|
+      steps.where(id: advancing_ids, klass: klass).find_each do |step|
         if to_transition.in?(%w[chain divide])
           advance_to_next_steps(step.id, edge)
         elsif to_transition == "expand"
@@ -205,15 +207,15 @@ module Ductwork
       if too_many
         halted!
       else
-        edge[:to].each do |klass|
+        edge[:to].each do |node|
           input_arg = Ductwork::Job.find_by(step_id:).return_value
-          create_step_and_enqueue_job(edge:, input_arg:, klass:)
+          create_step_and_enqueue_job(edge:, input_arg:, node:)
         end
       end
     end
 
     def conditionally_combine_next_steps(edges, advancing_ids)
-      if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
+      if steps.where(status: %w[pending in_progress], node: edges.keys).none?
         combine_next_steps(edges, advancing_ids)
       else
         Ductwork.logger.debug(
@@ -225,13 +227,13 @@ module Ductwork
     end
 
     def combine_next_steps(edges, advancing_ids)
-      edge = edges.values.sample[-1]
+      edge = edges.values.sample
       groups = steps
                .where(id: advancing_ids)
-               .group(:klass)
+               .group(:node)
                .count
                .keys
-               .map { |k| steps.where(id: advancing_ids).where(klass: k) }
+               .map { |node| steps.where(id: advancing_ids).where(node:) }
 
       groups.first.zip(*groups[1..]).each do |group|
         input_arg = Ductwork::Job
@@ -257,8 +259,8 @@ module Ductwork
       end
     end
 
-    def conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
-      if steps.where(status: %w[pending in_progress], klass: step_klass).none?
+    def conditionally_collapse_next_steps(node, edge, advancing_ids)
+      if steps.where(status: %w[pending in_progress], node: node).none?
         collapse_next_steps(edge, advancing_ids)
       else
         Ductwork.logger.debug(

data/lib/ductwork/models/step.rb

@@ -5,6 +5,7 @@ module Ductwork
     belongs_to :pipeline, class_name: "Ductwork::Pipeline"
     has_one :job, class_name: "Ductwork::Job", foreign_key: "step_id", dependent: :destroy
 
+    validates :node, presence: true
     validates :klass, presence: true
     validates :status, presence: true
     validates :to_transition, presence: true

data/lib/ductwork/processes/job_worker.rb

@@ -3,18 +3,44 @@
 module Ductwork
   module Processes
     class JobWorker
-      def initialize(pipeline, running_context)
+      attr_reader :thread, :last_hearthbeat_at
+
+      def initialize(pipeline, id)
         @pipeline = pipeline
-        @running_context = running_context
+        @id = id
+        @running_context = Ductwork::RunningContext.new
+        @thread = nil
+        @last_hearthbeat_at = Time.current
+      end
+
+      def start
+        @thread = Thread.new { work_loop }
+        @thread.name = "ductwork.job_worker.#{id}"
+      end
+
+      alias restart start
+
+      def alive?
+        thread&.alive? || false
+      end
+
+      def stop
+        running_context.shutdown!
       end
 
-      def run
+      private
+
+      attr_reader :pipeline, :id, :running_context
+
+      def work_loop
         run_hooks_for(:start)
+
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker,
           pipeline: pipeline
         )
+
         while running_context.running?
           Ductwork.logger.debug(
             msg: "Attempting to claim job",
@@ -37,21 +63,16 @@ module Ductwork
             )
             sleep(polling_timeout)
           end
-        end
-
-        shutdown
-      end
-
-      private
 
-      attr_reader :pipeline, :running_context
+          @last_hearthbeat_at = Time.current
+        end
 
-      def shutdown
         Ductwork.logger.debug(
           msg: "Shutting down",
           role: :job_worker,
           pipeline: pipeline
         )
+
         run_hooks_for(:stop)
       end
 

data/lib/ductwork/processes/job_worker_runner.rb

@@ -6,7 +6,7 @@ module Ductwork
       def initialize(pipeline)
         @pipeline = pipeline
         @running_context = Ductwork::RunningContext.new
-        @threads = create_threads
+        @job_workers = []
 
         Signal.trap(:INT) { running_context.shutdown! }
         Signal.trap(:TERM) { running_context.shutdown! }
@@ -24,7 +24,9 @@ module Ductwork
       end
 
       def run
-        create_process!
+        create_process_record!
+        start_job_workers
+
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker_runner,
@@ -34,7 +36,7 @@ module Ductwork
         while running?
           # TODO: Increase or make configurable
           sleep(5)
-          attempt_synchronize_threads
+          check_thread_health
           report_heartbeat!
         end
 
@@ -43,43 +45,29 @@ module Ductwork
 
       private
 
-      attr_reader :pipeline, :running_context, :threads
+      attr_reader :pipeline, :running_context, :job_workers
 
-      def worker_count
-        Ductwork.configuration.job_worker_count(pipeline)
+      def create_process_record!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.create!(
+            pid: ::Process.pid,
+            machine_identifier: Ductwork::MachineIdentifier.fetch,
+            last_heartbeat_at: Time.current
+          )
+        end
       end
 
-      def create_threads
-        worker_count.times.map do |i|
-          Ductwork.logger.debug(
-            msg: "Creating new thread",
-            role: :job_worker_runner,
-            pipeline: pipeline
-          )
-          thread = Thread.new do
-            Ductwork::Processes::JobWorker
-              .new(pipeline, running_context)
-              .run
-          end
-          thread.name = "ductwork.job_worker.#{i}"
+      def start_job_workers
+        Ductwork.configuration.job_worker_count(pipeline).times do |i|
+          job_worker = Ductwork::Processes::JobWorker.new(pipeline, i)
+          job_workers.push(job_worker)
+          job_worker.start
 
           Ductwork.logger.debug(
-            msg: "Created new thread",
+            msg: "Created new job worker",
             role: :job_worker_runner,
             pipeline: pipeline
           )
-
-          thread
-        end
-      end
-
-      def create_process!
-        Ductwork.wrap_with_app_executor do
-          Ductwork::Process.create!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch,
-            last_heartbeat_at: Time.current
-          )
         end
       end
 
@@ -87,13 +75,17 @@ module Ductwork
         running_context.running?
       end
 
-      def attempt_synchronize_threads
+      def check_thread_health
         Ductwork.logger.debug(
           msg: "Attempting to synchronize threads",
           role: :job_worker_runner,
           pipeline: pipeline
         )
-        threads.each { |thread| thread.join(0.1) }
+        job_workers.each do |job_worker|
+          if !job_worker.alive?
+            job_worker.restart
+          end
+        end
         Ductwork.logger.debug(
           msg: "Synchronizing threads timed out",
           role: :job_worker_runner,
@@ -111,31 +103,36 @@ module Ductwork
 
       def shutdown!
         running_context.shutdown!
+        job_workers.each(&:stop)
         await_threads_graceful_shutdown
-        kill_remaining_threads
-        delete_process
+        kill_remaining_job_workers
+        delete_process_record!
       end
 
       def await_threads_graceful_shutdown
         timeout = Ductwork.configuration.job_worker_shutdown_timeout
         deadline = Time.current + timeout
 
-        Ductwork.logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
-        while Time.current < deadline && threads.any?(&:alive?)
-          threads.each do |thread|
+        Ductwork.logger.debug(
+          msg: "Attempting graceful shutdown",
+          role: :job_worker_runner
+        )
+
+        while Time.current < deadline && job_workers.any?(&:alive?)
+          job_workers.each do |job_worker|
             break if Time.current < deadline
 
             # TODO: Maybe make this configurable. If there's a ton of workers
             # it may not even get to the "later" ones depending on the timeout
-            thread.join(1)
+            job_worker.thread.join(1)
           end
         end
       end
 
-      def kill_remaining_threads
-        threads.each do |thread|
-          if thread.alive?
-            thread.kill
+      def kill_remaining_job_workers
+        job_workers.each do |job_worker|
+          if job_worker.alive?
+            job_worker.thread.kill
             Ductwork.logger.debug(
               msg: "Killed thread",
               role: :job_worker_runner,
@@ -145,7 +142,7 @@ module Ductwork
         end
       end
 
-      def delete_process
+      def delete_process_record!
         Ductwork.wrap_with_app_executor do
           Ductwork::Process.find_by!(
             pid: ::Process.pid,

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.8.1"
+  VERSION = "0.9.1"
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -4,6 +4,7 @@ class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>
   def change
     create_table :ductwork_steps do |table|
       table.belongs_to :pipeline, index: true, null: false, foreign_key: { to_table: :ductwork_pipelines }
+      table.string :node, null: false
       table.string :klass, null: false
       table.string :to_transition, null: false
       table.timestamp :started_at
@@ -14,9 +15,9 @@ class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>
       table.timestamps null: false
     end
 
-    add_index :ductwork_steps, %i[pipeline_id status klass]
-    add_index :ductwork_steps, %i[pipeline_id klass status]
-    add_index :ductwork_steps, %i[status klass]
+    add_index :ductwork_steps, %i[pipeline_id status node]
+    add_index :ductwork_steps, %i[pipeline_id node status]
+    add_index :ductwork_steps, %i[status node]
     add_index :ductwork_steps, %i[pipeline_id status]
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.1
 platform: ruby
 authors:
 - Tyler Ewing