ductwork 0.1.0

data/lib/ductwork/dsl/definition_builder.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module DSL
+    class DefinitionBuilder
+      class StartError < StandardError; end
+      class CollapseError < StandardError; end
+      class CombineError < StandardError; end
+
+      def initialize
+        @definition = {
+          nodes: [],
+          edges: {},
+        }
+      end
+
+      def start(klass)
+        validate_start_once!
+        add_new_nodes(klass)
+
+        self
+      end
+
+      # NOTE: there is a bug here that does not allow the user to reuse step
+      # classes in the same pipeline. i'll fix this later
+      def chain(klass)
+        validate_definition_started!(action: "chaining")
+        add_edge_to_last_node(klass, type: :chain)
+        add_new_nodes(klass)
+
+        self
+      end
+
+      def divide(to:)
+        validate_definition_started!(action: "dividing chain")
+        add_edge_to_last_node(*to, type: :divide)
+        add_new_nodes(*to)
+
+        if block_given?
+          branches = to.map do |klass|
+            Ductwork::DSL::BranchBuilder
+              .new(klass:, definition:)
+          end
+
+          yield branches
+        end
+
+        self
+      end
+
+      def combine(into:)
+        validate_definition_started!(action: "combining steps")
+        validate_definition_divided!
+
+        last_nodes = definition[:nodes].reverse.select do |node|
+          definition[:edges][node].empty?
+        end
+        last_nodes.each do |node|
+          definition[:edges][node] << {
+            to: [into.name],
+            type: :combine,
+          }
+        end
+        add_new_nodes(into)
+
+        self
+      end
+
+      def expand(to:)
+        validate_definition_started!(action: "expanding chain")
+        add_edge_to_last_node(to, type: :expand)
+        add_new_nodes(to)
+
+        self
+      end
+
+      def collapse(into:)
+        validate_definition_started!(action: "collapsing steps")
+        validate_definition_expanded!
+        add_edge_to_last_node(into, type: :collapse)
+        add_new_nodes(into)
+
+        self
+      end
+
+      def on_halt(klass)
+        definition[:metadata] ||= {}
+        definition[:metadata][:on_halt] = {}
+        definition[:metadata][:on_halt][:klass] = klass.name
+
+        self
+      end
+
+      def complete
+        validate_definition_started!(action: "completing")
+
+        definition
+      end
+
+      private
+
+      attr_reader :definition
+
+      def validate_start_once!
+        if definition[:nodes].any?
+          raise StartError, "Can only start pipeline definition once"
+        end
+      end
+
+      def validate_definition_started!(action:)
+        if definition[:nodes].empty?
+          raise StartError, "Must start pipeline definition before #{action}"
+        end
+      end
+
+      def validate_definition_divided!
+        if last_edge.nil? || last_edge[:type] != :divide
+          raise CombineError, "Must divide pipeline definition before combining steps"
+        end
+      end
+
+      def validate_definition_expanded!
+        if last_edge.nil? || last_edge[:type] != :expand
+          raise CollapseError, "Must expand pipeline definition before collapsing steps"
+        end
+      end
+
+      def last_edge
+        last_edge_node = definition[:nodes].reverse.find do |node|
+          definition[:edges][node].any?
+        end
+
+        definition.dig(:edges, last_edge_node, -1)
+      end
+
+      def add_new_nodes(*klasses)
+        definition[:nodes].push(*klasses.map(&:name))
+        klasses.each do |klass|
+          definition[:edges][klass.name] ||= []
+        end
+      end
+
+      def add_edge_to_last_node(*klasses, type:)
+        last_node = definition.dig(:nodes, -1)
+
+        definition[:edges][last_node] << {
+          to: klasses.map(&:name),
+          type: type,
+        }
+      end
+    end
+  end
+end

data/lib/ductwork/engine.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Engine < ::Rails::Engine
+    initializer "ductwork.app_executor", before: :run_prepare_callbacks do |app|
+      Ductwork.app_executor = app.executor
+    end
+
+    initializer "ductwork.configure" do
+      Ductwork.configuration ||= Ductwork::Configuration.new
+      Ductwork.configuration.logger ||= Rails.logger
+    end
+  end
+end

data/lib/ductwork/machine_identifier.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class MachineIdentifier
+    def self.fetch
+      File.read("/etc/machine-id").strip
+    rescue Errno::ENOENT
+      Socket.gethostname
+    end
+  end
+end

data/lib/ductwork/processes/job_worker.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class JobWorker
+      def initialize(pipeline, running_context)
+        @pipeline = pipeline
+        @running_context = running_context
+      end
+
+      def run
+        run_hooks_for(:start)
+        logger.debug(
+          msg: "Entering main work loop",
+          role: :job_worker,
+          pipeline: pipeline
+        )
+        while running_context.running?
+          logger.debug(
+            msg: "Attempting to claim job",
+            role: :job_worker,
+            pipeline: pipeline
+          )
+          job = Ductwork.wrap_with_app_executor do
+            Job.claim_latest(pipeline)
+          end
+
+          if job.present?
+            Ductwork.wrap_with_app_executor do
+              job.execute(pipeline)
+            end
+          else
+            logger.debug(
+              msg: "No job to claim, looping",
+              role: :job_worker,
+              pipeline: pipeline
+            )
+            sleep(Ductwork.configuration.job_worker_polling_timeout)
+          end
+        end
+
+        shutdown
+      end
+
+      private
+
+      attr_reader :pipeline, :running_context
+
+      def shutdown
+        logger.debug(
+          msg: "Shutting down",
+          role: :job_worker,
+          pipeline: pipeline
+        )
+        run_hooks_for(:stop)
+      end
+
+      def run_hooks_for(event)
+        Ductwork.hooks[:worker].fetch(event, []).each do |block|
+          Ductwork.wrap_with_app_executor do
+            block.call(self)
+          end
+        end
+      end
+
+      def logger
+        Ductwork.configuration.logger
+      end
+    end
+  end
+end

data/lib/ductwork/processes/job_worker_runner.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class JobWorkerRunner
+      def initialize(pipeline)
+        @pipeline = pipeline
+        @running_context = Ductwork::RunningContext.new
+        @threads = create_threads
+
+        Signal.trap(:INT) { running_context.shutdown! }
+        Signal.trap(:TERM) { running_context.shutdown! }
+        Signal.trap(:TTIN) do
+          Thread.list.each do |thread|
+            puts thread.name
+            if thread.backtrace
+              puts thread.backtrace.join("\n")
+            else
+              puts "No backtrace to dump"
+            end
+            puts
+          end
+        end
+      end
+
+      def run
+        create_process!
+        logger.debug(
+          msg: "Entering main work loop",
+          role: :job_worker_runner,
+          pipeline: pipeline
+        )
+
+        while running?
+          # TODO: Increase or make configurable
+          sleep(5)
+          attempt_synchronize_threads
+          report_heartbeat!
+        end
+
+        shutdown!
+      end
+
+      private
+
+      attr_reader :pipeline, :running_context, :threads
+
+      def worker_count
+        Ductwork.configuration.job_worker_count(pipeline)
+      end
+
+      def create_threads
+        worker_count.times.map do |i|
+          job_worker = Ductwork::Processes::JobWorker.new(
+            pipeline,
+            running_context
+          )
+          logger.debug(
+            msg: "Creating new thread",
+            role: :job_worker_runner,
+            pipeline: pipeline
+          )
+          thread = Thread.new do
+            job_worker.run
+          end
+          thread.name = "ductwork.job_worker_#{i}"
+
+          logger.debug(
+            msg: "Created new thread",
+            role: :job_worker_runner,
+            pipeline: pipeline
+          )
+
+          thread
+        end
+      end
+
+      def create_process!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.create!(
+            pid: ::Process.pid,
+            machine_identifier: Ductwork::MachineIdentifier.fetch,
+            last_heartbeat_at: Time.current
+          )
+        end
+      end
+
+      def running?
+        running_context.running?
+      end
+
+      def attempt_synchronize_threads
+        logger.debug(
+          msg: "Attempting to synchronize threads",
+          role: :job_worker_runner,
+          pipeline: pipeline
+        )
+        threads.each { |thread| thread.join(0.1) }
+        logger.debug(
+          msg: "Synchronizing threads timed out",
+          role: :job_worker_runner,
+          pipeline: pipeline
+        )
+      end
+
+      def report_heartbeat!
+        logger.debug(msg: "Reporting heartbeat", role: :job_worker_runner)
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.report_heartbeat!
+        end
+        logger.debug(msg: "Reported heartbeat", role: :job_worker_runner)
+      end
+
+      def shutdown!
+        running_context.shutdown!
+        await_threads_graceful_shutdown
+        kill_remaining_threads
+        delete_process
+      end
+
+      def await_threads_graceful_shutdown
+        timeout = Ductwork.configuration.job_worker_shutdown_timeout
+        deadline = Time.current + timeout
+
+        logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
+        while Time.current < deadline && threads.any?(&:alive?)
+          threads.each do |thread|
+            break if Time.current < deadline
+
+            # TODO: Maybe make this configurable. If there's a ton of workers
+            # it may not even get to the "later" ones depending on the timeout
+            thread.join(1)
+          end
+        end
+      end
+
+      def kill_remaining_threads
+        threads.each do |thread|
+          if thread.alive?
+            thread.kill
+            logger.debug(
+              msg: "Killed thread",
+              role: :job_worker_runner,
+              thread: thread.name
+            )
+          end
+        end
+      end
+
+      def delete_process
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.find_by!(
+            pid: ::Process.pid,
+            machine_identifier: Ductwork::MachineIdentifier.fetch
+          ).delete
+        end
+      end
+
+      def logger
+        Ductwork.configuration.logger
+      end
+    end
+  end
+end

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class PipelineAdvancer
+      def initialize(running_context, klass)
+        @running_context = running_context
+        @klass = klass
+      end
+
+      def run # rubocop:todo Metrics/AbcSize, Metrics/MethodLength
+        run_hooks_for(:start)
+        while running_context.running?
+          id = Ductwork::Record.uncached do
+            Ductwork::Pipeline
+              .in_progress
+              .where(klass: klass, claimed_for_advancing_at: nil)
+              .where.not(steps: Ductwork::Step.where.not(status: %w[advancing completed]))
+              .order(:last_advanced_at)
+              .limit(1)
+              .pluck(:id)
+              .first
+          end
+
+          if id.present?
+            rows_updated = Ductwork::Pipeline
+                           .where(id: id, claimed_for_advancing_at: nil)
+                           .update_all(claimed_for_advancing_at: Time.current)
+
+            if rows_updated == 1
+              logger.debug(
+                msg: "Pipeline claimed",
+                pipeline: klass,
+                role: :pipeline_advancer
+              )
+
+              pipeline = Ductwork::Pipeline.find(id)
+              pipeline.advance!
+
+              logger.debug(
+                msg: "Pipeline advanced",
+                pipeline: klass,
+                role: :pipeline_advancer
+              )
+            else
+              logger.debug(
+                msg: "Did not claim pipeline, avoided race condition",
+                pipeline: klass,
+                role: :pipeline_advancer
+              )
+            end
+
+            # release the pipeline and set last advanced at so it doesnt block.
+            # we're not using a queue so we have to use a db timestamp
+            Ductwork::Pipeline.find(id).update!(
+              claimed_for_advancing_at: nil,
+              last_advanced_at: Time.current
+            )
+          else
+            logger.debug(
+              msg: "No pipeline needs advancing",
+              pipeline: klass,
+              id: id,
+              role: :pipeline_advancer
+            )
+          end
+
+          sleep(Ductwork.configuration.pipeline_polling_timeout)
+        end
+
+        run_hooks_for(:stop)
+      end
+
+      private
+
+      attr_reader :running_context, :klass
+
+      def run_hooks_for(event)
+        Ductwork.hooks[:advancer].fetch(event, []).each do |block|
+          Ductwork.wrap_with_app_executor do
+            block.call(self)
+          end
+        end
+      end
+
+      def logger
+        Ductwork.configuration.logger
+      end
+    end
+  end
+end

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class PipelineAdvancerRunner
+      def initialize(*klasses)
+        @klasses = klasses
+        @running_context = Ductwork::RunningContext.new
+        @threads = create_threads
+
+        Signal.trap(:INT) { running_context.shutdown! }
+        Signal.trap(:TERM) { running_context.shutdown! }
+        Signal.trap(:TTIN) do
+          Thread.list.each do |thread|
+            puts thread.name
+            if thread.backtrace
+              puts thread.backtrace.join("\n")
+            else
+              puts "No backtrace to dump"
+            end
+            puts
+          end
+        end
+      end
+
+      def run
+        create_process!
+        logger.debug(
+          msg: "Entering main work loop",
+          role: :pipeline_advancer_runner
+        )
+
+        while running_context.running?
+          # TODO: Increase or make configurable
+          sleep(5)
+          attempt_synchronize_threads
+          report_heartbeat!
+        end
+
+        shutdown
+      end
+
+      private
+
+      attr_reader :klasses, :running_context, :threads
+
+      def create_threads
+        klasses.map do |klass|
+          pipeline_advancer = Ductwork::Processes::PipelineAdvancer.new(
+            running_context,
+            klass
+          )
+
+          logger.debug(
+            msg: "Creating new thread",
+            role: :pipeline_advancer_runner,
+            pipeline: klass
+          )
+          thread = Thread.new do
+            Ductwork.wrap_with_app_executor do
+              pipeline_advancer.run
+            end
+          end
+          thread.name = "ductwork.pipeline_advancer.#{klass}"
+
+          logger.debug(
+            msg: "Created new thread",
+            role: :pipeline_advancer_runner,
+            thread: thread.name,
+            pipeline: klass
+          )
+
+          thread
+        end
+      end
+
+      def attempt_synchronize_threads
+        logger.debug(
+          msg: "Attempting to synchronize threads",
+          role: :pipeline_advancer_runner
+        )
+        threads.each { |thread| thread.join(0.1) }
+        logger.debug(
+          msg: "Synchronizing threads timed out",
+          role: :pipeline_advancer_runner
+        )
+      end
+
+      def create_process!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.create!(
+            pid: ::Process.pid,
+            machine_identifier: Ductwork::MachineIdentifier.fetch,
+            last_heartbeat_at: Time.current
+          )
+        end
+      end
+
+      def report_heartbeat!
+        logger.debug(msg: "Reporting heartbeat", role: :pipeline_advancer_runner)
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.report_heartbeat!
+        end
+        logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
+      end
+
+      def shutdown
+        log_shutting_down
+        stop_running_context
+        await_threads_graceful_shutdown
+        kill_remaining_threads
+        delete_process!
+      end
+
+      def log_shutting_down
+        logger.debug(msg: "Shutting down", role: :pipeline_advancer_runner)
+      end
+
+      def stop_running_context
+        running_context.shutdown!
+      end
+
+      def await_threads_graceful_shutdown
+        timeout = Ductwork.configuration.pipeline_shutdown_timeout
+        deadline = Time.current + timeout
+
+        logger.debug(
+          msg: "Attempting graceful shutdown",
+          role: :pipeline_advancer_runner
+        )
+        while Time.current < deadline && threads.any?(&:alive?)
+          threads.each do |thread|
+            break if Time.current < deadline
+
+            # TODO: Maybe make this configurable. If there's a ton of workers
+            # it may not even get to the "later" ones depending on the timeout
+            thread.join(1)
+          end
+        end
+      end
+
+      def kill_remaining_threads
+        threads.each do |thread|
+          if thread.alive?
+            thread.kill
+            logger.debug(
+              msg: "Killed thread",
+              role: :pipeline_advancer_runner,
+              thread: thread.name
+            )
+          end
+        end
+      end
+
+      def delete_process!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.find_by!(
+            pid: ::Process.pid,
+            machine_identifier: Ductwork::MachineIdentifier.fetch
+          ).delete
+        end
+      end
+
+      def logger
+        Ductwork.configuration.logger
+      end
+    end
+  end
+end