ductwork 0.1.0

data/app/models/ductwork/pipeline.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Pipeline < Ductwork::Record # rubocop:todo Metrics/ClassLength
+    has_many :steps, class_name: "Ductwork::Step", foreign_key: "pipeline_id", dependent: :destroy
+
+    validates :klass, presence: true
+    validates :definition, presence: true
+    validates :definition_sha1, presence: true
+    validates :status, presence: true
+    validates :triggered_at, presence: true
+    validates :last_advanced_at, presence: true
+
+    enum :status,
+         pending: "pending",
+         in_progress: "in_progress",
+         halted: "halted",
+         completed: "completed"
+
+    def self.inherited(subclass)
+      super
+
+      subclass.class_eval do
+        default_scope { where(klass: name.to_s) }
+      end
+    end
+
+    class DefinitionError < StandardError; end
+
+    class << self
+      attr_reader :pipeline_definition
+
+      def define(&block)
+        if !block_given?
+          raise DefinitionError, "Definition block must be given"
+        end
+
+        if pipeline_definition
+          raise DefinitionError, "Pipeline has already been defined"
+        end
+
+        builder = Ductwork::DSL::DefinitionBuilder.new
+
+        block.call(builder)
+
+        @pipeline_definition = builder.complete
+
+        Ductwork.defined_pipelines << name.to_s
+      end
+
+      def trigger(*args)
+        if pipeline_definition.nil?
+          raise DefinitionError, "Pipeline must be defined before triggering"
+        end
+
+        step_klass = pipeline_definition.dig(:nodes, 0)
+        definition = JSON.dump(pipeline_definition)
+
+        Record.transaction do
+          pipeline = create!(
+            klass: name.to_s,
+            status: :in_progress,
+            definition: definition,
+            definition_sha1: Digest::SHA1.hexdigest(definition),
+            triggered_at: Time.current,
+            last_advanced_at: Time.current
+          )
+          step = pipeline.steps.create!(
+            klass: step_klass,
+            status: :in_progress,
+            step_type: :start,
+            started_at: Time.current
+          )
+          Ductwork::Job.enqueue(step, *args)
+
+          pipeline
+        end
+      end
+    end
+
+    def advance!
+      step = steps.advancing.take
+      edge = if step.present?
+               parsed_definition.dig(:edges, step.klass, 0)
+             end
+
+      Ductwork::Record.transaction do
+        steps.advancing.update!(status: :completed, completed_at: Time.current)
+
+        if edge.nil?
+          conditionally_complete_pipeline
+        else
+          advance_to_next_step_by_type(edge, step)
+        end
+      end
+    end
+
+    private
+
+    def create_step_and_enqueue_job(klass:, step_type:, input_arg:)
+      status = :in_progress
+      started_at = Time.current
+      next_step = steps.create!(klass:, status:, step_type:, started_at:)
+      Ductwork::Job.enqueue(next_step, input_arg)
+    end
+
+    def parsed_definition
+      @parsed_definition ||= JSON.parse(definition).with_indifferent_access
+    end
+
+    def conditionally_complete_pipeline
+      if steps.where(status: %w[in_progress pending]).none?
+        update!(status: :completed, completed_at: Time.current)
+      end
+    end
+
+    def advance_to_next_step_by_type(edge, step)
+      # NOTE: "chain" is used by ActiveRecord so we have to call
+      # this enum value "default" :sad:
+      step_type = edge[:type] == "chain" ? "default" : edge[:type]
+
+      if step_type.in?(%w[default divide])
+        advance_to_next_steps(step_type, step, edge)
+      elsif step_type == "combine"
+        combine_next_steps(step_type, edge)
+      elsif step_type == "expand"
+        expand_to_next_steps(step_type, step, edge)
+      elsif step_type == "collapse"
+        collapse_next_steps(step_type, step, edge)
+      else
+        Ductwork.configuration.logger.error(
+          msg: "Invalid step type",
+          role: :pipeline_advancer
+        )
+      end
+    end
+
+    def advance_to_next_steps(step_type, step, edge)
+      edge[:to].each do |to_klass|
+        next_step = steps.create!(
+          klass: to_klass,
+          status: :in_progress,
+          step_type: step_type,
+          started_at: Time.current
+        )
+        Ductwork::Job.enqueue(next_step, step.job.return_value)
+      end
+    end
+
+    def combine_next_steps(step_type, edge)
+      previous_klasses = parsed_definition[:edges].select do |_, v|
+        v.dig(0, :to, 0) == edge[:to].sole && v.dig(0, :type) == "combine"
+      end.keys
+
+      if steps.not_completed.where(klass: previous_klasses).none?
+        input_arg = Job.where(
+          step: steps.completed.where(klass: previous_klasses)
+        ).map(&:return_value)
+        create_step_and_enqueue_job(
+          klass: edge[:to].sole,
+          step_type: step_type,
+          input_arg: input_arg
+        )
+      end
+    end
+
+    def expand_to_next_steps(step_type, step, edge)
+      step.job.return_value.each do |input_arg|
+        create_step_and_enqueue_job(
+          klass: edge[:to].sole,
+          step_type: step_type,
+          input_arg: input_arg
+        )
+      end
+    end
+
+    def collapse_next_steps(step_type, step, edge)
+      if steps.not_completed.where(klass: step.klass).none?
+        input_arg = Job.where(
+          step: steps.completed.where(klass: step.klass)
+        ).map(&:return_value)
+        create_step_and_enqueue_job(
+          klass: edge[:to].sole,
+          step_type: step_type,
+          input_arg: input_arg
+        )
+      else
+        Ductwork.configuration.logger.debug(
+          msg: "Not all expanded steps have completed",
+          role: :pipeline_advancer
+        )
+      end
+    end
+  end
+end

data/app/models/ductwork/process.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Process < Ductwork::Record
+    class NotFoundError < StandardError; end
+
+    validates :pid, uniqueness: { scope: :machine_identifier }
+
+    def self.report_heartbeat!
+      pid = ::Process.pid
+      machine_identifier = Ductwork::MachineIdentifier.fetch
+
+      find_by!(pid:, machine_identifier:)
+        .update!(last_heartbeat_at: Time.current)
+    rescue ActiveRecord::RecordNotFound
+      raise NotFoundError, "Process #{pid} not found"
+    end
+  end
+end

data/app/models/ductwork/record.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Record < ActiveRecord::Base
+    self.abstract_class = true
+
+    if Ductwork.configuration.database.present?
+      connects_to(database: { writing: Ductwork.configuration.database.to_sym })
+    end
+
+    def self.table_name_prefix
+      "ductwork_"
+    end
+  end
+end

data/app/models/ductwork/result.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Result < Ductwork::Record
+    belongs_to :execution, class_name: "Ductwork::Execution"
+
+    validates :result_type, presence: true
+
+    enum :result_type,
+         success: "success",
+         failure: "failure"
+  end
+end

data/app/models/ductwork/run.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Run < Ductwork::Record
+    belongs_to :execution, class_name: "Ductwork::Execution"
+
+    validates :started_at, presence: true
+  end
+end

data/app/models/ductwork/step.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Step < Ductwork::Record
+    belongs_to :pipeline, class_name: "Ductwork::Pipeline"
+    has_one :job, class_name: "Ductwork::Job", foreign_key: "step_id"
+
+    validates :klass, presence: true
+    validates :status, presence: true
+    validates :step_type, presence: true
+
+    enum :status,
+         pending: "pending",
+         in_progress: "in_progress",
+         advancing: "advancing",
+         failed: "failed",
+         completed: "completed"
+
+    enum :step_type,
+         start: "start",
+         default: "default", # `chain` is used by AR
+         divide: "divide",
+         combine: "combine",
+         expand: "expand",
+         collapse: "collapse"
+  end
+end

data/lib/ductwork/cli.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module Ductwork
+  class CLI
+    class << self
+      def start!(args)
+        options = parse_options(args)
+        Ductwork.configuration = Configuration.new(**options)
+        Ductwork.configuration.logger = if Ductwork.configuration.logger_source == "rails"
+                                          Rails.logger
+                                        else
+                                          Ductwork::Configuration::DEFAULT_LOGGER
+                                        end
+        Ductwork.configuration.logger.level = Ductwork.configuration.logger_level
+
+        Ductwork::Processes::SupervisorRunner.start!
+      end
+
+      private
+
+      def parse_options(args)
+        options = {}
+
+        OptionParser.new do |op|
+          op.banner = "ductwork [options]"
+
+          op.on("-c", "--config PATH", "path to YAML config file") do |arg|
+            options[:path] = arg
+          end
+
+          op.on("-h", "--help", "Prints this help") do
+            puts op
+            exit
+          end
+
+          op.on("-v", "--version", "Prints the version") do
+            puts "Ductwork #{Ductwork::VERSION}"
+            exit
+          end
+        end.parse!(args)
+
+        options
+      end
+    end
+  end
+end

data/lib/ductwork/configuration.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Configuration
+    DEFAULT_ENV = :default
+    DEFAULT_FILE_PATH = "config/ductwork.yml"
+    DEFAULT_JOB_WORKER_COUNT = 5 # threads
+    DEFAULT_JOB_WORKER_MAX_RETRY = 3 # attempts
+    DEFAULT_JOB_WORKER_POLLING_TIMEOUT = 1 # second
+    DEFAULT_JOB_WORKER_SHUTDOWN_TIMEOUT = 20 # seconds
+    DEFAULT_LOGGER_LEVEL = ::Logger::INFO
+    DEFAULT_LOGGER_SOURCE = "default" # `Logger` instance writing to STDOUT
+    DEFAULT_PIPELINE_POLLING_TIMEOUT = 1 # second
+    DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT = 20 # seconds
+    DEFAULT_SUPERVISOR_POLLING_TIMEOUT = 1 # second
+    DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT = 30 # seconds
+    DEFAULT_LOGGER = ::Logger.new($stdout)
+    PIPELINES_WILDCARD = "*"
+
+    attr_accessor :logger
+    attr_writer :job_worker_polling_timeout, :job_worker_shutdown_timeout,
+                :job_worker_max_retry,
+                :logger_level,
+                :pipeline_polling_timeout, :pipeline_shutdown_timeout,
+                :supervisor_polling_timeout, :supervisor_shutdown_timeout
+
+    def initialize(path: DEFAULT_FILE_PATH)
+      full_path = Pathname.new(path)
+      data = ActiveSupport::ConfigurationFile.parse(full_path).deep_symbolize_keys
+      env = defined?(Rails) ? Rails.env.to_sym : DEFAULT_ENV
+      @config = data[env]
+    rescue Errno::ENOENT
+      @config = {}
+    end
+
+    def pipelines
+      raw_pipelines = config[:pipelines] || []
+
+      if raw_pipelines == PIPELINES_WILDCARD
+        Dir
+          .glob("**/*.rb", base: "app/pipelines")
+          .map { |path| path.delete_suffix(".rb").camelize }
+      else
+        raw_pipelines.map(&:strip)
+      end
+    end
+
+    def database
+      config[:database]
+    end
+
+    def job_worker_count(pipeline)
+      raw_count = config.dig(:job_worker, :count) || DEFAULT_JOB_WORKER_COUNT
+
+      if raw_count.is_a?(Hash)
+        raw_count[pipeline.to_sym]
+      else
+        raw_count
+      end
+    end
+
+    def job_worker_max_retry
+      @job_worker_max_retry ||= fetch_job_worker_max_retry
+    end
+
+    def job_worker_polling_timeout
+      @job_worker_polling_timeout ||= fetch_job_worker_polling_timeout
+    end
+
+    def job_worker_shutdown_timeout
+      @job_worker_shutdown_timeout ||= fetch_job_worker_shutdown_timeout
+    end
+
+    def logger_level
+      @logger_level ||= fetch_logger_level
+    end
+
+    def logger_source
+      @logger_source ||= fetch_logger_source
+    end
+
+    def pipeline_polling_timeout
+      @pipeline_polling_timeout ||= fetch_pipeline_polling_timeout
+    end
+
+    def pipeline_shutdown_timeout
+      @pipeline_shutdown_timeout ||= fetch_pipeline_shutdown_timeout
+    end
+
+    def supervisor_polling_timeout
+      @supervisor_polling_timeout ||= fetch_supervisor_polling_timeout
+    end
+
+    def supervisor_shutdown_timeout
+      @supervisor_shutdown_timeout ||= fetch_supervisor_shutdown_timeout
+    end
+
+    private
+
+    attr_reader :config
+
+    def fetch_job_worker_max_retry
+      config.dig(:job_worker, :max_retry) ||
+        DEFAULT_JOB_WORKER_MAX_RETRY
+    end
+
+    def fetch_job_worker_polling_timeout
+      config.dig(:job_worker, :polling_timeout) ||
+        DEFAULT_JOB_WORKER_POLLING_TIMEOUT
+    end
+
+    def fetch_job_worker_shutdown_timeout
+      config.dig(:job_worker, :shutdown_timeout) ||
+        DEFAULT_JOB_WORKER_SHUTDOWN_TIMEOUT
+    end
+
+    def fetch_logger_level
+      config.dig(:logger, :level) || DEFAULT_LOGGER_LEVEL
+    end
+
+    def fetch_logger_source
+      config.dig(:logger, :source) || DEFAULT_LOGGER_SOURCE
+    end
+
+    def fetch_pipeline_polling_timeout
+      config.dig(:pipeline_advancer, :polling_timeout) ||
+        DEFAULT_PIPELINE_POLLING_TIMEOUT
+    end
+
+    def fetch_pipeline_shutdown_timeout
+      config.dig(:pipeline_advancer, :shutdown_timeout) ||
+        DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT
+    end
+
+    def fetch_supervisor_polling_timeout
+      config.dig(:supervisor, :polling_timeout) ||
+        DEFAULT_SUPERVISOR_POLLING_TIMEOUT
+    end
+
+    def fetch_supervisor_shutdown_timeout
+      config.dig(:supervisor, :shutdown_timeout) ||
+        DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT
+    end
+  end
+end

data/lib/ductwork/dsl/branch_builder.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module DSL
+    class BranchBuilder
+      class CollapseError < StandardError; end
+
+      attr_reader :last_node
+
+      def initialize(klass:, definition:)
+        @last_node = klass.name
+        @definition = definition
+        @expansions = 0
+      end
+
+      def chain(next_klass)
+        definition[:edges][last_node] << {
+          to: [next_klass.name],
+          type: :chain,
+        }
+
+        definition[:nodes].push(next_klass.name)
+        definition[:edges][next_klass.name] ||= []
+        @last_node = next_klass.name
+
+        self
+      end
+
+      def divide(to:)
+        definition[:edges][last_node] << {
+          to: to.map(&:name),
+          type: :divide,
+        }
+
+        definition[:nodes].push(*to.map(&:name))
+        sub_branches = to.map do |klass|
+          definition[:edges][klass.name] ||= []
+
+          Ductwork::DSL::BranchBuilder.new(klass:, definition:)
+        end
+
+        yield sub_branches
+
+        self
+      end
+
+      def combine(*branch_builders, into:)
+        definition[:edges][last_node] << {
+          to: [into.name],
+          type: :combine,
+        }
+        branch_builders.each do |branch|
+          definition[:edges][branch.last_node] << {
+            to: [into.name],
+            type: :combine,
+          }
+        end
+        definition[:nodes].push(into.name)
+        definition[:edges][into.name] ||= []
+
+        self
+      end
+
+      def expand(to:)
+        definition[:edges][last_node] << {
+          to: [to.name],
+          type: :expand,
+        }
+
+        definition[:nodes].push(to.name)
+        definition[:edges][to.name] ||= []
+        @last_node = to.name
+        @expansions += 1
+
+        self
+      end
+
+      def collapse(into:)
+        if expansions.zero?
+          raise CollapseError,
+                "Must expand pipeline definition before collapsing steps"
+        end
+
+        definition[:edges][last_node] << {
+          to: [into.name],
+          type: :collapse,
+        }
+
+        definition[:nodes].push(into.name)
+        definition[:edges][into.name] ||= []
+        @last_node = into.name
+        @expansions -= 1
+
+        self
+      end
+
+      private
+
+      attr_reader :definition, :expansions
+    end
+  end
+end