ductwork 0.1.0

data/lib/ductwork/processes/supervisor.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class Supervisor
+      attr_reader :workers
+
+      def initialize
+        @running = true
+        @workers = []
+
+        run_hooks_for(:start)
+
+        Signal.trap(:INT) { @running = false }
+        Signal.trap(:TERM) { @running = false }
+        Signal.trap(:TTIN) { puts "No threads to dump" }
+      end
+
+      def add_worker(metadata: {}, &block)
+        pid = fork do
+          block.call(metadata)
+        end
+
+        workers << { metadata:, pid:, block: }
+        logger.debug(
+          msg: "Started child process (#{pid}) with metadata #{metadata}",
+          pid: pid
+        )
+      end
+
+      def run
+        logger.debug(msg: "Entering main work loop", role: :supervisor, pid: ::Process.pid)
+
+        while running
+          sleep(Ductwork.configuration.supervisor_polling_timeout)
+          check_workers
+        end
+
+        shutdown
+      end
+
+      def shutdown
+        @running = false
+
+        logger.debug(msg: "Beginning shutdown", role: :supervisor)
+        terminate_gracefully
+        wait_for_workers_to_exit
+        terminate_immediately
+        run_hooks_for(:stop)
+      end
+
+      private
+
+      attr_reader :running
+
+      def check_workers
+        logger.debug(msg: "Checking workers are alive", role: :supervisor)
+
+        workers.each do |worker|
+          if process_dead?(worker[:pid])
+            old_pid = worker[:pid]
+            new_pid = fork do
+              worker[:block].call(worker[:metadata])
+            end
+            worker[:pid] = new_pid
+            logger.debug(
+              msg: "Restarted process (#{old_pid}) as (#{new_pid})",
+              role: :supervisor,
+              old_pid: old_pid,
+              new_pid: new_pid
+            )
+          end
+        end
+
+        logger.debug(msg: "All workers are alive or revived", role: :supervisor)
+      end
+
+      def terminate_gracefully
+        workers.each do |worker|
+          logger.debug(
+            msg: "Sending TERM signal to process (#{worker[:pid]})",
+            role: :supervisor,
+            pid: worker[:pid],
+            signal: :TERM
+          )
+          ::Process.kill(:TERM, worker[:pid])
+        end
+      end
+
+      def wait_for_workers_to_exit
+        deadline = now + Ductwork.configuration.supervisor_shutdown_timeout
+
+        while workers.any? && now < deadline
+          sleep(0.1)
+          workers.each_with_index do |worker, index|
+            if ::Process.wait(worker[:pid], ::Process::WNOHANG)
+              workers[index] = nil
+              logger.debug(
+                msg: "Child process (#{worker[:pid]}) stopped successfully",
+                role: :supervisor,
+                pid: worker[:pid]
+              )
+            end
+          end
+          @workers = workers.compact
+        end
+      end
+
+      def terminate_immediately
+        workers.each_with_index do |worker, index|
+          logger.debug(
+            msg: "Sending KILL signal to process (#{worker[:pid]})",
+            role: :supervisor,
+            pid: worker[:pid],
+            signal: :KILL
+          )
+          ::Process.kill(:KILL, worker[:pid])
+          ::Process.wait(worker[:pid])
+          workers[index] = nil
+          logger.debug(
+            msg: "Child process (#{worker[:pid]}) killed after timeout",
+            role: :supervisor,
+            pid: worker[:pid]
+          )
+        rescue Errno::ESRCH, Errno::ECHILD
+          # no-op because process is already dead
+        end
+
+        @workers = workers.compact
+      end
+
+      def process_dead?(pid)
+        machine_identifier = Ductwork::MachineIdentifier.fetch
+
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process
+            .where(pid:, machine_identifier:)
+            .where("last_heartbeat_at < ?", 5.minutes.ago)
+            .exists?
+        end
+      end
+
+      def run_hooks_for(event)
+        Ductwork.hooks[:supervisor].fetch(event, []).each do |block|
+          Ductwork.wrap_with_app_executor do
+            block.call(self)
+          end
+        end
+      end
+
+      def now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
+      def logger
+        Ductwork.configuration.logger
+      end
+    end
+  end
+end

data/lib/ductwork/processes/supervisor_runner.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class SupervisorRunner
+      def self.start!
+        supervisor = Ductwork::Processes::Supervisor.new
+        pipelines_to_advance = Ductwork.configuration.pipelines
+        logger = Ductwork.configuration.logger
+
+        supervisor.add_worker(metadata: { pipelines: pipelines_to_advance }) do
+          logger.debug(
+            msg: "Starting Pipeline Advancer process",
+            role: :supervisor_runner
+          )
+          Ductwork::Processes::PipelineAdvancerRunner
+            .new(*pipelines_to_advance).run
+        end
+
+        pipelines_to_advance.each do |pipeline|
+          supervisor.add_worker(metadata: { pipeline: }) do
+            logger.debug(
+              msg: "Starting Job Worker Runner process",
+              role: :supervisor_runner,
+              pipeline: pipeline
+            )
+            Ductwork::Processes::JobWorkerRunner.new(pipeline).run
+          end
+        end
+
+        supervisor.run
+      end
+    end
+  end
+end

data/lib/ductwork/running_context.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class RunningContext
+    def initialize
+      @mutex = Mutex.new
+      @running = true
+    end
+
+    def running?
+      mutex.synchronize { running }
+    end
+
+    def shutdown!
+      @running = false
+    end
+
+    private
+
+    attr_reader :mutex, :running
+  end
+end

data/lib/ductwork/testing/helpers.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Testing
+    module Helpers
+      def pipelines_created_around(&block)
+        before_ids = Ductwork::Pipeline.ids
+
+        block.call
+
+        after_ids = Ductwork::Pipeline.ids
+        ids_delta = after_ids - before_ids
+
+        Ductwork::Pipeline.where(id: ids_delta)
+      end
+    end
+  end
+end

data/lib/ductwork/testing/minitest.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Testing
+    module Minitest
+    end
+  end
+end

data/lib/ductwork/testing/rspec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+RSpec::Matchers.define(:have_triggered_pipeline) do |expected|
+  include Ductwork::Testing::Helpers
+
+  supports_block_expectations
+
+  match do |block|
+    pipelines = pipelines_created_around(&block)
+    delta = pipelines.count
+    expected_count = count || 1
+
+    if delta == expected_count
+      pipelines.pluck(:klass).uniq.sort == Array(expected).map(&:name).sort
+    else
+      @failure_result = if delta.zero?
+                          :none
+                        elsif delta > 1
+                          :too_many
+                        else
+                          :other
+                        end
+
+      false
+    end
+  end
+
+  chain :exactly, :count
+
+  chain :times do # rubocop:disable Lint/EmptyBlock
+  end
+
+  failure_message do |actual|
+    case @failure_result
+    when :none
+      "expected to trigger pipeline #{expected} but triggered none"
+    when :too_many
+      "expected to trigger pipeline #{expected} but triggered more than one"
+    when :other
+      "expected to trigger pipeline #{expected} but triggered #{actual}"
+    else
+      "expected to trigger pipeline #{expected} but did not"
+    end
+  end
+end
+
+RSpec::Matchers.define(:have_triggered_pipelines) do |*expected|
+  include Ductwork::Testing::Helpers
+
+  supports_block_expectations
+
+  match do |block|
+    pipelines = pipelines_created_around(&block)
+
+    pipelines.map(&:klass).sort == expected.map(&:name).sort
+  end
+
+  failure_message do |_actual|
+    pipeline_names = expected.map(&:name).join(", ")
+
+    "expected to trigger pipelines: #{pipeline_names} but did not"
+  end
+end

data/lib/ductwork/testing.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "ductwork/testing/helpers"
+
+module Ductwork
+  module Testing
+    if defined?(RSpec)
+      require "ductwork/testing/rspec"
+    elsif defined?(Minitest)
+      require "ductwork/testing/minitest"
+    else
+      raise "Testing framework is not supported"
+    end
+  end
+end

data/lib/ductwork/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Ductwork
+  VERSION = "0.1.0"
+end

data/lib/ductwork.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "active_support"
+require "logger"
+require "rails/engine"
+require "securerandom"
+require "zeitwerk"
+
+require "ductwork/engine"
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect("cli" => "CLI")
+loader.inflector.inflect("dsl" => "DSL")
+loader.ignore("#{__dir__}/generators")
+loader.ignore("#{__dir__}/ductwork/testing")
+loader.ignore("#{__dir__}/ductwork/testing.rb")
+loader.setup
+
+module Ductwork
+  class << self
+    attr_accessor :app_executor, :configuration
+    attr_writer :defined_pipelines, :hooks
+
+    def wrap_with_app_executor(&block)
+      if app_executor.present?
+        app_executor.wrap(&block)
+      else
+        yield
+      end
+    end
+
+    def hooks
+      @hooks ||= {
+        supervisor: { start: [], stop: [] },
+        advancer: { start: [], stop: [] },
+        worker: { start: [], stop: [] },
+      }
+    end
+
+    def on_supervisor_start(&block)
+      add_lifecycle_hook(:supervisor, :start, block)
+    end
+
+    def on_supervisor_stop(&block)
+      add_lifecycle_hook(:supervisor, :stop, block)
+    end
+
+    def on_advancer_start(&block)
+      add_lifecycle_hook(:advancer, :start, block)
+    end
+
+    def on_advancer_stop(&block)
+      add_lifecycle_hook(:advancer, :stop, block)
+    end
+
+    def on_worker_start(&block)
+      add_lifecycle_hook(:worker, :start, block)
+    end
+
+    def on_worker_stop(&block)
+      add_lifecycle_hook(:worker, :stop, block)
+    end
+
+    def defined_pipelines
+      @defined_pipelines ||= []
+    end
+
+    private
+
+    def add_lifecycle_hook(target, event, block)
+      hooks[target] ||= {}
+      hooks[target][event] ||= []
+      hooks[target][event].push(block)
+    end
+  end
+end

data/lib/generators/ductwork/install/USAGE

@@ -0,0 +1,11 @@
+Description:
+    Installs necessary ductwork files
+
+Example:
+    bin/rails generate ductwork:install
+
+    This will perform the following:
+        Adds migrations
+        Adds default configurations
+        Installs bin/ductwork binstub to start the supervisor
+

data/lib/generators/ductwork/install/install_generator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "rails/generators/migration"
+require "rails/generators/active_record/migration"
+
+module Ductwork
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+
+    source_root File.expand_path("templates", __dir__)
+
+    def create_files
+      template "config/ductwork.yml"
+      template "bin/ductwork"
+
+      migration_template "db/create_ductwork_pipelines.rb",
+                         "db/migrate/create_ductwork_pipelines.rb"
+      migration_template "db/create_ductwork_steps.rb",
+                         "db/migrate/create_ductwork_steps.rb"
+      migration_template "db/create_ductwork_jobs.rb",
+                         "db/migrate/create_ductwork_jobs.rb"
+      migration_template "db/create_ductwork_executions.rb",
+                         "db/migrate/create_ductwork_executions.rb"
+      migration_template "db/create_ductwork_availabilities.rb",
+                         "db/migrate/create_ductwork_availabilities.rb"
+      migration_template "db/create_ductwork_runs.rb",
+                         "db/migrate/create_ductwork_runs.rb"
+      migration_template "db/create_ductwork_results.rb",
+                         "db/migrate/create_ductwork_results.rb"
+      migration_template "db/create_ductwork_processes.rb",
+                         "db/migrate/create_ductwork_processes.rb"
+
+      chmod "bin/ductwork", 0o755 & ~File.umask, verbose: false
+    end
+  end
+end

data/lib/generators/ductwork/install/templates/bin/ductwork

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require_relative "../config/environment"
+require "ductwork/cli"
+
+Ductwork::CLI.start!(ARGV)

data/lib/generators/ductwork/install/templates/config/ductwork.yml

@@ -0,0 +1,25 @@
+default: &default
+  pipelines: "*"
+  logger:
+    level: 1
+    source: default
+  job_worker:
+    count: 5
+    max_retry: 3
+    polling_timeout: 1
+    shutdown_timeout: 20
+  pipeline_advancer:
+    polling_timeout: 1
+    shutdown_timeout: 20
+  supervisor:
+    polling_timeout: 1
+    shutdown_timeout: 30
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+
+production:
+  <<: *default

data/lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class CreateDuctworkAvailabilities < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+  def change
+    create_table :ductwork_availabilities do |table|
+      table.belongs_to :execution, index: false, null: false, foreign_key: { to_table: :ductwork_executions }
+      table.timestamp :started_at, null: false
+      table.timestamp :completed_at
+      table.integer :process_id
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_availabilities, :execution_id, unique: true
+    add_index :ductwork_availabilities, %i[id process_id]
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class CreateDuctworkExecutions < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+  def change
+    create_table :ductwork_executions do |table|
+      table.belongs_to :job, index: true, null: false, foreign_key: { to_table: :ductwork_jobs }
+      table.timestamp :started_at, null: false
+      table.timestamp :completed_at
+      table.integer :retry_count, null: false
+      table.integer :process_id
+      table.timestamps null: false
+    end
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_jobs.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class CreateDuctworkJobs < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+  def change
+    create_table :ductwork_jobs do |table|
+      table.belongs_to :step, index: false, null: false, foreign_key: { to_table: :ductwork_steps }
+      table.string :klass, null: false
+      table.timestamp :started_at, null: false
+      table.timestamp :completed_at
+      table.string :input_args, null: false
+      table.string :output_payload
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_jobs, :step_id, unique: true
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class CreateDuctworkPipelines < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+  def change
+    create_table :ductwork_pipelines do |table|
+      table.string :klass, null: false
+      table.string :definition, null: false
+      table.string :definition_sha1, null: false
+      table.timestamp :triggered_at, null: false
+      table.timestamp :completed_at
+      table.timestamp :claimed_for_advancing_at
+      table.timestamp :last_advanced_at, null: false
+      table.string :status, null: false
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_pipelines, :klass
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_processes.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class CreateDuctworkProcesses < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+  def change
+    create_table :ductwork_processes do |table|
+      table.integer :pid, null: false
+      table.string :machine_identifier, null: false
+      table.timestamp :last_heartbeat_at, null: false
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_processes, %i[pid machine_identifier], unique: true
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_results.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class CreateDuctworkResults < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+  def change
+    create_table :ductwork_results do |table|
+      table.belongs_to :execution, index: false, null: false, foreign_key: { to_table: :ductwork_executions }
+      table.string :result_type, null: false
+      table.string :error_klass
+      table.string :error_message
+      table.text :error_backtrace
+      table.timestamps null: false
+    end
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class CreateDuctworkRuns < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+  def change
+    create_table :ductwork_runs do |table|
+      table.belongs_to :execution, index: false, null: false, foreign_key: { to_table: :ductwork_executions }
+      table.timestamp :started_at, null: false
+      table.timestamp :completed_at
+      table.timestamps null: false
+    end
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+  def change
+    create_table :ductwork_steps do |table|
+      table.belongs_to :pipeline, index: true, null: false, foreign_key: { to_table: :ductwork_pipelines }
+      table.string :klass, null: false
+      table.string :step_type, null: false
+      table.timestamp :started_at
+      table.timestamp :completed_at
+      table.string :status, null: false
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_steps, %i[pipeline_id status]
+  end
+end