ductwork 0.25.0 → 1.0.0

data/lib/ductwork/database_clock.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Ductwork
+  # NOTE: these are SQL fragments that resolve against the database server's
+  # clock instead of the calling Ruby process's clock. use them in `WHERE`
+  # clauses that compare between a stored timestamp and "now" so that NTP
+  # drift between hosts cannot make healthy work look stale (or vice versa)
+  class DatabaseClock
+    def self.ago_sql(column, interval)
+      new(column).ago_sql(interval)
+    end
+
+    def self.now_sql(column)
+      new(column).now_sql
+    end
+
+    def self.now
+      adapter = Ductwork::Record.connection.adapter_name.downcase
+      sql =
+        case adapter
+        when /postgresql|cockroach/i
+          "SELECT clock_timestamp()"
+        when /mysql|trilogy/i
+          "SELECT CURRENT_TIMESTAMP(6)"
+        when /sqlite/i
+          "SELECT strftime('%Y-%m-%d %H:%M:%f', 'now')"
+        when /oracle/i
+          "SELECT CURRENT_TIMESTAMP FROM dual"
+        else
+          raise NotImplementedError, "Database clock does not support adapter #{adapter}"
+        end
+      raw = Ductwork::Record.connection.select_value(sql)
+
+      case raw
+      when ::Time, ::DateTime
+        raw.in_time_zone
+      else
+        # NOTE: SQLite returns an ISO-8601 string already expressed in UTC
+        ::Time.find_zone!("UTC").parse(raw.to_s)
+      end
+    end
+
+    def initialize(column)
+      @adapter = Ductwork::Record.connection.adapter_name.downcase
+      @column = column
+    end
+
+    def ago_sql(interval)
+      seconds = interval.to_i
+
+      case adapter
+      when /postgresql|cockroach/i
+        "#{column} <= clock_timestamp() - INTERVAL '#{seconds} seconds'"
+      when /mysql|trilogy/i
+        "#{column} <= CURRENT_TIMESTAMP(6) - INTERVAL #{seconds} SECOND"
+      when /sqlite/i
+        "julianday(#{column}) <= julianday('now', '-#{seconds} seconds')"
+      when /oracle/i
+        "#{column} <= CURRENT_TIMESTAMP - NUMTODSINTERVAL(#{seconds}, 'SECOND')"
+      else
+        raise NotImplementedError, "Database clock does not support adapter #{adapter}"
+      end
+    end
+
+    def now_sql
+      case adapter
+      when /sqlite/i
+        "julianday(#{column}) <= julianday('now')"
+      when /mysql|trilogy/i
+        "#{column} <= CURRENT_TIMESTAMP(6)"
+      when /postgresql|cockroach/i
+        "#{column} <= clock_timestamp()"
+      when /oracle/i
+        "#{column} <= CURRENT_TIMESTAMP"
+      else
+        raise NotImplementedError, "Database clock does not support adapter #{adapter}"
+      end
+    end
+
+    private
+
+    attr_reader :adapter, :column
+  end
+end

data/lib/ductwork/dsl/branch_builder.rb

@@ -11,6 +11,7 @@ module Ductwork
         @last_node = last_node
         @definition = definition
         @expansions = 0
+        @expand_sources = []
       end
 
       def chain(next_klass)
@@ -43,6 +44,31 @@ module Ductwork
         self
       end
 
+      def divert(to:)
+        to_map = {}
+        next_nodes = []
+        to.each do |key, klass|
+          node = node_name(klass)
+          definition[:edges][node] ||= { klass: klass.name }
+          to_map[key.to_s] = node
+          next_nodes << node
+        end
+
+        definition[:edges][last_node][:to] = to_map
+        definition[:edges][last_node][:type] = :divert
+        definition[:nodes].push(*next_nodes)
+
+        if block_given?
+          sub_branches = next_nodes.map do |last_node|
+            Ductwork::DSL::BranchBuilder.new(last_node:, definition:)
+          end
+
+          yield sub_branches
+        end
+
+        self
+      end
+
       def combine(*branch_builders, into:)
         next_klass_name = node_name(into)
         definition[:edges][last_node][:to] = [next_klass_name]
@@ -58,8 +84,26 @@ module Ductwork
         self
       end
 
+      def converge(*branch_builders, into:)
+        next_klass_name = node_name(into)
+        definition[:edges][last_node][:to] = [next_klass_name]
+        definition[:edges][last_node][:type] = :converge
+
+        branch_builders.each do |branch|
+          definition[:edges][branch.last_node][:to] = [next_klass_name]
+          definition[:edges][branch.last_node][:type] = :converge
+        end
+        definition[:nodes].push(next_klass_name)
+        definition[:edges][next_klass_name] ||= { klass: into.name }
+
+        self
+      end
+
       def expand(to:)
         next_klass_name = node_name(to)
+        # NOTE: record the expanding node so the matching `collapse` can stamp it
+        # as `barrier_node` (see `Ductwork::DSL::DefinitionBuilder#expand`).
+        expand_sources.push(last_node)
         definition[:edges][last_node][:to] = [next_klass_name]
         definition[:edges][last_node][:type] = :expand
         definition[:nodes].push(next_klass_name)
@@ -79,6 +123,7 @@ module Ductwork
         next_klass_name = node_name(into)
         definition[:edges][last_node][:to] = [next_klass_name]
         definition[:edges][last_node][:type] = :collapse
+        definition[:edges][last_node][:barrier_node] = expand_sources.pop
 
         definition[:nodes].push(next_klass_name)
         definition[:edges][next_klass_name] ||= { klass: into.name }
@@ -90,7 +135,7 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :expansions
+      attr_reader :definition, :expansions, :expand_sources
 
       def node_name(klass)
         "#{klass.name}.#{SecureRandom.hex(4)}"

data/lib/ductwork/dsl/definition_builder.rb

@@ -2,10 +2,12 @@
 
 module Ductwork
   module DSL
-    class DefinitionBuilder
+    class DefinitionBuilder # rubocop:todo Metrics/ClassLength
       class StartError < StandardError; end
       class CollapseError < StandardError; end
       class CombineError < StandardError; end
+      class DivertError < StandardError; end
+      class ConvergeError < StandardError; end
 
       def initialize
         @definition = {
@@ -15,6 +17,7 @@ module Ductwork
         }
         @divergences = []
         @last_nodes = []
+        @expand_sources = []
       end
 
       def start(klass)
@@ -96,6 +99,13 @@ module Ductwork
       def expand(to:)
         validate_classes!(to)
         validate_definition_started!(action: "expanding chain")
+
+        # NOTE: record the node(s) carrying this `expand` edge before
+        # `add_edge_to_last_nodes` reassigns `@last_nodes`. The matching
+        # `collapse` stamps this on its edge as `barrier_node` so the runtime
+        # fan-in can find the expanding branch (which carries the barrier) no
+        # matter how many transitions sit between the `expand` and `collapse`.
+        expand_sources.push(last_nodes.dup)
         add_edge_to_last_nodes(to, type: :expand)
         divergences.push(:expand)
 
@@ -106,8 +116,73 @@ module Ductwork
         validate_classes!(into)
         validate_definition_started!(action: "collapsing steps")
         validate_can_collapse!
+        collapse_sources = last_nodes.dup
         add_edge_to_last_nodes(into, type: :collapse)
         divergences.pop
+        barrier_node = pop_barrier_node
+        collapse_sources.each do |source|
+          definition[:edges][source][:barrier_node] = barrier_node
+        end
+
+        self
+      end
+
+      def divert(to:) # rubocop:todo Metrics/AbcSize
+        validate_classes!(to.values)
+        validate_definition_started!(action: "diverting chain")
+        validate_fallback_step!(to.keys)
+
+        to_map = {}
+        to_nodes = []
+        to.each do |key, klass|
+          node = node_name(klass)
+          definition[:edges][node] ||= { klass: klass.name }
+          to_map[key.to_s] = node
+          to_nodes << node
+        end
+
+        last_nodes.each do |last_node|
+          definition[:edges][last_node][:to] = to_map
+          definition[:edges][last_node][:type] = :divert
+        end
+
+        @last_nodes = to_nodes
+
+        definition[:nodes].push(*to_nodes)
+        divergences.push(:divert)
+
+        if block_given?
+          branches = last_nodes.map do |last_node|
+            Ductwork::DSL::BranchBuilder
+              .new(last_node:, definition:)
+          end
+
+          yield branches
+        end
+
+        self
+      end
+
+      def converge(into:) # rubocop:todo Metrics/AbcSize
+        validate_classes!(into)
+        validate_definition_started!(action: "converging steps")
+        validate_can_converge!
+
+        divergences.pop
+
+        into_node = node_name(into)
+        definition[:edges][into_node] ||= { klass: into.name }
+        last_nodes = definition[:nodes].reverse.select do |node|
+          definition.dig(:edges, node, :to).blank?
+        end
+        last_nodes.each do |last_node|
+          definition[:edges][last_node][:to] = [into_node]
+          definition[:edges][last_node][:type] = :converge
+        end
+
+        @last_nodes = Array(into_node)
+
+        definition[:nodes].push(into_node)
 
         self
       end
@@ -128,7 +203,17 @@ module Ductwork
 
       private
 
-      attr_reader :definition, :last_nodes, :divergences
+      attr_reader :definition, :last_nodes, :divergences, :expand_sources
+
+      def pop_barrier_node
+        sources = expand_sources.pop || []
+
+        if sources.length == 1
+          sources.first
+        else
+          sources
+        end
+      end
 
       def validate_classes!(klasses)
         valid = Array(klasses).all? do |klass|
@@ -176,6 +261,20 @@ module Ductwork
         end
       end
 
+      def validate_can_converge!
+        if divergences.empty?
+          raise ConvergeError, "Must divert pipeline definition before converging steps"
+        elsif divergences[-1] != :divert
+          raise ConvergeError, "Ambiguous converge on most recently divided/expanded definition"
+        end
+      end
+
+      def validate_fallback_step!(keys)
+        if keys.exclude?(:otherwise)
+          raise DivertError, "Must specify an `otherwise` branch"
+        end
+      end
+
       def add_edge_to_last_nodes(*klasses, type:)
         to_nodes = klasses.map do |klass|
           node = node_name(klass)

data/lib/ductwork/{job_claim.rb → execution_claim.rb}

@@ -1,25 +1,26 @@
 # frozen_string_literal: true
 
 module Ductwork
-  class JobClaim
-    def initialize(klass)
+  class ExecutionClaim
+    def initialize(klass, owner_process_id)
       @klass = klass
+      @owner_process_id = owner_process_id
       @adapter = Ductwork::Record.connection.adapter_name.downcase
     end
 
     def latest
       claim = if supports_row_locking?
-                RowLockingJobClaim
+                RowLockingExecutionClaim
               else
-                OptimisticLockingJobClaim
+                OptimisticLockingExecutionClaim
               end
 
-      claim.new(klass).latest
+      claim.new(klass, owner_process_id).latest
     end
 
     private
 
-    attr_reader :klass, :adapter
+    attr_reader :klass, :owner_process_id, :adapter
 
     def supports_row_locking?
       adapter.match?(/postgresql/i) ||

data/lib/ductwork/fault_injection.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module FaultInjection
+    ENV_KEY = "DUCTWORK_FAULT"
+
+    class << self
+      def checkpoint(key)
+        return if !ENV.key?(ENV_KEY)
+
+        spec = ENV.fetch(ENV_KEY, "").split(":")
+
+        return if spec[0] != key.to_s
+
+        warn "FAULT FIRING: #{spec[0]} -> #{spec[1]} (pid=#{::Process.pid})"
+
+        case spec[1]
+        when "kill" then ::Process.kill(:KILL, ::Process.pid)
+        when "raise" then raise "FaultInjection: #{key}"
+        when "sleep" then sleep(1)
+        when "exit" then exit!(1)
+        end
+      end
+
+      def with(key, action)
+        previous_value = ENV.fetch(ENV_KEY, nil)
+        ENV[ENV_KEY] = "#{key}:#{action}"
+
+        yield
+      ensure
+        ENV[ENV_KEY] = previous_value
+      end
+    end
+  end
+end

data/lib/ductwork/migration_helper.rb

@@ -36,7 +36,7 @@ module Ductwork
     end
 
     def mysql?
-      connection.adapter_name.match?(/mysql/i)
+      connection.adapter_name.match?(/mysql|trilogy/i)
     end
   end
 end

data/lib/ductwork/models/advancement.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Advancement < Ductwork::Record
+    belongs_to :process, class_name: "Ductwork::Process", optional: true
+    belongs_to :transition, class_name: "Ductwork::Transition"
+
+    validates :started_at, presence: true
+
+    CRASH_ERROR_KLASSES = %w[Ductwork::ProcessCrash Ductwork::ThreadCrash].freeze
+
+    def crash?
+      CRASH_ERROR_KLASSES.include?(error_klass)
+    end
+
+    def process_crashed!
+      Ductwork::Record.transaction do
+        rows_updated = self.class
+                           .where(id: id, completed_at: nil)
+                           .update_all(
+                             completed_at: Time.current,
+                             error_klass: "Ductwork::ProcessCrash",
+                             error_message: "Reaped from orphaned process"
+                           )
+
+        return if rows_updated.zero?
+
+        transition.branch.release!
+      end
+    end
+
+    def thread_crashed!(expected_token)
+      Ductwork::Record.transaction do
+        rows_updated = self.class
+                           .where(id: id, completed_at: nil)
+                           .update_all(
+                             completed_at: Time.current,
+                             error_klass: "Ductwork::ThreadCrash",
+                             error_message: "Advancement abandoned from a thread crash"
+                           )
+
+        return if rows_updated.zero?
+
+        transition.branch.release!(expected_token)
+      end
+    end
+  end
+end

data/lib/ductwork/models/attempt.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Attempt < Ductwork::Record
+    belongs_to :execution, class_name: "Ductwork::Execution"
+
+    validates :started_at, presence: true
+  end
+end

data/lib/ductwork/models/availability.rb

@@ -3,7 +3,9 @@
 module Ductwork
   class Availability < Ductwork::Record
     belongs_to :execution, class_name: "Ductwork::Execution"
+    belongs_to :process, class_name: "Ductwork::Process", optional: true
 
     validates :started_at, presence: true
+    validates :pipeline_klass, presence: true
   end
 end