ductwork 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 252ad8643950e6ea9e5024d0e63885aea8c3c7020bef1586faf0b414b41206f0
-  data.tar.gz: 1133562c0ff6c84c6a91c87e3d73f734e01626842b42c31e49d87127aa25cd7d
+  metadata.gz: 8ab7c6e79c26352d990680a3913a26ea8bd4bc9999136b4ee1a1846c06e74523
+  data.tar.gz: 7adb0603119c80d5edc54e94de44828bba2c927edad418e5944d594ba21f734a
 SHA512:
-  metadata.gz: 36460be6d4d35656be5ce2cc0006eaa539f00992c6b32f0c07610d42bc889ea0e198e17d9c405bdfca73469895f9d3076fda4335aea6ad7e6dc5011de4e41612
-  data.tar.gz: ddb569815f9ac837881fa08b13541084a5048a34fceafe718026fc228966b35377d16af56edddd5dd74a8cf210a76a3b0986e40644a0d3d3d4efb2db91b475f6
+  metadata.gz: 83be2952cf26ebbee28521df07b68ee29e87ade164a70a87ede7c7bcfc1ef5525097b1346757d76e34d4fc25535c5489f56900fa763006d923bd8491dcba520f
+  data.tar.gz: 53cabb6cdfa4f96f847ddc68818a3f0a87d59ca443282700eca5f690a8a01805e46dbd925990565630bb5c45dc728183de251629488d055f8062d136f06dff7e

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Ductwork Changelog
 
+## [0.2.1]
+
+- fix: do not splat arguments when executing a job nor triggering a pipeline
+- fix: do not splat arguments when enqueuing a job and fix related spec
+- fix: add missing `dependent: :destroy` on certain associations
+
+## [0.2.0]
+
+- feat: validate all pipeline definitions on rails boot
+- feat: validate argument(s) passed to step transition DSL methods to be valid step class
+- fix: allow steps to be chained while pipeline is expanded or divided (before collapsing or combining) - before this incorrectly raised a `CollapseError` or `CombineError`
+
 ## [0.1.0]
 
 - Initial release - see [documentation](https://docs.getductwork.io/) for details

data/README.md

@@ -1,5 +1,8 @@
 # Ductwork
 
+[![CI](https://github.com/ductwork/ductwork/actions/workflows/main.yml/badge.svg)](https://github.com/ductwork/ductwork/actions/workflows/main.yml)
+[![Gem Version](https://badge.fury.io/rb/ductwork.svg?icon=si%3Arubygems)](https://rubygems.org/gems/ductwork)
+
 A Ruby pipeline framework.
 
 Ductwork lets you build complex pipelines quickly and easily using intuitive Ruby tooling and a natural DSL. No need to learn complicated unified object models or stand up separate runner instances—just write Ruby code and let Ductwork handle the orchestration.
@@ -72,7 +75,7 @@ class UsersRequiringEnrichment
 
   def execute
     ids = User.where("data_last_refreshed_at < ?", @days_outdated.days.ago).ids
-    Rails.logger.info("Enriching #{ids.length} users' data")
+    Ductwork.logger.info("Enriching #{ids.length} users' data")
 
     # Return value becomes input to the next step
     ids

data/app/models/ductwork/execution.rb

@@ -3,9 +3,9 @@
 module Ductwork
   class Execution < Ductwork::Record
     belongs_to :job, class_name: "Ductwork::Job"
-    has_one :availability, class_name: "Ductwork::Availability", foreign_key: "execution_id"
-    has_one :run, class_name: "Ductwork::Run", foreign_key: "execution_id"
-    has_one :result, class_name: "Ductwork::Result", foreign_key: "execution_id"
+    has_one :availability, class_name: "Ductwork::Availability", foreign_key: "execution_id", dependent: :destroy
+    has_one :run, class_name: "Ductwork::Run", foreign_key: "execution_id", dependent: :destroy
+    has_one :result, class_name: "Ductwork::Result", foreign_key: "execution_id", dependent: :destroy
 
     validates :retry_count, presence: true
     validates :started_at, presence: true

data/app/models/ductwork/job.rb

@@ -60,7 +60,7 @@ module Ductwork
       end
     end
 
-    def self.enqueue(step, *args)
+    def self.enqueue(step, args)
       Ductwork::Record.transaction do
         job = step.create_job!(
           klass: step.klass,
@@ -89,7 +89,7 @@ module Ductwork
         job_klass: klass
       )
       args = JSON.parse(input_args)["args"]
-      instance = Object.const_get(klass).new(*args)
+      instance = Object.const_get(klass).new(args)
       run = execution.create_run!(
         started_at: Time.current
       )

data/app/models/ductwork/pipeline.rb

@@ -48,7 +48,7 @@ module Ductwork
         Ductwork.defined_pipelines << name.to_s
       end
 
-      def trigger(*args)
+      def trigger(args)
         if pipeline_definition.nil?
           raise DefinitionError, "Pipeline must be defined before triggering"
         end
@@ -71,7 +71,7 @@ module Ductwork
             step_type: :start,
             started_at: Time.current
           )
-          Ductwork::Job.enqueue(step, *args)
+          Ductwork::Job.enqueue(step, args)
 
           pipeline
         end

data/app/models/ductwork/step.rb

@@ -3,7 +3,7 @@
 module Ductwork
   class Step < Ductwork::Record
     belongs_to :pipeline, class_name: "Ductwork::Pipeline"
-    has_one :job, class_name: "Ductwork::Job", foreign_key: "step_id"
+    has_one :job, class_name: "Ductwork::Job", foreign_key: "step_id", dependent: :destroy
 
     validates :klass, presence: true
     validates :status, presence: true

data/lib/ductwork/dsl/definition_builder.rb

@@ -12,9 +12,12 @@ module Ductwork
           nodes: [],
           edges: {},
         }
+        @divisions = 0
+        @expansions = 0
       end
 
       def start(klass)
+        validate_classes!(klass)
         validate_start_once!
         add_new_nodes(klass)
 
@@ -24,6 +27,7 @@ module Ductwork
       # NOTE: there is a bug here that does not allow the user to reuse step
       # classes in the same pipeline. i'll fix this later
       def chain(klass)
+        validate_classes!(klass)
         validate_definition_started!(action: "chaining")
         add_edge_to_last_node(klass, type: :chain)
         add_new_nodes(klass)
@@ -32,10 +36,13 @@ module Ductwork
       end
 
       def divide(to:)
+        validate_classes!(to)
         validate_definition_started!(action: "dividing chain")
         add_edge_to_last_node(*to, type: :divide)
         add_new_nodes(*to)
 
+        @divisions += 1
+
         if block_given?
           branches = to.map do |klass|
             Ductwork::DSL::BranchBuilder
@@ -49,9 +56,12 @@ module Ductwork
       end
 
       def combine(into:)
+        validate_classes!(into)
         validate_definition_started!(action: "combining steps")
         validate_definition_divided!
 
+        @divisions -= 1
+
         last_nodes = definition[:nodes].reverse.select do |node|
           definition[:edges][node].empty?
         end
@@ -67,23 +77,31 @@ module Ductwork
       end
 
       def expand(to:)
+        validate_classes!(to)
         validate_definition_started!(action: "expanding chain")
         add_edge_to_last_node(to, type: :expand)
         add_new_nodes(to)
 
+        @expansions += 1
+
         self
       end
 
       def collapse(into:)
+        validate_classes!(into)
         validate_definition_started!(action: "collapsing steps")
         validate_definition_expanded!
         add_edge_to_last_node(into, type: :collapse)
         add_new_nodes(into)
 
+        @expansions -= 1
+
         self
       end
 
       def on_halt(klass)
+        validate_classes!(klass)
+
         definition[:metadata] ||= {}
         definition[:metadata][:on_halt] = {}
         definition[:metadata][:on_halt][:klass] = klass.name
@@ -99,7 +117,25 @@ module Ductwork
 
       private
 
-      attr_reader :definition
+      attr_reader :definition, :divisions, :expansions
+
+      def validate_classes!(klasses)
+        valid = Array(klasses).all? do |klass|
+          klass.is_a?(Class) &&
+            klass.method_defined?(:execute) &&
+            klass.instance_method(:execute).arity.zero?
+        end
+
+        if !valid
+          word = if Array(klasses).length > 1
+                   "Arguments"
+                 else
+                   "Argument"
+                 end
+
+          raise ArgumentError, "#{word} must be a valid step class"
+        end
+      end
 
       def validate_start_once!
         if definition[:nodes].any?
@@ -114,25 +150,17 @@ module Ductwork
       end
 
       def validate_definition_divided!
-        if last_edge.nil? || last_edge[:type] != :divide
+        if divisions.zero?
           raise CombineError, "Must divide pipeline definition before combining steps"
         end
       end
 
       def validate_definition_expanded!
-        if last_edge.nil? || last_edge[:type] != :expand
+        if expansions.zero?
           raise CollapseError, "Must expand pipeline definition before collapsing steps"
         end
       end
 
-      def last_edge
-        last_edge_node = definition[:nodes].reverse.find do |node|
-          definition[:edges][node].any?
-        end
-
-        definition.dig(:edges, last_edge_node, -1)
-      end
-
       def add_new_nodes(*klasses)
         definition[:nodes].push(*klasses.map(&:name))
         klasses.each do |klass|

data/lib/ductwork/engine.rb

@@ -10,5 +10,15 @@ module Ductwork
       Ductwork.configuration ||= Ductwork::Configuration.new
       Ductwork.configuration.logger ||= Rails.logger
     end
+
+    initializer "ductwork.validate_definitions", after: :load_config_initializers do
+      ActiveSupport.on_load(:active_record) do
+        # Load steps and pipelines so definition validation runs and bugs
+        # can be caught simply by booting the app or running tests
+        loader = Rails.autoloaders.main
+        loader.eager_load_dir(Rails.root.join("app/steps"))
+        loader.eager_load_dir(Rails.root.join("app/pipelines"))
+      end
+    end
   end
 end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.1.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Tyler Ewing