ductwork 0.14.1 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99db2113458db663a866d26859c59ba4c1672d6a9180a89662834ef1c667ea58
-  data.tar.gz: cc0c38a5202a791db820efca5501cf31bd1cb71f214dfae8ed9a7b831d442788
+  metadata.gz: cd378a2cedabe241b11559ecc7611a6ca8aa631ca2739a97199fdf4a34387f4e
+  data.tar.gz: 3faddccd331cd7bd94cc7e1487a4f4f823fc0037ca7109bf2c093ac357488263
 SHA512:
-  metadata.gz: 7316e5d9d6b103a754e2f4fa12a9e7b33474fbbcb6522c75b8da8cf2e514c8bba61a84ecd801428093a4dc77395a358c30a59e47f1f146c99b489fa2e9f6f64c
-  data.tar.gz: a14268df21d58fcc1e92de2a01d475d0fe14b8974a9a0da7fef49be84a201e574f743d5ed999f7d0b46150f6da837d225c0b5c000fb3a40c528d2c66b27c7a28
+  metadata.gz: eb6051a653d7aa5f9dbf4cc0784ed69d364784fb14d3d24120683616eb2601a4d4f97c028c2cc8df137c2a992429af450f63589f6381977dcbabc68eb926314f
+  data.tar.gz: 8e484847a038021d399272e3565a58b4204f2752d346b515ee59503204fae7f6ee3756c780376b4df03dd73d960ef191c484dd0bd2ec33e384b268272a63aae8

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Ductwork Changelog
 
+## [0.15.0]
+
+- chore: remove unnecessary transaction in job enqueueing method
+- feat: expose pipeline and step classes validation method instead of running in a rails initializer
+- feat!: introduce pipeline Context - this is a BREAKING CHANGE because step classes now need to inherit from `Ductwork::Step`; they are no longer POROs
+
 ## [0.14.1]
 
 - fix: check if steps and pipelines directories exist before eager loading

data/README.md

@@ -7,6 +7,8 @@ A Ruby pipeline framework.
 
 Ductwork lets you build complex pipelines quickly and easily using intuitive Ruby tooling and a natural DSL. No need to learn complicated unified object models or stand up separate runner instances—just write Ruby code and let Ductwork handle the orchestration.
 
+There is also a paid [Ductwork Pro](https://www.getductwork.io/) version with more features and support. See the [Pricing](https://www.getductwork.io/#pricing) page to buy a license.
+
 **[Full Documentation](https://docs.getductwork.io/)**
 
 ## Installation
@@ -60,7 +62,7 @@ end
 
 ### 2. Define Steps
 
-Steps are Plain Old Ruby Objects (POROs) that implement two methods:
+Steps are Ruby objects that inherit from `Ductwork::Step` and implement two methods:
 - `initialize` - accepts parameters from the trigger call or previous step's return value
 - `execute` - performs the work and returns data for the next step
 
@@ -68,7 +70,7 @@ Steps live in `app/steps`:
 
 ```ruby
 # app/steps/users_requiring_enrichment.rb
-class UsersRequiringEnrichment
+class QueryUsersRequiringEnrichment < Ductwork::Step
   def initialize(days_outdated)
     @days_outdated = days_outdated
   end
@@ -90,7 +92,7 @@ Connect steps together using Ductwork's fluent interface DSL. The key principle:
 ```ruby
 class EnrichUserDataPipeline < Ductwork::Pipeline
   define do |pipeline|
-    pipeline.start(UsersRequiringEnrichment)       # Start with a single step
+    pipeline.start(QueryUsersRequiringEnrichment)  # Start with a single step
             .expand(to: LoadUserData)              # Fan out to multiple steps
             .divide(to: [FetchDataFromSourceA,     # Split into parallel branches
                          FetchDataFromSourceB])

data/lib/ductwork/context.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Context
+    class OverwriteError < StandardError; end
+
+    def initialize(pipeline_id)
+      @pipeline_id = pipeline_id
+    end
+
+    def get(key)
+      raise ArgumentError, "Key must be a string" if !key.is_a?(String)
+
+      Ductwork.wrap_with_app_executor do
+        Ductwork::Tuple
+          .select(:serialized_value)
+          .find_by(pipeline_id:, key:)
+          &.value
+      end
+    end
+
+    def set(key, value, overwrite: false)
+      attributes = {
+        pipeline_id: pipeline_id,
+        key: key,
+        serialized_value: Ductwork::Tuple.serialize(value),
+        first_set_at: Time.current,
+        last_set_at: Time.current,
+      }
+      unique_by = %i[pipeline_id key]
+
+      if overwrite
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Tuple.upsert(attributes, unique_by:)
+        end
+      else
+        result = Ductwork.wrap_with_app_executor do
+          Ductwork::Tuple.insert(attributes, unique_by:)
+        end
+
+        if result.rows.none?
+          raise Ductwork::Context::OverwriteError, "Can only set value once"
+        end
+      end
+
+      value
+    end
+
+    private
+
+    attr_reader :pipeline_id
+  end
+end

data/lib/ductwork/engine.rb

@@ -23,23 +23,5 @@ module Ductwork
       Ductwork.configuration ||= Ductwork::Configuration.new
       Ductwork.logger ||= Ductwork::Configuration::DEFAULT_LOGGER
     end
-
-    initializer "ductwork.validate_definitions", after: :load_config_initializers do
-      config.after_initialize do
-        # Load steps and pipelines so definition validation runs and bugs
-        # can be caught simply by booting the app or running tests
-        loader = Rails.autoloaders.main
-        step_directory = Rails.root.join("app/steps")
-        pipeline_directory = Rails.root.join("app/pipelines")
-
-        if step_directory.exist?
-          loader.eager_load_dir(step_directory)
-        end
-
-        if pipeline_directory.exist?
-          loader.eager_load_dir(pipeline_directory)
-        end
-      end
-    end
   end
 end

data/lib/ductwork/models/job.rb

@@ -68,22 +68,18 @@ module Ductwork
     end
 
     def self.enqueue(step, args)
-      job = Ductwork::Record.transaction do
-        j = step.create_job!(
-          klass: step.klass,
-          started_at: Time.current,
-          input_args: JSON.dump({ args: })
-        )
-        execution = j.executions.create!(
-          started_at: Time.current,
-          retry_count: 0
-        )
-        execution.create_availability!(
-          started_at: Time.current
-        )
-
-        j
-      end
+      job = step.create_job!(
+        klass: step.klass,
+        started_at: Time.current,
+        input_args: JSON.dump({ args: })
+      )
+      execution = job.executions.create!(
+        started_at: Time.current,
+        retry_count: 0
+      )
+      execution.create_availability!(
+        started_at: Time.current
+      )
 
       Ductwork.logger.info(
         msg: "Job enqueued",
@@ -104,7 +100,7 @@ module Ductwork
         job_klass: klass
       )
       args = JSON.parse(input_args)["args"]
-      instance = Object.const_get(klass).new(args)
+      instance = Object.const_get(klass).build_for_execution(step, args)
       run = execution.create_run!(
         started_at: Time.current
       )

data/lib/ductwork/models/pipeline.rb

@@ -3,6 +3,7 @@
 module Ductwork
   class Pipeline < Ductwork::Record # rubocop:todo Metrics/ClassLength
     has_many :steps, class_name: "Ductwork::Step", foreign_key: "pipeline_id", dependent: :destroy
+    has_many :tuples, class_name: "Ductwork::Tuple", foreign_key: "pipeline_id", dependent: :destroy
 
     validates :klass, presence: true
     validates :definition, presence: true
@@ -267,6 +268,9 @@ module Ductwork
       if max_depth != -1 && return_value.count > max_depth
         halt!
       else
+        # TODO: Brainstorm on using `insert_all` instead of iterating.
+        # Performance is bad when the return value has a lot of elements
+        # and we create a step and job individually
         Array(return_value).each do |input_arg|
           create_step_and_enqueue_job(edge:, input_arg:)
         end

data/lib/ductwork/models/step.rb

@@ -25,5 +25,20 @@ module Ductwork
          combine: "combine",
          expand: "expand",
          collapse: "collapse"
+
+    def self.build_for_execution(record, *args, **kwargs)
+      instance = allocate
+      instance.instance_variable_set(:@pipeline_id, record.pipeline_id)
+      instance.send(:initialize, *args, **kwargs)
+      instance
+    end
+
+    def pipeline_id
+      @pipeline_id || (@attributes && super)
+    end
+
+    def context
+      @_context ||= Ductwork::Context.new(pipeline_id)
+    end
   end
 end

data/lib/ductwork/models/tuple.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Tuple < Ductwork::Record
+    belongs_to :pipeline, class_name: "Ductwork::Pipeline"
+
+    # NOTE: These model-level validations are here for symmetry. The current
+    # methods used to create `Tuple` records skip validations lol
+    validates :key, presence: true
+    validates :key, uniqueness: { scope: :pipeline_id }
+    validates :first_set_at, presence: true
+    validates :last_set_at, presence: true
+
+    def self.serialize(raw_value)
+      { raw_value: }.to_json
+    end
+
+    def value=(raw_value)
+      self.serialized_value = self.class.serialize(raw_value)
+    end
+
+    def value
+      if serialized_value.present?
+        JSON
+          .parse(serialized_value, symbolize_names: true)
+          .fetch(:raw_value)
+      end
+    end
+  end
+end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.14.1"
+  VERSION = "0.15.0"
 end

data/lib/ductwork.rb

@@ -62,6 +62,26 @@ module Ductwork
       @defined_pipelines ||= []
     end
 
+    def validate!
+      step_directory = Rails.root.join("app/steps")
+      pipeline_directory = Rails.root.join("app/pipelines")
+      loader = if defined?(Rails.autoloaders)
+                 Rails.autoloaders.main
+               else
+                 Ductwork.loader
+               end
+
+      if step_directory.exist?
+        loader.eager_load_dir(step_directory)
+      end
+
+      if pipeline_directory.exist?
+        loader.eager_load_dir(pipeline_directory)
+      end
+
+      true
+    end
+
     private
 
     def add_lifecycle_hook(target, event, block)

data/lib/generators/ductwork/install/install_generator.rb

@@ -31,6 +31,8 @@ module Ductwork
                          "db/migrate/create_ductwork_results.rb"
       migration_template "db/create_ductwork_processes.rb",
                          "db/migrate/create_ductwork_processes.rb"
+      migration_template "db/create_ductwork_tuples.rb",
+                         "db/migrate/create_ductwork_tuples.rb"
 
       route <<~ROUTE
         # This mounts the web dashboard. It is recommended to add authentication around it.

data/lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class CreateDuctworkTuples < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+  def change
+    create_table :ductwork_tuples do |table|
+      table.belongs_to :pipeline, index: true, null: false, foreign_key: { to_table: :ductwork_pipelines }
+      table.string :key, null: false
+      table.string :serialized_value
+      table.datetime :first_set_at, null: false
+      table.datetime :last_set_at, null: false
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_tuples, %i[key pipeline_id], unique: true
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.15.0
 platform: ruby
 authors:
 - Tyler Ewing
@@ -135,6 +135,7 @@ files:
 - lib/ductwork.rb
 - lib/ductwork/cli.rb
 - lib/ductwork/configuration.rb
+- lib/ductwork/context.rb
 - lib/ductwork/dsl/branch_builder.rb
 - lib/ductwork/dsl/definition_builder.rb
 - lib/ductwork/engine.rb
@@ -148,6 +149,7 @@ files:
 - lib/ductwork/models/result.rb
 - lib/ductwork/models/run.rb
 - lib/ductwork/models/step.rb
+- lib/ductwork/models/tuple.rb
 - lib/ductwork/processes/job_worker.rb
 - lib/ductwork/processes/job_worker_runner.rb
 - lib/ductwork/processes/pipeline_advancer.rb
@@ -172,6 +174,7 @@ files:
 - lib/generators/ductwork/install/templates/db/create_ductwork_results.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb
+- lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb
 homepage: https://github.com/ductwork/ductwork
 licenses:
 - LGPL-3.0-or-later