ductwork 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18ebf23ac85cb80300ecc0c35475e0a2364fa4296a6012fdc21e95b8c5943023
-  data.tar.gz: e08b2cda1074255a7b7a79c5fbe789ac373909eef57b7836be49b3f7554a4350
+  metadata.gz: fb1c47566dc19fe61a94c69c48cca984c8844952ce6c0b48917aa479f43f6709
+  data.tar.gz: d0a58e6426f2ca032b70ed3228e7e3de0a540f60693d8a2fdd4ad200235bb8ff
 SHA512:
-  metadata.gz: 9d3b28265bfdf2f90210c43e2609d4255dda6d120ab597b74e54e9c020cbdc4be0b07235bb104873a62ae309511fc841c7d236f20dd52124150760c8dfaa94c8
-  data.tar.gz: fb6db1e8385b880b942404a25aba9ef02637d0c1a2dd53b1a6ddd78a8c70e6494c6349b8d7bbf837727cee5d2ed2c2e2afd67fe506c0be193797fc2de8867619
+  metadata.gz: 1ff44e54ab30c89ebfadf3402ce79b11a8ca7704e8b07a06e8434f07af8216b18bf2db63c2d94cf352edcbc490545a8c3a5bb608f3807a37df35af6f5562e617
+  data.tar.gz: e1c09a239bb49b17143a3f4fb4f248dcbe25c9d5602e763db89bb7ff0411a6233b761ac59e30b5e03e7d0354253924871e3e1510757f3155bc56100954a1f7d3

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Ductwork Changelog
 
+## [0.6.0]
+
+- feat: expose `Ductwork.eager_load` method for eager loading code via `zeitwerk`
+- chore: let `zeitwerk` autoload models from `lib/models` directory instead of letting rails autoload them from the `app/models` directory via the rails engine
+- feat: add `started_at` column to `ductwork_pipelines` table - for now, this will only be used in Pro features.
+
+## [0.5.0]
+
+- chore: add "waiting" status to `Step` model
+- chore: add "waiting" status to `Pipeline` model
+- fix: change `jobs.input_args` and `jobs.output_payload` column type to `text`
+- fix: change `pipelines.definition` column type to `text` - this prevents larger definitions from being clipped if there is a size limit on the string column
+- feat: add missing unique index on `ductwork_results` and `ductwork_runs` tables
+- feat: add missing composite index on `ductwork_executions` table for `Ductwork::Job.claim_latest` method
+- feat: add missing composite index on `ductwork_availabilities` table for `Ductwork::Job.claim_latest` method
+- feat: use array instead of ActiveRecord relation when advancing pipelines - this has major performance benefits but comes with memory-usage implications (see comments)
+- fix: add condition to query to return correct pipelines that need advancing
+- fix: release pipeline claim only if successfully claimed
+- chore: add pipeline ID to misc log lines
+- feat: add missing composite indexes on `ductwork_steps` table
+
 ## [0.4.0]
 
 - chore: change job worker thread name format

data/{app/models/ductwork → lib/ductwork/models}/pipeline.rb

@@ -8,12 +8,14 @@ module Ductwork
     validates :definition, presence: true
     validates :definition_sha1, presence: true
     validates :status, presence: true
+    validates :started_at, presence: true
     validates :triggered_at, presence: true
     validates :last_advanced_at, presence: true
 
     enum :status,
          pending: "pending",
          in_progress: "in_progress",
+         waiting: "waiting",
          halted: "halted",
          completed: "completed"
 
@@ -63,6 +65,7 @@ module Ductwork
             definition: definition,
             definition_sha1: Digest::SHA1.hexdigest(definition),
             triggered_at: Time.current,
+            started_at: Time.current,
             last_advanced_at: Time.current
           )
           step = p.steps.create!(
@@ -87,22 +90,19 @@ module Ductwork
     end
 
     def advance!
-      # NOTE: there could be A LOT of steps advancing for a single pipeline
-      # so instead of loading everything into memory and using ruby collection
-      # methods we make multiple queries. may need to revisist this once
-      # we do extensive load testing
-      advancing = steps.advancing
-      edges = if advancing.exists?
-                parsed_definition
-                  .fetch(:edges, {})
-                  .select { |k| k.in?(advancing.pluck(:klass)) }
-              end
+      # NOTE: if we've expanded the pipeline there could be a lot of
+      # advancing records which may cause memory issues. something to
+      # watch out for here and maybe add in config to use AR relation
+      # at certain counts or even memory limits.
+      advancing_steps = steps.advancing.pluck(:id, :klass)
+      advancing_ids = advancing_steps.map(&:first)
+      edges = find_edges(advancing_steps)
 
       Ductwork::Record.transaction do
         if edges.nil? || edges.values.all?(&:empty?)
-          conditionally_complete_pipeline(advancing)
+          conditionally_complete_pipeline(advancing_ids)
         else
-          advance_to_next_steps_by_type(edges, advancing)
+          advance_to_next_steps_by_type(edges, advancing_ids)
         end
       end
     end
@@ -120,10 +120,25 @@ module Ductwork
       Ductwork::Job.enqueue(next_step, input_arg)
     end
 
-    def conditionally_complete_pipeline(advancing)
-      advancing.update!(status: :completed, completed_at: Time.current)
+    def find_edges(advancing_steps)
+      if advancing_steps.any?
+        klasses = advancing_steps.map(&:last)
 
-      if steps.where(status: %w[in_progress pending]).none?
+        parsed_definition.fetch(:edges, {}).select { |k| k.in?(klasses) }
+      end
+    end
+
+    def conditionally_complete_pipeline(advancing_ids)
+      steps
+        .where(id: advancing_ids)
+        .update_all(status: :completed, completed_at: Time.current)
+
+      remaining = steps
+                  .where(status: %w[in_progress pending advancing])
+                  .where.not(id: advancing_ids)
+                  .exists?
+
+      if !remaining
         update!(status: :completed, completed_at: Time.current)
 
         Ductwork.logger.info(
@@ -134,9 +149,11 @@ module Ductwork
       end
     end
 
-    def advance_to_next_steps_by_type(edges, advancing)
+    def advance_to_next_steps_by_type(edges, advancing_ids)
+      steps.where(id: advancing_ids).update_all(status: :completed, completed_at: Time.current)
+
       if edges.all? { |_, v| v.dig(-1, :type) == "combine" }
-        conditionally_combine_next_steps(edges, advancing)
+        conditionally_combine_next_steps(edges, advancing_ids)
       else
         edges.each do |step_klass, step_edges|
           edge = step_edges[-1]
@@ -145,27 +162,25 @@ module Ductwork
           step_type = edge[:type] == "chain" ? "default" : edge[:type]
 
           if step_type == "collapse"
-            conditionally_collapse_next_steps(step_klass, edge, advancing)
+            conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
           else
-            advance_non_merging_steps(step_klass, edges, advancing)
+            advance_non_merging_steps(step_klass, edge, advancing_ids)
           end
         end
       end
-      advancing.update!(status: :completed, completed_at: Time.current)
       log_pipeline_advanced(edges)
     end
 
-    def advance_non_merging_steps(step_klass, edges, advancing)
-      advancing.where(klass: step_klass).find_each do |step|
-        edge = edges.dig(step.klass, -1)
-        # NOTE: "chain" is used by ActiveRecord so we have to call
-        # this enum value "default" :sad:
-        step_type = edge[:type] == "chain" ? "default" : edge[:type]
+    def advance_non_merging_steps(step_klass, edge, advancing_ids)
+      # NOTE: "chain" is used by ActiveRecord so we have to call
+      # this enum value "default" :sad:
+      step_type = edge[:type] == "chain" ? "default" : edge[:type]
 
+      steps.where(id: advancing_ids, klass: step_klass).find_each do |step|
         if step_type.in?(%w[default divide])
-          advance_to_next_steps(step_type, advancing, edge)
+          advance_to_next_steps(step_type, step.id, edge)
         elsif step_type == "expand"
-          expand_to_next_steps(step_type, advancing, edge)
+          expand_to_next_steps(step_type, step.id, edge)
         else
           Ductwork.logger.error(
             msg: "Invalid step type",
@@ -177,7 +192,7 @@ module Ductwork
       end
     end
 
-    def advance_to_next_steps(step_type, advancing, edge)
+    def advance_to_next_steps(step_type, step_id, edge)
       too_many = edge[:to].tally.any? do |to_klass, count|
         depth = Ductwork
                 .configuration
@@ -196,14 +211,15 @@ module Ductwork
             step_type: step_type,
             started_at: Time.current
           )
-          Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
+          return_value = Ductwork::Job.find_by(step_id:).return_value
+          Ductwork::Job.enqueue(next_step, return_value)
         end
       end
     end
 
-    def conditionally_combine_next_steps(edges, advancing)
+    def conditionally_combine_next_steps(edges, advancing_ids)
       if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
-        combine_next_steps(edges, advancing)
+        combine_next_steps(edges, advancing_ids)
       else
         Ductwork.logger.debug(
           msg: "Not all divided steps have completed; not combining",
@@ -213,14 +229,15 @@ module Ductwork
       end
     end
 
-    def combine_next_steps(edges, advancing)
+    def combine_next_steps(edges, advancing_ids)
       klass = edges.values.sample.dig(-1, :to).sole
       step_type = "combine"
-      groups = advancing
+      groups = steps
+               .where(id: advancing_ids)
                .group(:klass)
                .count
                .keys
-               .map { |k| advancing.where(klass: k) }
+               .map { |k| steps.where(id: advancing_ids).where(klass: k) }
 
       groups.first.zip(*groups[1..]).each do |group|
         input_arg = Ductwork::Job
@@ -230,9 +247,11 @@ module Ductwork
       end
     end
 
-    def expand_to_next_steps(step_type, advancing, edge)
+    def expand_to_next_steps(step_type, step_id, edge)
       next_klass = edge[:to].sole
-      return_value = advancing.take.job.return_value
+      return_value = Ductwork::Job
+                     .find_by(step_id:)
+                     .return_value
       max_depth = Ductwork.configuration.steps_max_depth(pipeline: klass, step: next_klass)
 
       if max_depth != -1 && return_value.count > max_depth
@@ -248,9 +267,9 @@ module Ductwork
       end
     end
 
-    def conditionally_collapse_next_steps(step_klass, edge, advancing)
+    def conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
       if steps.where(status: %w[pending in_progress], klass: step_klass).none?
-        collapse_next_steps(edge[:to].sole, advancing)
+        collapse_next_steps(edge[:to].sole, advancing_ids)
       else
         Ductwork.logger.debug(
           msg: "Not all expanded steps have completed; not collapsing",
@@ -260,14 +279,11 @@ module Ductwork
       end
     end
 
-    def collapse_next_steps(klass, advancing)
+    def collapse_next_steps(klass, advancing_ids)
       step_type = "collapse"
       input_arg = []
 
-      # NOTE: because of expanding based on return values, there
-      # could be A LOT of jobs so we want to use batch methods
-      # to avoid creating too many in-memory objects
-      Ductwork::Job.where(step_id: advancing.ids).find_each do |job|
+      Ductwork::Job.where(step_id: advancing_ids).find_each do |job|
         input_arg << job.return_value
       end
 

data/{app/models/ductwork → lib/ductwork/models}/step.rb

@@ -12,6 +12,7 @@ module Ductwork
     enum :status,
          pending: "pending",
          in_progress: "in_progress",
+         waiting: "waiting",
          advancing: "advancing",
          failed: "failed",
          completed: "completed"

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -15,6 +15,7 @@ module Ductwork
             Ductwork::Pipeline
               .in_progress
               .where(klass: klass, claimed_for_advancing_at: nil)
+              .where(steps: Ductwork::Step.where(status: :advancing))
               .where.not(steps: Ductwork::Step.where.not(status: %w[advancing completed]))
               .order(:last_advanced_at)
               .limit(1)
@@ -30,6 +31,7 @@ module Ductwork
             if rows_updated == 1
               Ductwork.logger.debug(
                 msg: "Pipeline claimed",
+                pipeline_id: id,
                 pipeline: klass,
                 role: :pipeline_advancer
               )
@@ -39,28 +41,29 @@ module Ductwork
 
               Ductwork.logger.debug(
                 msg: "Pipeline advanced",
+                pipeline_id: id,
                 pipeline: klass,
                 role: :pipeline_advancer
               )
+
+              # release the pipeline and set last advanced at so it doesnt block.
+              # we're not using a queue so we have to use a db timestamp
+              pipeline.update!(
+                claimed_for_advancing_at: nil,
+                last_advanced_at: Time.current
+              )
             else
               Ductwork.logger.debug(
                 msg: "Did not claim pipeline, avoided race condition",
+                pipeline_id: id,
                 pipeline: klass,
                 role: :pipeline_advancer
               )
             end
-
-            # release the pipeline and set last advanced at so it doesnt block.
-            # we're not using a queue so we have to use a db timestamp
-            Ductwork::Pipeline.find(id).update!(
-              claimed_for_advancing_at: nil,
-              last_advanced_at: Time.current
-            )
           else
             Ductwork.logger.debug(
               msg: "No pipeline needs advancing",
               pipeline: klass,
-              id: id,
               role: :pipeline_advancer
             )
           end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

data/lib/ductwork.rb

@@ -9,19 +9,15 @@ require "zeitwerk"
 
 require "ductwork/engine"
 
-loader = Zeitwerk::Loader.for_gem
-loader.inflector.inflect("cli" => "CLI")
-loader.inflector.inflect("dsl" => "DSL")
-loader.ignore("#{__dir__}/generators")
-loader.ignore("#{__dir__}/ductwork/testing")
-loader.ignore("#{__dir__}/ductwork/testing.rb")
-loader.setup
-
 module Ductwork
   class << self
-    attr_accessor :app_executor, :configuration, :logger
+    attr_accessor :app_executor, :configuration, :loader, :logger
     attr_writer :defined_pipelines, :hooks
 
+    def eager_load
+      loader.eager_load
+    end
+
     def wrap_with_app_executor(&block)
       if app_executor.present?
         app_executor.wrap(&block)
@@ -75,3 +71,14 @@ module Ductwork
     end
   end
 end
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect("cli" => "CLI")
+loader.inflector.inflect("dsl" => "DSL")
+loader.collapse("#{__dir__}/ductwork/models")
+loader.ignore("#{__dir__}/generators")
+loader.ignore("#{__dir__}/ductwork/testing")
+loader.ignore("#{__dir__}/ductwork/testing.rb")
+loader.setup
+
+Ductwork.loader = loader

data/lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb

@@ -12,5 +12,8 @@ class CreateDuctworkAvailabilities < ActiveRecord::Migration[<%= Rails::VERSION:
 
     add_index :ductwork_availabilities, :execution_id, unique: true
     add_index :ductwork_availabilities, %i[id process_id]
+    add_index :ductwork_availabilities,
+              %i[completed_at started_at created_at],
+              name: "index_ductwork_availabilities_on_claim_latest"
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb

@@ -10,5 +10,7 @@ class CreateDuctworkExecutions < ActiveRecord::Migration[<%= Rails::VERSION::MAJ
       table.integer :process_id
       table.timestamps null: false
     end
+
+    add_index :ductwork_executions, %i[job_id created_at]
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_jobs.rb

@@ -7,8 +7,8 @@ class CreateDuctworkJobs < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.
       table.string :klass, null: false
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
-      table.string :input_args, null: false
-      table.string :output_payload
+      table.text :input_args, null: false
+      table.text :output_payload
       table.timestamps null: false
     end
 

data/lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb

@@ -4,9 +4,10 @@ class CreateDuctworkPipelines < ActiveRecord::Migration[<%= Rails::VERSION::MAJO
   def change
     create_table :ductwork_pipelines do |table|
       table.string :klass, null: false
-      table.string :definition, null: false
+      table.text :definition, null: false
       table.string :definition_sha1, null: false
       table.timestamp :triggered_at, null: false
+      table.timestamp :started_at, null: false
       table.timestamp :completed_at
       table.timestamp :claimed_for_advancing_at
       table.timestamp :last_advanced_at, null: false

data/lib/generators/ductwork/install/templates/db/create_ductwork_results.rb

@@ -10,5 +10,7 @@ class CreateDuctworkResults < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR
       table.text :error_backtrace
       table.timestamps null: false
     end
+
+    add_index :ductwork_results, :execution_id, unique: true
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb

@@ -8,5 +8,7 @@ class CreateDuctworkRuns < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.
       table.timestamp :completed_at
       table.timestamps null: false
     end
+
+    add_index :ductwork_runs, :execution_id, unique: true
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -12,6 +12,9 @@ class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>
       table.timestamps null: false
     end
 
+    add_index :ductwork_steps, %i[pipeline_id status klass]
+    add_index :ductwork_steps, %i[pipeline_id klass status]
+    add_index :ductwork_steps, %i[status klass]
     add_index :ductwork_steps, %i[pipeline_id status]
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Tyler Ewing
@@ -95,15 +95,6 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- app/models/ductwork/availability.rb
-- app/models/ductwork/execution.rb
-- app/models/ductwork/job.rb
-- app/models/ductwork/pipeline.rb
-- app/models/ductwork/process.rb
-- app/models/ductwork/record.rb
-- app/models/ductwork/result.rb
-- app/models/ductwork/run.rb
-- app/models/ductwork/step.rb
 - lib/ductwork.rb
 - lib/ductwork/cli.rb
 - lib/ductwork/configuration.rb
@@ -111,6 +102,15 @@ files:
 - lib/ductwork/dsl/definition_builder.rb
 - lib/ductwork/engine.rb
 - lib/ductwork/machine_identifier.rb
+- lib/ductwork/models/availability.rb
+- lib/ductwork/models/execution.rb
+- lib/ductwork/models/job.rb
+- lib/ductwork/models/pipeline.rb
+- lib/ductwork/models/process.rb
+- lib/ductwork/models/record.rb
+- lib/ductwork/models/result.rb
+- lib/ductwork/models/run.rb
+- lib/ductwork/models/step.rb
 - lib/ductwork/processes/job_worker.rb
 - lib/ductwork/processes/job_worker_runner.rb
 - lib/ductwork/processes/pipeline_advancer.rb