ductwork 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18ebf23ac85cb80300ecc0c35475e0a2364fa4296a6012fdc21e95b8c5943023
-  data.tar.gz: e08b2cda1074255a7b7a79c5fbe789ac373909eef57b7836be49b3f7554a4350
+  metadata.gz: e10d600812e131829fe76b4a694bf755a707d4a4c66e29afd21701ee17e2fa8b
+  data.tar.gz: 101e1a25cac0c898f678b0da11f2c3e0d679bfb9c11741ed99c8e4f5bb8e3782
 SHA512:
-  metadata.gz: 9d3b28265bfdf2f90210c43e2609d4255dda6d120ab597b74e54e9c020cbdc4be0b07235bb104873a62ae309511fc841c7d236f20dd52124150760c8dfaa94c8
-  data.tar.gz: fb6db1e8385b880b942404a25aba9ef02637d0c1a2dd53b1a6ddd78a8c70e6494c6349b8d7bbf837727cee5d2ed2c2e2afd67fe506c0be193797fc2de8867619
+  metadata.gz: be40286349648fa967100abdcecc2fd3519e616881e635362903fb90656c6568e59a005e1015ac477bcae7eff61cda7b05378b35603302f2ca2ea08f3558de6d
+  data.tar.gz: ed9f573f3d6f2913c2d22dc0081064cb6d67eb743f92a1e573260d31c56592837d093292029a7de82b6ca40ebe022dc2092522167b84f52eb805ff9768fbe6f8

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Ductwork Changelog
 
+## [0.5.0]
+
+- core: add "waiting" status to `Step` model
+- core: add "waiting" status to `Pipeline` model
+- fix: change `jobs.input_args` and `jobs.output_payload` column type to `text`
+- fix: change `pipelines.definition` column type to `text` - this prevents larger definitions from being clipped if there is a size limit on the string column
+- feat: add missing unique index on `ductwork_results` and `ductwork_runs` tables
+- feat: add missing composite index on `ductwork_executions` table for `Ductwork::Job.claim_latest` method
+- feat: add missing composite index on `ductwork_availabilities` table for `Ductwork::Job.claim_latest` method
+- feat: use array instead of ActiveRecord relation when advancing pipelines - this has major performance benefits but comes with memory-usage implications (see comments)
+- fix: add condition to query to return correct pipelines that need advancing
+- fix: release pipeline claim only if successfully claimed
+- chore: add pipeline ID to misc log lines
+- feat: add missing composite indexes on `ductwork_steps` table
+
 ## [0.4.0]
 
 - chore: change job worker thread name format

data/app/models/ductwork/pipeline.rb

@@ -14,6 +14,7 @@ module Ductwork
     enum :status,
          pending: "pending",
          in_progress: "in_progress",
+         waiting: "waiting",
          halted: "halted",
          completed: "completed"
 
@@ -87,22 +88,19 @@ module Ductwork
     end
 
     def advance!
-      # NOTE: there could be A LOT of steps advancing for a single pipeline
-      # so instead of loading everything into memory and using ruby collection
-      # methods we make multiple queries. may need to revisist this once
-      # we do extensive load testing
-      advancing = steps.advancing
-      edges = if advancing.exists?
-                parsed_definition
-                  .fetch(:edges, {})
-                  .select { |k| k.in?(advancing.pluck(:klass)) }
-              end
+      # NOTE: if we've expanded the pipeline there could be a lot of
+      # advancing records which may cause memory issues. something to
+      # watch out for here and maybe add in config to use AR relation
+      # at certain counts or even memory limits.
+      advancing_steps = steps.advancing.pluck(:id, :klass)
+      advancing_ids = advancing_steps.map(&:first)
+      edges = find_edges(advancing_steps)
 
       Ductwork::Record.transaction do
         if edges.nil? || edges.values.all?(&:empty?)
-          conditionally_complete_pipeline(advancing)
+          conditionally_complete_pipeline(advancing_ids)
         else
-          advance_to_next_steps_by_type(edges, advancing)
+          advance_to_next_steps_by_type(edges, advancing_ids)
         end
       end
     end
@@ -120,10 +118,25 @@ module Ductwork
       Ductwork::Job.enqueue(next_step, input_arg)
     end
 
-    def conditionally_complete_pipeline(advancing)
-      advancing.update!(status: :completed, completed_at: Time.current)
+    def find_edges(advancing_steps)
+      if advancing_steps.any?
+        klasses = advancing_steps.map(&:last)
 
-      if steps.where(status: %w[in_progress pending]).none?
+        parsed_definition.fetch(:edges, {}).select { |k| k.in?(klasses) }
+      end
+    end
+
+    def conditionally_complete_pipeline(advancing_ids)
+      steps
+        .where(id: advancing_ids)
+        .update_all(status: :completed, completed_at: Time.current)
+
+      remaining = steps
+                  .where(status: %w[in_progress pending advancing])
+                  .where.not(id: advancing_ids)
+                  .exists?
+
+      if !remaining
         update!(status: :completed, completed_at: Time.current)
 
         Ductwork.logger.info(
@@ -134,9 +147,11 @@ module Ductwork
       end
     end
 
-    def advance_to_next_steps_by_type(edges, advancing)
+    def advance_to_next_steps_by_type(edges, advancing_ids)
+      steps.where(id: advancing_ids).update_all(status: :completed, completed_at: Time.current)
+
       if edges.all? { |_, v| v.dig(-1, :type) == "combine" }
-        conditionally_combine_next_steps(edges, advancing)
+        conditionally_combine_next_steps(edges, advancing_ids)
       else
         edges.each do |step_klass, step_edges|
           edge = step_edges[-1]
@@ -145,27 +160,25 @@ module Ductwork
           step_type = edge[:type] == "chain" ? "default" : edge[:type]
 
           if step_type == "collapse"
-            conditionally_collapse_next_steps(step_klass, edge, advancing)
+            conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
           else
-            advance_non_merging_steps(step_klass, edges, advancing)
+            advance_non_merging_steps(step_klass, edge, advancing_ids)
           end
         end
       end
-      advancing.update!(status: :completed, completed_at: Time.current)
       log_pipeline_advanced(edges)
     end
 
-    def advance_non_merging_steps(step_klass, edges, advancing)
-      advancing.where(klass: step_klass).find_each do |step|
-        edge = edges.dig(step.klass, -1)
-        # NOTE: "chain" is used by ActiveRecord so we have to call
-        # this enum value "default" :sad:
-        step_type = edge[:type] == "chain" ? "default" : edge[:type]
+    def advance_non_merging_steps(step_klass, edge, advancing_ids)
+      # NOTE: "chain" is used by ActiveRecord so we have to call
+      # this enum value "default" :sad:
+      step_type = edge[:type] == "chain" ? "default" : edge[:type]
 
+      steps.where(id: advancing_ids, klass: step_klass).find_each do |step|
         if step_type.in?(%w[default divide])
-          advance_to_next_steps(step_type, advancing, edge)
+          advance_to_next_steps(step_type, step.id, edge)
         elsif step_type == "expand"
-          expand_to_next_steps(step_type, advancing, edge)
+          expand_to_next_steps(step_type, step.id, edge)
         else
           Ductwork.logger.error(
             msg: "Invalid step type",
@@ -177,7 +190,7 @@ module Ductwork
       end
     end
 
-    def advance_to_next_steps(step_type, advancing, edge)
+    def advance_to_next_steps(step_type, step_id, edge)
       too_many = edge[:to].tally.any? do |to_klass, count|
         depth = Ductwork
                 .configuration
@@ -196,14 +209,15 @@ module Ductwork
             step_type: step_type,
             started_at: Time.current
           )
-          Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
+          return_value = Ductwork::Job.find_by(step_id:).return_value
+          Ductwork::Job.enqueue(next_step, return_value)
         end
       end
     end
 
-    def conditionally_combine_next_steps(edges, advancing)
+    def conditionally_combine_next_steps(edges, advancing_ids)
       if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
-        combine_next_steps(edges, advancing)
+        combine_next_steps(edges, advancing_ids)
       else
         Ductwork.logger.debug(
           msg: "Not all divided steps have completed; not combining",
@@ -213,14 +227,15 @@ module Ductwork
       end
     end
 
-    def combine_next_steps(edges, advancing)
+    def combine_next_steps(edges, advancing_ids)
       klass = edges.values.sample.dig(-1, :to).sole
       step_type = "combine"
-      groups = advancing
+      groups = steps
+               .where(id: advancing_ids)
                .group(:klass)
                .count
                .keys
-               .map { |k| advancing.where(klass: k) }
+               .map { |k| steps.where(id: advancing_ids).where(klass: k) }
 
       groups.first.zip(*groups[1..]).each do |group|
         input_arg = Ductwork::Job
@@ -230,9 +245,11 @@ module Ductwork
       end
     end
 
-    def expand_to_next_steps(step_type, advancing, edge)
+    def expand_to_next_steps(step_type, step_id, edge)
       next_klass = edge[:to].sole
-      return_value = advancing.take.job.return_value
+      return_value = Ductwork::Job
+                     .find_by(step_id:)
+                     .return_value
       max_depth = Ductwork.configuration.steps_max_depth(pipeline: klass, step: next_klass)
 
       if max_depth != -1 && return_value.count > max_depth
@@ -248,9 +265,9 @@ module Ductwork
       end
     end
 
-    def conditionally_collapse_next_steps(step_klass, edge, advancing)
+    def conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
       if steps.where(status: %w[pending in_progress], klass: step_klass).none?
-        collapse_next_steps(edge[:to].sole, advancing)
+        collapse_next_steps(edge[:to].sole, advancing_ids)
       else
         Ductwork.logger.debug(
           msg: "Not all expanded steps have completed; not collapsing",
@@ -260,14 +277,11 @@ module Ductwork
       end
     end
 
-    def collapse_next_steps(klass, advancing)
+    def collapse_next_steps(klass, advancing_ids)
       step_type = "collapse"
       input_arg = []
 
-      # NOTE: because of expanding based on return values, there
-      # could be A LOT of jobs so we want to use batch methods
-      # to avoid creating too many in-memory objects
-      Ductwork::Job.where(step_id: advancing.ids).find_each do |job|
+      Ductwork::Job.where(step_id: advancing_ids).find_each do |job|
         input_arg << job.return_value
       end
 

data/app/models/ductwork/step.rb

@@ -12,6 +12,7 @@ module Ductwork
     enum :status,
          pending: "pending",
          in_progress: "in_progress",
+         waiting: "waiting",
          advancing: "advancing",
          failed: "failed",
          completed: "completed"

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -15,6 +15,7 @@ module Ductwork
             Ductwork::Pipeline
               .in_progress
               .where(klass: klass, claimed_for_advancing_at: nil)
+              .where(steps: Ductwork::Step.where(status: :advancing))
               .where.not(steps: Ductwork::Step.where.not(status: %w[advancing completed]))
               .order(:last_advanced_at)
               .limit(1)
@@ -30,6 +31,7 @@ module Ductwork
             if rows_updated == 1
               Ductwork.logger.debug(
                 msg: "Pipeline claimed",
+                pipeline_id: id,
                 pipeline: klass,
                 role: :pipeline_advancer
               )
@@ -39,28 +41,29 @@ module Ductwork
 
               Ductwork.logger.debug(
                 msg: "Pipeline advanced",
+                pipeline_id: id,
                 pipeline: klass,
                 role: :pipeline_advancer
               )
+
+              # release the pipeline and set last advanced at so it doesnt block.
+              # we're not using a queue so we have to use a db timestamp
+              pipeline.update!(
+                claimed_for_advancing_at: nil,
+                last_advanced_at: Time.current
+              )
             else
               Ductwork.logger.debug(
                 msg: "Did not claim pipeline, avoided race condition",
+                pipeline_id: id,
                 pipeline: klass,
                 role: :pipeline_advancer
               )
             end
-
-            # release the pipeline and set last advanced at so it doesnt block.
-            # we're not using a queue so we have to use a db timestamp
-            Ductwork::Pipeline.find(id).update!(
-              claimed_for_advancing_at: nil,
-              last_advanced_at: Time.current
-            )
           else
             Ductwork.logger.debug(
               msg: "No pipeline needs advancing",
               pipeline: klass,
-              id: id,
               role: :pipeline_advancer
             )
           end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb

@@ -12,5 +12,8 @@ class CreateDuctworkAvailabilities < ActiveRecord::Migration[<%= Rails::VERSION:
 
     add_index :ductwork_availabilities, :execution_id, unique: true
     add_index :ductwork_availabilities, %i[id process_id]
+    add_index :ductwork_availabilities,
+              %i[completed_at started_at created_at],
+              name: "index_ductwork_availabilities_on_claim_latest"
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb

@@ -10,5 +10,7 @@ class CreateDuctworkExecutions < ActiveRecord::Migration[<%= Rails::VERSION::MAJ
       table.integer :process_id
       table.timestamps null: false
     end
+
+    add_index :ductwork_executions, %i[job_id created_at]
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_jobs.rb

@@ -7,8 +7,8 @@ class CreateDuctworkJobs < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.
       table.string :klass, null: false
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
-      table.string :input_args, null: false
-      table.string :output_payload
+      table.text :input_args, null: false
+      table.text :output_payload
       table.timestamps null: false
     end
 

data/lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb

@@ -4,7 +4,7 @@ class CreateDuctworkPipelines < ActiveRecord::Migration[<%= Rails::VERSION::MAJO
   def change
     create_table :ductwork_pipelines do |table|
       table.string :klass, null: false
-      table.string :definition, null: false
+      table.text :definition, null: false
       table.string :definition_sha1, null: false
       table.timestamp :triggered_at, null: false
       table.timestamp :completed_at

data/lib/generators/ductwork/install/templates/db/create_ductwork_results.rb

@@ -10,5 +10,7 @@ class CreateDuctworkResults < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR
       table.text :error_backtrace
       table.timestamps null: false
     end
+
+    add_index :ductwork_results, :execution_id, unique: true
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb

@@ -8,5 +8,7 @@ class CreateDuctworkRuns < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.
       table.timestamp :completed_at
       table.timestamps null: false
     end
+
+    add_index :ductwork_runs, :execution_id, unique: true
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -12,6 +12,9 @@ class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>
       table.timestamps null: false
     end
 
+    add_index :ductwork_steps, %i[pipeline_id status klass]
+    add_index :ductwork_steps, %i[pipeline_id klass status]
+    add_index :ductwork_steps, %i[status klass]
     add_index :ductwork_steps, %i[pipeline_id status]
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Tyler Ewing