ductwork 0.3.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab38ab3c5ce61b7fd6d48eac93d93bcaa33b3476a7b8b6e126403da7b4c2ea04
-  data.tar.gz: 7f3c09913cf33f5f2d6b42f2f3b786b09b35d343d1277262f77b0f9ff666b653
+  metadata.gz: e10d600812e131829fe76b4a694bf755a707d4a4c66e29afd21701ee17e2fa8b
+  data.tar.gz: 101e1a25cac0c898f678b0da11f2c3e0d679bfb9c11741ed99c8e4f5bb8e3782
 SHA512:
-  metadata.gz: cd1bbacdd428ed77064e526a7a2bfe639aad474c19d002417b5d3893856cef916baaba805124e1f4042c90ce972328f3de8c9747c88a96d8b3a60d88a26b183a
-  data.tar.gz: cef8abdee38bea2530ba058fde0062699fc53f190e3e67deb8c369f7c1badae7a7e9538f11ce6fc8f1bce488e15a9ab97271bf6fbc0c59f38f8eea1187f376e8
+  metadata.gz: be40286349648fa967100abdcecc2fd3519e616881e635362903fb90656c6568e59a005e1015ac477bcae7eff61cda7b05378b35603302f2ca2ea08f3558de6d
+  data.tar.gz: ed9f573f3d6f2913c2d22dc0081064cb6d67eb743f92a1e573260d31c56592837d093292029a7de82b6ca40ebe022dc2092522167b84f52eb805ff9768fbe6f8

data/CHANGELOG.md

@@ -1,5 +1,36 @@
 # Ductwork Changelog
 
+## [0.5.0]
+
+- core: add "waiting" status to `Step` model
+- core: add "waiting" status to `Pipeline` model
+- fix: change `jobs.input_args` and `jobs.output_payload` column type to `text`
+- fix: change `pipelines.definition` column type to `text` - this prevents larger definitions from being clipped if there is a size limit on the string column
+- feat: add missing unique index on `ductwork_results` and `ductwork_runs` tables
+- feat: add missing composite index on `ductwork_executions` table for `Ductwork::Job.claim_latest` method
+- feat: add missing composite index on `ductwork_availabilities` table for `Ductwork::Job.claim_latest` method
+- feat: use array instead of ActiveRecord relation when advancing pipelines - this has major performance benefits but comes with memory-usage implications (see comments)
+- fix: add condition to query to return correct pipelines that need advancing
+- fix: release pipeline claim only if successfully claimed
+- chore: add pipeline ID to misc log lines
+- feat: add missing composite indexes on `ductwork_steps` table
+
+## [0.4.0]
+
+- chore: change job worker thread name format
+- feat: add and respect pipeline-level `pipeline_advancer.polling_timeout` configuration in pipeline advancer
+- feat: respect `job_worker.polling_timeout` configuration in job runner
+- feat: add pipeline-level `job_worker.polling_timeout` configuration
+- feat: check pipeline and step-level max retry configurations when retrying a job
+- feat: add pipeline and step-level `job_worker.max_retry` configurations
+- feat: add ability to set `job_worker.count` config manually
+- chore: move configuration specs under their own directory
+- feat: halt pipeline instead of erroring if max step depth is exceeded
+- chore: move specs under directory
+- feat: allow setting `pipeline_advancer.steps_max_depth` configuration manually
+- feat: raise `Ductwork::Pipeline::StepDepthError` error if return payload count exceeds the configuration
+- feat: add `pipeline_advancer.steps.max_depth` configuration
+
 ## [0.3.1]
 
 - chore: bump dependencies and update necessary files

data/app/models/ductwork/job.rb

@@ -142,9 +142,12 @@ module Ductwork
       end
     end
 
-    def execution_failed!(execution, run, error) # rubocop:disable Metrics/AbcSize
+    def execution_failed!(execution, run, error) # rubocop:todo Metrics
       halted = false
       pipeline = step.pipeline
+      max_retry = Ductwork
+                  .configuration
+                  .job_worker_max_retry(pipeline: pipeline.klass, step: klass)
 
       Ductwork::Record.transaction do
         execution.update!(completed_at: Time.current)
@@ -156,7 +159,7 @@ module Ductwork
           error_backtrace: error.backtrace
         )
 
-        if execution.retry_count < Ductwork.configuration.job_worker_max_retry
+        if execution.retry_count < max_retry
           new_execution = executions.create!(
             retry_count: execution.retry_count + 1,
             started_at: FAILED_EXECUTION_TIMEOUT.from_now

data/app/models/ductwork/pipeline.rb

@@ -14,6 +14,7 @@ module Ductwork
     enum :status,
          pending: "pending",
          in_progress: "in_progress",
+         waiting: "waiting",
          halted: "halted",
          completed: "completed"
 
@@ -87,22 +88,19 @@ module Ductwork
     end
 
     def advance!
-      # NOTE: there could be A LOT of steps advancing for a single pipeline
-      # so instead of loading everything into memory and using ruby collection
-      # methods we make multiple queries. may need to revisist this once
-      # we do extensive load testing
-      advancing = steps.advancing
-      edges = if advancing.exists?
-                parsed_definition
-                  .fetch(:edges, {})
-                  .select { |k| k.in?(advancing.pluck(:klass)) }
-              end
+      # NOTE: if we've expanded the pipeline there could be a lot of
+      # advancing records which may cause memory issues. something to
+      # watch out for here and maybe add in config to use AR relation
+      # at certain counts or even memory limits.
+      advancing_steps = steps.advancing.pluck(:id, :klass)
+      advancing_ids = advancing_steps.map(&:first)
+      edges = find_edges(advancing_steps)
 
       Ductwork::Record.transaction do
         if edges.nil? || edges.values.all?(&:empty?)
-          conditionally_complete_pipeline(advancing)
+          conditionally_complete_pipeline(advancing_ids)
         else
-          advance_to_next_steps_by_type(edges, advancing)
+          advance_to_next_steps_by_type(edges, advancing_ids)
         end
       end
     end
@@ -120,10 +118,25 @@ module Ductwork
       Ductwork::Job.enqueue(next_step, input_arg)
     end
 
-    def conditionally_complete_pipeline(advancing)
-      advancing.update!(status: :completed, completed_at: Time.current)
+    def find_edges(advancing_steps)
+      if advancing_steps.any?
+        klasses = advancing_steps.map(&:last)
 
-      if steps.where(status: %w[in_progress pending]).none?
+        parsed_definition.fetch(:edges, {}).select { |k| k.in?(klasses) }
+      end
+    end
+
+    def conditionally_complete_pipeline(advancing_ids)
+      steps
+        .where(id: advancing_ids)
+        .update_all(status: :completed, completed_at: Time.current)
+
+      remaining = steps
+                  .where(status: %w[in_progress pending advancing])
+                  .where.not(id: advancing_ids)
+                  .exists?
+
+      if !remaining
         update!(status: :completed, completed_at: Time.current)
 
         Ductwork.logger.info(
@@ -134,9 +147,11 @@ module Ductwork
       end
     end
 
-    def advance_to_next_steps_by_type(edges, advancing)
+    def advance_to_next_steps_by_type(edges, advancing_ids)
+      steps.where(id: advancing_ids).update_all(status: :completed, completed_at: Time.current)
+
       if edges.all? { |_, v| v.dig(-1, :type) == "combine" }
-        conditionally_combine_next_steps(edges, advancing)
+        conditionally_combine_next_steps(edges, advancing_ids)
       else
         edges.each do |step_klass, step_edges|
           edge = step_edges[-1]
@@ -145,27 +160,25 @@ module Ductwork
           step_type = edge[:type] == "chain" ? "default" : edge[:type]
 
           if step_type == "collapse"
-            conditionally_collapse_next_steps(step_klass, edge, advancing)
+            conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
           else
-            advance_non_merging_steps(step_klass, edges, advancing)
+            advance_non_merging_steps(step_klass, edge, advancing_ids)
           end
         end
       end
-      advancing.update!(status: :completed, completed_at: Time.current)
       log_pipeline_advanced(edges)
     end
 
-    def advance_non_merging_steps(step_klass, edges, advancing)
-      advancing.where(klass: step_klass).find_each do |step|
-        edge = edges.dig(step.klass, -1)
-        # NOTE: "chain" is used by ActiveRecord so we have to call
-        # this enum value "default" :sad:
-        step_type = edge[:type] == "chain" ? "default" : edge[:type]
+    def advance_non_merging_steps(step_klass, edge, advancing_ids)
+      # NOTE: "chain" is used by ActiveRecord so we have to call
+      # this enum value "default" :sad:
+      step_type = edge[:type] == "chain" ? "default" : edge[:type]
 
+      steps.where(id: advancing_ids, klass: step_klass).find_each do |step|
         if step_type.in?(%w[default divide])
-          advance_to_next_steps(step_type, advancing, edge)
+          advance_to_next_steps(step_type, step.id, edge)
         elsif step_type == "expand"
-          expand_to_next_steps(step_type, advancing, edge)
+          expand_to_next_steps(step_type, step.id, edge)
         else
           Ductwork.logger.error(
             msg: "Invalid step type",
@@ -177,21 +190,34 @@ module Ductwork
       end
     end
 
-    def advance_to_next_steps(step_type, advancing, edge)
-      edge[:to].each do |to_klass|
-        next_step = steps.create!(
-          klass: to_klass,
-          status: :in_progress,
-          step_type: step_type,
-          started_at: Time.current
-        )
-        Ductwork::Job.enqueue(next_step, advancing.take.job.return_value)
+    def advance_to_next_steps(step_type, step_id, edge)
+      too_many = edge[:to].tally.any? do |to_klass, count|
+        depth = Ductwork
+                .configuration
+                .steps_max_depth(pipeline: klass, step: to_klass)
+
+        depth != -1 && count > depth
+      end
+
+      if too_many
+        halted!
+      else
+        edge[:to].each do |to_klass|
+          next_step = steps.create!(
+            klass: to_klass,
+            status: :in_progress,
+            step_type: step_type,
+            started_at: Time.current
+          )
+          return_value = Ductwork::Job.find_by(step_id:).return_value
+          Ductwork::Job.enqueue(next_step, return_value)
+        end
       end
     end
 
-    def conditionally_combine_next_steps(edges, advancing)
+    def conditionally_combine_next_steps(edges, advancing_ids)
       if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
-        combine_next_steps(edges, advancing)
+        combine_next_steps(edges, advancing_ids)
       else
         Ductwork.logger.debug(
           msg: "Not all divided steps have completed; not combining",
@@ -201,14 +227,15 @@ module Ductwork
       end
     end
 
-    def combine_next_steps(edges, advancing)
+    def combine_next_steps(edges, advancing_ids)
       klass = edges.values.sample.dig(-1, :to).sole
       step_type = "combine"
-      groups = advancing
+      groups = steps
+               .where(id: advancing_ids)
                .group(:klass)
                .count
                .keys
-               .map { |k| advancing.where(klass: k) }
+               .map { |k| steps.where(id: advancing_ids).where(klass: k) }
 
       groups.first.zip(*groups[1..]).each do |group|
         input_arg = Ductwork::Job
@@ -218,19 +245,29 @@ module Ductwork
       end
     end
 
-    def expand_to_next_steps(step_type, advancing, edge)
-      Array(advancing.take.job.return_value).each do |input_arg|
-        create_step_and_enqueue_job(
-          klass: edge[:to].sole,
-          step_type: step_type,
-          input_arg: input_arg
-        )
+    def expand_to_next_steps(step_type, step_id, edge)
+      next_klass = edge[:to].sole
+      return_value = Ductwork::Job
+                     .find_by(step_id:)
+                     .return_value
+      max_depth = Ductwork.configuration.steps_max_depth(pipeline: klass, step: next_klass)
+
+      if max_depth != -1 && return_value.count > max_depth
+        halted!
+      else
+        Array(return_value).each do |input_arg|
+          create_step_and_enqueue_job(
+            klass: next_klass,
+            step_type: step_type,
+            input_arg: input_arg
+          )
+        end
       end
     end
 
-    def conditionally_collapse_next_steps(step_klass, edge, advancing)
+    def conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
       if steps.where(status: %w[pending in_progress], klass: step_klass).none?
-        collapse_next_steps(edge[:to].sole, advancing)
+        collapse_next_steps(edge[:to].sole, advancing_ids)
       else
         Ductwork.logger.debug(
           msg: "Not all expanded steps have completed; not collapsing",
@@ -240,14 +277,11 @@ module Ductwork
       end
     end
 
-    def collapse_next_steps(klass, advancing)
+    def collapse_next_steps(klass, advancing_ids)
       step_type = "collapse"
       input_arg = []
 
-      # NOTE: because of expanding based on return values, there
-      # could be A LOT of jobs so we want to use batch methods
-      # to avoid creating too many in-memory objects
-      Ductwork::Job.where(step_id: advancing.ids).find_each do |job|
+      Ductwork::Job.where(step_id: advancing_ids).find_each do |job|
         input_arg << job.return_value
       end
 

data/app/models/ductwork/step.rb

@@ -12,6 +12,7 @@ module Ductwork
     enum :status,
          pending: "pending",
          in_progress: "in_progress",
+         waiting: "waiting",
          advancing: "advancing",
          failed: "failed",
          completed: "completed"

data/lib/ductwork/configuration.rb

@@ -12,15 +12,17 @@ module Ductwork
     DEFAULT_LOGGER_SOURCE = "default" # `Logger` instance writing to STDOUT
     DEFAULT_PIPELINE_POLLING_TIMEOUT = 1 # second
     DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT = 20 # seconds
+    DEFAULT_STEPS_MAX_DEPTH = -1 # unlimited count
     DEFAULT_SUPERVISOR_POLLING_TIMEOUT = 1 # second
     DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT = 30 # seconds
     DEFAULT_LOGGER = ::Logger.new($stdout)
     PIPELINES_WILDCARD = "*"
 
-    attr_writer :job_worker_polling_timeout, :job_worker_shutdown_timeout,
-                :job_worker_max_retry,
+    attr_writer :job_worker_count, :job_worker_polling_timeout,
+                :job_worker_shutdown_timeout, :job_worker_max_retry,
                 :logger_level,
                 :pipeline_polling_timeout, :pipeline_shutdown_timeout,
+                :steps_max_depth,
                 :supervisor_polling_timeout, :supervisor_shutdown_timeout
 
     def initialize(path: DEFAULT_FILE_PATH)
@@ -49,6 +51,8 @@ module Ductwork
     end
 
     def job_worker_count(pipeline)
+      return @job_worker_count if instance_variable_defined?(:@job_worker_count)
+
       raw_count = config.dig(:job_worker, :count) || DEFAULT_JOB_WORKER_COUNT
 
       if raw_count.is_a?(Hash)
@@ -58,12 +62,36 @@ module Ductwork
       end
     end
 
-    def job_worker_max_retry
-      @job_worker_max_retry ||= fetch_job_worker_max_retry
+    def job_worker_max_retry(pipeline: nil, step: nil) # rubocop:disable Metrics
+      return @job_worker_max_retry if instance_variable_defined?(:@job_worker_max_retry)
+
+      pipeline ||= :default
+      step ||= :default
+      base_config = config.dig(:job_worker, :max_retry)
+
+      if base_config.is_a?(Hash) && base_config[pipeline.to_sym].is_a?(Hash)
+        pipeline_config = config.dig(:job_worker, :max_retry, pipeline.to_sym)
+
+        pipeline_config[step.to_sym] || pipeline_config[:default] || DEFAULT_JOB_WORKER_MAX_RETRY
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || DEFAULT_JOB_WORKER_MAX_RETRY
+      else
+        base_config || DEFAULT_JOB_WORKER_MAX_RETRY
+      end
     end
 
-    def job_worker_polling_timeout
-      @job_worker_polling_timeout ||= fetch_job_worker_polling_timeout
+    def job_worker_polling_timeout(pipeline = nil)
+      pipeline ||= :default
+      default = DEFAULT_JOB_WORKER_POLLING_TIMEOUT
+      base_config = config.dig(:job_worker, :polling_timeout)
+
+      if instance_variable_defined?(:@job_worker_polling_timeout)
+        @job_worker_polling_timeout
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || default
+      else
+        base_config || default
+      end
     end
 
     def job_worker_shutdown_timeout
@@ -78,14 +106,46 @@ module Ductwork
       @logger_source ||= fetch_logger_source
     end
 
-    def pipeline_polling_timeout
-      @pipeline_polling_timeout ||= fetch_pipeline_polling_timeout
+    def pipeline_polling_timeout(pipeline = nil)
+      pipeline ||= :default
+      default = DEFAULT_PIPELINE_POLLING_TIMEOUT
+      base_config = config.dig(:pipeline_advancer, :polling_timeout)
+
+      if instance_variable_defined?(:@pipeline_polling_timeout)
+        @pipeline_polling_timeout
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] || base_config[:default] || default
+      else
+        base_config || default
+      end
     end
 
     def pipeline_shutdown_timeout
       @pipeline_shutdown_timeout ||= fetch_pipeline_shutdown_timeout
     end
 
+    def steps_max_depth(pipeline: nil, step: nil) # rubocop:disable Metrics
+      return @steps_max_depth if instance_variable_defined?(:@steps_max_depth)
+
+      pipeline ||= :default
+      step ||= :default
+      base_config = config.dig(:pipeline_advancer, :steps, :max_depth)
+
+      if base_config.is_a?(Hash) && base_config[pipeline.to_sym].is_a?(Hash)
+        pipeline_config = config.dig(:pipeline_advancer, :steps, :max_depth, pipeline.to_sym)
+
+        pipeline_config[step.to_sym] ||
+          pipeline_config[:default] ||
+          DEFAULT_STEPS_MAX_DEPTH
+      elsif base_config.is_a?(Hash)
+        base_config[pipeline.to_sym] ||
+          base_config[:default] ||
+          DEFAULT_STEPS_MAX_DEPTH
+      else
+        base_config || DEFAULT_STEPS_MAX_DEPTH
+      end
+    end
+
     def supervisor_polling_timeout
       @supervisor_polling_timeout ||= fetch_supervisor_polling_timeout
     end
@@ -98,16 +158,6 @@ module Ductwork
 
     attr_reader :config
 
-    def fetch_job_worker_max_retry
-      config.dig(:job_worker, :max_retry) ||
-        DEFAULT_JOB_WORKER_MAX_RETRY
-    end
-
-    def fetch_job_worker_polling_timeout
-      config.dig(:job_worker, :polling_timeout) ||
-        DEFAULT_JOB_WORKER_POLLING_TIMEOUT
-    end
-
     def fetch_job_worker_shutdown_timeout
       config.dig(:job_worker, :shutdown_timeout) ||
         DEFAULT_JOB_WORKER_SHUTDOWN_TIMEOUT
@@ -121,11 +171,6 @@ module Ductwork
       config.dig(:logger, :source) || DEFAULT_LOGGER_SOURCE
     end
 
-    def fetch_pipeline_polling_timeout
-      config.dig(:pipeline_advancer, :polling_timeout) ||
-        DEFAULT_PIPELINE_POLLING_TIMEOUT
-    end
-
     def fetch_pipeline_shutdown_timeout
       config.dig(:pipeline_advancer, :shutdown_timeout) ||
         DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT

data/lib/ductwork/processes/job_worker.rb

@@ -35,7 +35,7 @@ module Ductwork
               role: :job_worker,
               pipeline: pipeline
             )
-            sleep(Ductwork.configuration.job_worker_polling_timeout)
+            sleep(polling_timeout)
           end
         end
 
@@ -62,6 +62,10 @@ module Ductwork
           end
         end
       end
+
+      def polling_timeout
+        Ductwork.configuration.job_worker_polling_timeout(pipeline)
+      end
     end
   end
 end

data/lib/ductwork/processes/job_worker_runner.rb

@@ -63,7 +63,7 @@ module Ductwork
           thread = Thread.new do
             job_worker.run
           end
-          thread.name = "ductwork.job_worker_#{i}"
+          thread.name = "ductwork.job_worker.#{i}"
 
           Ductwork.logger.debug(
             msg: "Created new thread",

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -15,6 +15,7 @@ module Ductwork
             Ductwork::Pipeline
               .in_progress
               .where(klass: klass, claimed_for_advancing_at: nil)
+              .where(steps: Ductwork::Step.where(status: :advancing))
               .where.not(steps: Ductwork::Step.where.not(status: %w[advancing completed]))
               .order(:last_advanced_at)
               .limit(1)
@@ -30,6 +31,7 @@ module Ductwork
             if rows_updated == 1
               Ductwork.logger.debug(
                 msg: "Pipeline claimed",
+                pipeline_id: id,
                 pipeline: klass,
                 role: :pipeline_advancer
               )
@@ -39,33 +41,34 @@ module Ductwork
 
               Ductwork.logger.debug(
                 msg: "Pipeline advanced",
+                pipeline_id: id,
                 pipeline: klass,
                 role: :pipeline_advancer
               )
+
+              # release the pipeline and set last advanced at so it doesnt block.
+              # we're not using a queue so we have to use a db timestamp
+              pipeline.update!(
+                claimed_for_advancing_at: nil,
+                last_advanced_at: Time.current
+              )
             else
               Ductwork.logger.debug(
                 msg: "Did not claim pipeline, avoided race condition",
+                pipeline_id: id,
                 pipeline: klass,
                 role: :pipeline_advancer
               )
             end
-
-            # release the pipeline and set last advanced at so it doesnt block.
-            # we're not using a queue so we have to use a db timestamp
-            Ductwork::Pipeline.find(id).update!(
-              claimed_for_advancing_at: nil,
-              last_advanced_at: Time.current
-            )
           else
             Ductwork.logger.debug(
               msg: "No pipeline needs advancing",
               pipeline: klass,
-              id: id,
               role: :pipeline_advancer
             )
           end
 
-          sleep(Ductwork.configuration.pipeline_polling_timeout)
+          sleep(polling_timeout)
         end
 
         run_hooks_for(:stop)
@@ -82,6 +85,10 @@ module Ductwork
           end
         end
       end
+
+      def polling_timeout
+        Ductwork.configuration.pipeline_polling_timeout(klass)
+      end
     end
   end
 end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.3.1"
+  VERSION = "0.5.0"
 end

data/lib/generators/ductwork/install/templates/config/ductwork.yml

@@ -11,6 +11,8 @@ default: &default
   pipeline_advancer:
     polling_timeout: 1
     shutdown_timeout: 20
+    steps:
+      max_depth: -1
   supervisor:
     polling_timeout: 1
     shutdown_timeout: 30

data/lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb

@@ -12,5 +12,8 @@ class CreateDuctworkAvailabilities < ActiveRecord::Migration[<%= Rails::VERSION:
 
     add_index :ductwork_availabilities, :execution_id, unique: true
     add_index :ductwork_availabilities, %i[id process_id]
+    add_index :ductwork_availabilities,
+              %i[completed_at started_at created_at],
+              name: "index_ductwork_availabilities_on_claim_latest"
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb

@@ -10,5 +10,7 @@ class CreateDuctworkExecutions < ActiveRecord::Migration[<%= Rails::VERSION::MAJ
       table.integer :process_id
       table.timestamps null: false
     end
+
+    add_index :ductwork_executions, %i[job_id created_at]
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_jobs.rb

@@ -7,8 +7,8 @@ class CreateDuctworkJobs < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.
       table.string :klass, null: false
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
-      table.string :input_args, null: false
-      table.string :output_payload
+      table.text :input_args, null: false
+      table.text :output_payload
       table.timestamps null: false
     end
 

data/lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb

@@ -4,7 +4,7 @@ class CreateDuctworkPipelines < ActiveRecord::Migration[<%= Rails::VERSION::MAJO
   def change
     create_table :ductwork_pipelines do |table|
       table.string :klass, null: false
-      table.string :definition, null: false
+      table.text :definition, null: false
       table.string :definition_sha1, null: false
       table.timestamp :triggered_at, null: false
       table.timestamp :completed_at

data/lib/generators/ductwork/install/templates/db/create_ductwork_results.rb

@@ -10,5 +10,7 @@ class CreateDuctworkResults < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR
       table.text :error_backtrace
       table.timestamps null: false
     end
+
+    add_index :ductwork_results, :execution_id, unique: true
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb

@@ -8,5 +8,7 @@ class CreateDuctworkRuns < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.
       table.timestamp :completed_at
       table.timestamps null: false
     end
+
+    add_index :ductwork_runs, :execution_id, unique: true
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -12,6 +12,9 @@ class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>
       table.timestamps null: false
     end
 
+    add_index :ductwork_steps, %i[pipeline_id status klass]
+    add_index :ductwork_steps, %i[pipeline_id klass status]
+    add_index :ductwork_steps, %i[status klass]
     add_index :ductwork_steps, %i[pipeline_id status]
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Tyler Ewing