ductwork 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6174d886b38c9a000eda9937e27efabd0f89a803c5c77fb5739c2b67d8c004c9
-  data.tar.gz: 6300c6f3d238e2f92aa74c5ebe01c37bcbf842693f186b9c56e8d8b2cbcf567d
+  metadata.gz: f64abfa6d358cd7b225baae1b801894032936bef4de86ff0843e3163552aa530
+  data.tar.gz: aedd9daa71379091503375926ea4863a7834193a1701ee265a9b0b99a7fbfe50
 SHA512:
-  metadata.gz: fcda67f0378eb7e4108820ce87ac4b508698abcd02e71ef77b7ec8442dfeeea019a77e35bacba5a32fda4101004d2c0f349441d3921838deee97aea6ac93eed5
-  data.tar.gz: e74fef5cf5bbf80b8b112616636a556a6ec7742ade2fe44dae3679b3c679ef7b9eb96dc81ad016fb1d4285a06cef874f74d1f6c67a20062b5eaa0bdcfde6061c
+  metadata.gz: 85598e36b5edb64f33217d19cb2f2af66e6991c68f74c0a8ede9c445d88884bd804da2b381b2dd9cdeaa6a6936a340628af25c45ff259632f36d1c4429665bcb
+  data.tar.gz: 4f59ad232e7a00c8e34403dfb8412af7e7351f0d921b3fd7fc6c27ee10524a1bd37e57bd18aa020c9e7daf1c3374127927e54474302364f3ee6962d157b9e468

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Ductwork Changelog
 
+## [0.8.0]
+
+- chore: re-organize `Ductwork::CLI` class
+- feat!: better name `ductwork_steps.step_type` to `to_transition` - the column rename is a breaking change but not bumping major version since the gem is still pre-1.0
+- feat: add `delay_seconds` and `timeout_seconds` columns to `ductwork_steps` table
+- feat: set pipeline status to `advancing` once pipeline is claimed and advancing starts
+- chore: add `advancing` status enum value to `Pipeline`
+
+## [0.7.2]
+
+- chore: small DRY refactor in pipeline advancement
+
 ## [0.7.1]
 
 - chore: isolate on halt execution in `Ductwork::Job#execution_failed!`

data/lib/ductwork/cli.rb

@@ -4,45 +4,57 @@ require "optparse"
 
 module Ductwork
   class CLI
-    class << self
-      def start!(args)
-        options = parse_options(args)
-        Ductwork.configuration = Configuration.new(**options)
-        Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
-                            Rails.logger
-                          else
-                            Ductwork::Configuration::DEFAULT_LOGGER
-                          end
-        Ductwork.logger.level = Ductwork.configuration.logger_level
-
-        Ductwork::Processes::SupervisorRunner.start!
-      end
+    def self.start!(args)
+      new(args).start!
+    end
 
-      private
+    def initialize(args)
+      @args = args
+      @options = {}
+    end
+
+    def start!
+      option_parser.parse!(args)
+      auto_configure
+      supervisor_runner.start!
+    end
 
-      def parse_options(args)
-        options = {}
+    private
 
-        OptionParser.new do |op|
-          op.banner = "ductwork [options]"
+    attr_reader :args, :options
 
-          op.on("-c", "--config PATH", "path to YAML config file") do |arg|
-            options[:path] = arg
-          end
+    def option_parser
+      OptionParser.new do |op|
+        op.banner = "ductwork [options]"
 
-          op.on("-h", "--help", "Prints this help") do
-            puts op
-            exit
-          end
+        op.on("-c", "--config PATH", "path to YAML config file") do |arg|
+          options[:path] = arg
+        end
 
-          op.on("-v", "--version", "Prints the version") do
-            puts "Ductwork #{Ductwork::VERSION}"
-            exit
-          end
-        end.parse!(args)
+        op.on("-h", "--help", "Prints this help") do
+          puts op
+          exit
+        end
 
-        options
+        op.on("-v", "--version", "Prints the version") do
+          puts "Ductwork #{Ductwork::VERSION}"
+          exit
+        end
       end
     end
+
+    def auto_configure
+      Ductwork.configuration = Configuration.new(**options)
+      Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
+                          Rails.logger
+                        else
+                          Ductwork::Configuration::DEFAULT_LOGGER
+                        end
+      Ductwork.logger.level = Ductwork.configuration.logger_level
+    end
+
+    def supervisor_runner
+      Ductwork::Processes::SupervisorRunner
+    end
   end
 end

data/lib/ductwork/models/pipeline.rb

@@ -16,6 +16,7 @@ module Ductwork
          pending: "pending",
          in_progress: "in_progress",
          waiting: "waiting",
+         advancing: "advancing",
          halted: "halted",
          completed: "completed"
 
@@ -71,7 +72,7 @@ module Ductwork
           step = p.steps.create!(
             klass: step_klass,
             status: :in_progress,
-            step_type: :start,
+            to_transition: :start,
             started_at: Time.current
           )
           Ductwork::Job.enqueue(step, args)
@@ -113,10 +114,15 @@ module Ductwork
 
     private
 
-    def create_step_and_enqueue_job(klass:, step_type:, input_arg:)
+    def create_step_and_enqueue_job(edge:, input_arg:, klass: nil)
       status = :in_progress
       started_at = Time.current
-      next_step = steps.create!(klass:, status:, step_type:, started_at:)
+      # NOTE: "chain" is used by ActiveRecord so we have to call
+      # this enum value "default" :sad:
+      to_transition = edge[:type] == "chain" ? "default" : edge[:type]
+      klass ||= edge.fetch(:to).sole
+
+      next_step = steps.create!(klass:, status:, to_transition:, started_at:)
       Ductwork::Job.enqueue(next_step, input_arg)
     end
 
@@ -157,11 +163,8 @@ module Ductwork
       else
         edges.each do |step_klass, step_edges|
           edge = step_edges[-1]
-          # NOTE: "chain" is used by ActiveRecord so we have to call
-          # this enum value "default" :sad:
-          step_type = edge[:type] == "chain" ? "default" : edge[:type]
 
-          if step_type == "collapse"
+          if edge[:type] == "collapse"
             conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
           else
             advance_non_merging_steps(step_klass, edge, advancing_ids)
@@ -172,19 +175,17 @@ module Ductwork
     end
 
     def advance_non_merging_steps(step_klass, edge, advancing_ids)
-      # NOTE: "chain" is used by ActiveRecord so we have to call
-      # this enum value "default" :sad:
-      step_type = edge[:type] == "chain" ? "default" : edge[:type]
+      to_transition = edge[:type]
 
       steps.where(id: advancing_ids, klass: step_klass).find_each do |step|
-        if step_type.in?(%w[default divide])
-          advance_to_next_steps(step_type, step.id, edge)
-        elsif step_type == "expand"
-          expand_to_next_steps(step_type, step.id, edge)
+        if to_transition.in?(%w[chain divide])
+          advance_to_next_steps(step.id, edge)
+        elsif to_transition == "expand"
+          expand_to_next_steps(step.id, edge)
         else
           Ductwork.logger.error(
-            msg: "Invalid step type",
-            step_type: step_type,
+            msg: "Invalid To Transition",
+            to_transition: to_transition,
             pipeline_id: id,
             role: :pipeline_advancer
           )
@@ -192,7 +193,7 @@ module Ductwork
       end
     end
 
-    def advance_to_next_steps(step_type, step_id, edge)
+    def advance_to_next_steps(step_id, edge)
       too_many = edge[:to].tally.any? do |to_klass, count|
         depth = Ductwork
                 .configuration
@@ -204,15 +205,9 @@ module Ductwork
       if too_many
         halted!
       else
-        edge[:to].each do |to_klass|
-          next_step = steps.create!(
-            klass: to_klass,
-            status: :in_progress,
-            step_type: step_type,
-            started_at: Time.current
-          )
-          return_value = Ductwork::Job.find_by(step_id:).return_value
-          Ductwork::Job.enqueue(next_step, return_value)
+        edge[:to].each do |klass|
+          input_arg = Ductwork::Job.find_by(step_id:).return_value
+          create_step_and_enqueue_job(edge:, input_arg:, klass:)
         end
       end
     end
@@ -230,8 +225,7 @@ module Ductwork
     end
 
     def combine_next_steps(edges, advancing_ids)
-      klass = edges.values.sample.dig(-1, :to).sole
-      step_type = "combine"
+      edge = edges.values.sample[-1]
       groups = steps
                .where(id: advancing_ids)
                .group(:klass)
@@ -243,11 +237,11 @@ module Ductwork
         input_arg = Ductwork::Job
                     .where(step_id: group.map(&:id))
                     .map(&:return_value)
-        create_step_and_enqueue_job(klass:, step_type:, input_arg:)
+        create_step_and_enqueue_job(edge:, input_arg:)
       end
     end
 
-    def expand_to_next_steps(step_type, step_id, edge)
+    def expand_to_next_steps(step_id, edge)
       next_klass = edge[:to].sole
       return_value = Ductwork::Job
                      .find_by(step_id:)
@@ -258,18 +252,14 @@ module Ductwork
         halted!
       else
         Array(return_value).each do |input_arg|
-          create_step_and_enqueue_job(
-            klass: next_klass,
-            step_type: step_type,
-            input_arg: input_arg
-          )
+          create_step_and_enqueue_job(edge:, input_arg:)
         end
       end
     end
 
     def conditionally_collapse_next_steps(step_klass, edge, advancing_ids)
       if steps.where(status: %w[pending in_progress], klass: step_klass).none?
-        collapse_next_steps(edge[:to].sole, advancing_ids)
+        collapse_next_steps(edge, advancing_ids)
       else
         Ductwork.logger.debug(
           msg: "Not all expanded steps have completed; not collapsing",
@@ -279,15 +269,14 @@ module Ductwork
       end
     end
 
-    def collapse_next_steps(klass, advancing_ids)
-      step_type = "collapse"
+    def collapse_next_steps(edge, advancing_ids)
       input_arg = []
 
       Ductwork::Job.where(step_id: advancing_ids).find_each do |job|
         input_arg << job.return_value
       end
 
-      create_step_and_enqueue_job(klass:, step_type:, input_arg:)
+      create_step_and_enqueue_job(edge:, input_arg:)
     end
 
     def log_pipeline_advanced(edges)

data/lib/ductwork/models/step.rb

@@ -7,7 +7,7 @@ module Ductwork
 
     validates :klass, presence: true
     validates :status, presence: true
-    validates :step_type, presence: true
+    validates :to_transition, presence: true
 
     enum :status,
          pending: "pending",
@@ -17,7 +17,7 @@ module Ductwork
          failed: "failed",
          completed: "completed"
 
-    enum :step_type,
+    enum :to_transition,
          start: "start",
          default: "default", # `chain` is used by AR
          divide: "divide",

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -26,7 +26,10 @@ module Ductwork
           if id.present?
             rows_updated = Ductwork::Pipeline
                            .where(id: id, claimed_for_advancing_at: nil)
-                           .update_all(claimed_for_advancing_at: Time.current)
+                           .update_all(
+                             claimed_for_advancing_at: Time.current,
+                             status: "advancing"
+                           )
 
             if rows_updated == 1
               Ductwork.logger.debug(
@@ -50,7 +53,8 @@ module Ductwork
               # we're not using a queue so we have to use a db timestamp
               pipeline.update!(
                 claimed_for_advancing_at: nil,
-                last_advanced_at: Time.current
+                last_advanced_at: Time.current,
+                status: "in_progress"
               )
             else
               Ductwork.logger.debug(

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.7.1"
+  VERSION = "0.8.0"
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -5,10 +5,12 @@ class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>
     create_table :ductwork_steps do |table|
       table.belongs_to :pipeline, index: true, null: false, foreign_key: { to_table: :ductwork_pipelines }
       table.string :klass, null: false
-      table.string :step_type, null: false
+      table.string :to_transition, null: false
       table.timestamp :started_at
       table.timestamp :completed_at
       table.string :status, null: false
+      table.integer :delay_seconds
+      table.integer :timeout_seconds
       table.timestamps null: false
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Tyler Ewing