ductwork 0.7.2 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3a59fb73965f325d3e23243f59e7e973419f3683d2de741cf5bbf115bccad8d
-  data.tar.gz: d66a5c291d197b7a8e679d23e9c68b6575d2e35100b06f7c412c97e5bc515ec3
+  metadata.gz: 6fde482283602dcf363baed46b5d8468757ddb8f6cc6fabb9b9d5e37497e10f4
+  data.tar.gz: 3aa0f1a9aa6f77b47997b64b246ed3dc7371e63554943110de8e06f98135f9d3
 SHA512:
-  metadata.gz: de5ca4b1c9dff4478992234183ae84d810cec930eede8d97dc7511959efad9521acb201ec5f39a2d96d430383ee3f240d3a3395bc55fc19ce272603000eb92a2
-  data.tar.gz: b86ea8a91f912ae55e509f2c059fc8db8d4ccff0055fe96db77593a98ce188255b2bbcb395709a0b672ded6b8df1a1537001fc56cfb3de7244a5dda9a4e48192
+  metadata.gz: bc475be1df97dada6099f8210b5694cbc4d90e5466d565a50674bac796fc252923f19a18e6beae3cf673db8d25fccf4bfb8178622d8574fd0e2b911e27dc8a0f
+  data.tar.gz: e1caa07d54fe23a2f2ea1b90809371a53bfd791eba6751bab9b65dac254c7f7935cf54f4b217adb68e084abd478192000090c91005d644188f64e2d7930264fd

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Ductwork Changelog
 
+## [0.8.1]
+
+- fix: properly wrap "units of work" in rails application executor in pipeline advancer
+- fix: remove wrapping thread creation with the rails application executor - these threads are long-running so they should not be wrapped; later commits will wrap each individual "unit of work" with the executor as recommended
+- fix: move pipeline advancer creation into thread initialization block - this effectively doesn't change anything but is useful in case we need to do something on the advancer thread in the initializer
+- fix: move job worker creation into thread initialization block - this effectively doesn't change anything but is useful in case we need to do something on the worker thread in the initializer
+
+## [0.8.0]
+
+- chore: re-organize `Ductwork::CLI` class
+- feat!: better name `ductwork_steps.step_type` to `to_transition` - the column rename is a breaking change but not bumping major version since the gem is still pre-1.0
+- feat: add `delay_seconds` and `timeout_seconds` columns to `ductwork_steps` table
+- feat: set pipeline status to `advancing` once pipeline is claimed and advancing starts
+- chore: add `advancing` status enum value to `Pipeline`
+
 ## [0.7.2]
 
 - chore: small DRY refactor in pipeline advancement

data/LICENSE.txt

@@ -4,7 +4,7 @@ the LGPLv3.0 license. Please see below for license text:
                    GNU LESSER GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
- Copyright (c) 2024-2025 Tyler Ewing
+ Copyright (c) 2024-2025 Pen and Paper Solutions LLC
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 

data/lib/ductwork/cli.rb

@@ -4,45 +4,57 @@ require "optparse"
 
 module Ductwork
   class CLI
-    class << self
-      def start!(args)
-        options = parse_options(args)
-        Ductwork.configuration = Configuration.new(**options)
-        Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
-                            Rails.logger
-                          else
-                            Ductwork::Configuration::DEFAULT_LOGGER
-                          end
-        Ductwork.logger.level = Ductwork.configuration.logger_level
-
-        Ductwork::Processes::SupervisorRunner.start!
-      end
+    def self.start!(args)
+      new(args).start!
+    end
 
-      private
+    def initialize(args)
+      @args = args
+      @options = {}
+    end
+
+    def start!
+      option_parser.parse!(args)
+      auto_configure
+      supervisor_runner.start!
+    end
 
-      def parse_options(args)
-        options = {}
+    private
 
-        OptionParser.new do |op|
-          op.banner = "ductwork [options]"
+    attr_reader :args, :options
 
-          op.on("-c", "--config PATH", "path to YAML config file") do |arg|
-            options[:path] = arg
-          end
+    def option_parser
+      OptionParser.new do |op|
+        op.banner = "ductwork [options]"
 
-          op.on("-h", "--help", "Prints this help") do
-            puts op
-            exit
-          end
+        op.on("-c", "--config PATH", "path to YAML config file") do |arg|
+          options[:path] = arg
+        end
 
-          op.on("-v", "--version", "Prints the version") do
-            puts "Ductwork #{Ductwork::VERSION}"
-            exit
-          end
-        end.parse!(args)
+        op.on("-h", "--help", "Prints this help") do
+          puts op
+          exit
+        end
 
-        options
+        op.on("-v", "--version", "Prints the version") do
+          puts "Ductwork #{Ductwork::VERSION}"
+          exit
+        end
       end
     end
+
+    def auto_configure
+      Ductwork.configuration = Configuration.new(**options)
+      Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
+                          Rails.logger
+                        else
+                          Ductwork::Configuration::DEFAULT_LOGGER
+                        end
+      Ductwork.logger.level = Ductwork.configuration.logger_level
+    end
+
+    def supervisor_runner
+      Ductwork::Processes::SupervisorRunner
+    end
   end
 end

data/lib/ductwork/models/pipeline.rb

@@ -16,6 +16,7 @@ module Ductwork
          pending: "pending",
          in_progress: "in_progress",
          waiting: "waiting",
+         advancing: "advancing",
          halted: "halted",
          completed: "completed"
 
@@ -71,7 +72,7 @@ module Ductwork
           step = p.steps.create!(
             klass: step_klass,
             status: :in_progress,
-            step_type: :start,
+            to_transition: :start,
             started_at: Time.current
           )
           Ductwork::Job.enqueue(step, args)
@@ -118,10 +119,10 @@ module Ductwork
       started_at = Time.current
       # NOTE: "chain" is used by ActiveRecord so we have to call
       # this enum value "default" :sad:
-      step_type = edge[:type] == "chain" ? "default" : edge[:type]
+      to_transition = edge[:type] == "chain" ? "default" : edge[:type]
       klass ||= edge.fetch(:to).sole
 
-      next_step = steps.create!(klass:, status:, step_type:, started_at:)
+      next_step = steps.create!(klass:, status:, to_transition:, started_at:)
       Ductwork::Job.enqueue(next_step, input_arg)
     end
 
@@ -174,17 +175,17 @@ module Ductwork
     end
 
     def advance_non_merging_steps(step_klass, edge, advancing_ids)
-      step_type = edge[:type]
+      to_transition = edge[:type]
 
       steps.where(id: advancing_ids, klass: step_klass).find_each do |step|
-        if step_type.in?(%w[chain divide])
+        if to_transition.in?(%w[chain divide])
           advance_to_next_steps(step.id, edge)
-        elsif step_type == "expand"
+        elsif to_transition == "expand"
           expand_to_next_steps(step.id, edge)
         else
           Ductwork.logger.error(
-            msg: "Invalid step type",
-            step_type: step_type,
+            msg: "Invalid To Transition",
+            to_transition: to_transition,
             pipeline_id: id,
             role: :pipeline_advancer
           )

data/lib/ductwork/models/step.rb

@@ -7,7 +7,7 @@ module Ductwork
 
     validates :klass, presence: true
     validates :status, presence: true
-    validates :step_type, presence: true
+    validates :to_transition, presence: true
 
     enum :status,
          pending: "pending",
@@ -17,7 +17,7 @@ module Ductwork
          failed: "failed",
          completed: "completed"
 
-    enum :step_type,
+    enum :to_transition,
          start: "start",
          default: "default", # `chain` is used by AR
          divide: "divide",

data/lib/ductwork/processes/job_worker_runner.rb

@@ -51,17 +51,15 @@ module Ductwork
 
       def create_threads
         worker_count.times.map do |i|
-          job_worker = Ductwork::Processes::JobWorker.new(
-            pipeline,
-            running_context
-          )
           Ductwork.logger.debug(
             msg: "Creating new thread",
             role: :job_worker_runner,
             pipeline: pipeline
           )
           thread = Thread.new do
-            job_worker.run
+            Ductwork::Processes::JobWorker
+              .new(pipeline, running_context)
+              .run
           end
           thread.name = "ductwork.job_worker.#{i}"
 

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -10,8 +10,9 @@ module Ductwork
 
       def run # rubocop:todo Metrics/AbcSize, Metrics/MethodLength
         run_hooks_for(:start)
+
         while running_context.running?
-          id = Ductwork::Record.uncached do
+          id = Ductwork.wrap_with_app_executor do
             Ductwork::Pipeline
               .in_progress
               .where(klass: klass, claimed_for_advancing_at: nil)
@@ -24,9 +25,14 @@ module Ductwork
           end
 
           if id.present?
-            rows_updated = Ductwork::Pipeline
-                           .where(id: id, claimed_for_advancing_at: nil)
-                           .update_all(claimed_for_advancing_at: Time.current)
+            rows_updated = Ductwork.wrap_with_app_executor do
+              Ductwork::Pipeline
+                .where(id: id, claimed_for_advancing_at: nil)
+                .update_all(
+                  claimed_for_advancing_at: Time.current,
+                  status: "advancing"
+                )
+            end
 
             if rows_updated == 1
               Ductwork.logger.debug(
@@ -36,22 +42,31 @@ module Ductwork
                 role: :pipeline_advancer
               )
 
-              pipeline = Ductwork::Pipeline.find(id)
-              pipeline.advance!
+              pipeline = Ductwork.wrap_with_app_executor do
+                pipeline = Ductwork::Pipeline.find(id)
+                pipeline.advance!
 
-              Ductwork.logger.debug(
-                msg: "Pipeline advanced",
-                pipeline_id: id,
-                pipeline: klass,
-                role: :pipeline_advancer
-              )
+                Ductwork.logger.debug(
+                  msg: "Pipeline advanced",
+                  pipeline_id: id,
+                  pipeline: klass,
+                  role: :pipeline_advancer
+                )
 
-              # release the pipeline and set last advanced at so it doesnt block.
-              # we're not using a queue so we have to use a db timestamp
-              pipeline.update!(
-                claimed_for_advancing_at: nil,
-                last_advanced_at: Time.current
-              )
+                # rubocop:todo Metrics/BlockNesting
+                status = pipeline.completed? ? "completed" : "in_progress"
+                # rubocop:enable Metrics/BlockNesting
+
+                # release the pipeline and set last advanced at so it doesn't
+                # block. we're not using a queue so we have to use a db
+                # timestamp
+              ensure
+                pipeline.update!(
+                  claimed_for_advancing_at: nil,
+                  last_advanced_at: Time.current,
+                  status: status
+                )
+              end
             else
               Ductwork.logger.debug(
                 msg: "Did not claim pipeline, avoided race condition",

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -46,20 +46,15 @@ module Ductwork
 
       def create_threads
         klasses.map do |klass|
-          pipeline_advancer = Ductwork::Processes::PipelineAdvancer.new(
-            running_context,
-            klass
-          )
-
           Ductwork.logger.debug(
             msg: "Creating new thread",
             role: :pipeline_advancer_runner,
             pipeline: klass
           )
           thread = Thread.new do
-            Ductwork.wrap_with_app_executor do
-              pipeline_advancer.run
-            end
+            Ductwork::Processes::PipelineAdvancer
+              .new(running_context, klass)
+              .run
           end
           thread.name = "ductwork.pipeline_advancer.#{klass}"
 

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.7.2"
+  VERSION = "0.8.1"
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -5,10 +5,12 @@ class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>
     create_table :ductwork_steps do |table|
       table.belongs_to :pipeline, index: true, null: false, foreign_key: { to_table: :ductwork_pipelines }
       table.string :klass, null: false
-      table.string :step_type, null: false
+      table.string :to_transition, null: false
       table.timestamp :started_at
       table.timestamp :completed_at
       table.string :status, null: false
+      table.integer :delay_seconds
+      table.integer :timeout_seconds
       table.timestamps null: false
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.1
 platform: ruby
 authors:
 - Tyler Ewing