ductwork 0.17.0 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08a25902c240e59967cbd79257cfaa9f7aabfc9418ac6324864c2264b0be0f4f'
-  data.tar.gz: 33e7eee080dfbc1e993ccf60388694e560eaa95c01d6c57797775351a4063686
+  metadata.gz: 4e9cbff3e15bf798c3ff2019f8b5637459b2b495f30c74e3b352bff97afa3b6b
+  data.tar.gz: 9d7a023b9c0018fe5aa8dc9291981146f0246deb6d8c1b98f658de146b2b78c5
 SHA512:
-  metadata.gz: 992ef0a0f5e6c971a3789d0ce3ea8da7753f32f354967dd788024393b6458d3b01089dbdd4a301478b59b316a7bd950bba205371ff431db35944e61920d739bf
-  data.tar.gz: 4fd645e68b7b66d3e5186418e3e2f18faceba41b05e700a3e7fd6a0d499abca32c3f021e95720f6350b78520bb5668df7f5e2d4bbb8b343e93cd8a84d4292af5
+  metadata.gz: 2c019906fc749f7a0b47e3094823f3c867068bcabb0fea945ad47f017ac9ae39f659f5ca2b6a6741a666ad53eb0867720a0ce9ec2b57fac3e188b87ad6055dae
+  data.tar.gz: 1f584787c2f6588a86e59cbe6a2dac6fdaf014ff5b1b9632dc6816b1648d756267cdff77df7ffe06ba10ba74a060b72e0611a60f863d875a84981612319cc1a0

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Ductwork Changelog
 
+## [0.19.0]
+
+- chore: bump rails-related dependencies to v8.1.2
+- chore: remove ruby v3.2.9 from CI testing matrix - support is ending in March '26 but it's being removed now to better support edge rails
+- feat: loosen rails version constraint to allow rails edge
+- feat: respect "role" configuration when booting - ie. run main process as supervisor, pipeline advancer, or job worker
+- feat: allow job worker runner to take a collection of pipelines and create workers for all configured pipelines - this will only happen when the "role" configuration is "worker" otherwise a process will be spun up for each pipeline
+- feat: add "role" configuration - to be used to set the role of the entire ductwork running instance
+
+## [0.18.0]
+
+- fix: show countdown for pipelines/steps scheduled in the future
+- feat: add input and output arguments to pipeline dashboard page
+- feat: add RSpec matcher for asserting pipeline context
+
 ## [0.17.0]
 
 - feat: assign step node names unique hex values in the pipeline definition

data/README.md

@@ -46,7 +46,7 @@ default: &default
   pipelines: "*"
 ```
 
-See the [Configuration Guide](https://docs.getductwork.io/advanced/configuration.html) for all available options including thread counts, timeouts, and database settings.
+See the [Configuration Guide](https://docs.getductwork.io/getting-started/configuration.html) for all available options including thread counts, timeouts, and database settings.
 
 ## Usage
 

data/app/assets/javascripts/ductwork/application.js

@@ -7,7 +7,7 @@ document.addEventListener('DOMContentLoaded', function() {
 
     function updateTimer() {
       const now = new Date();
-      const elapsed = Math.floor((now - startedAt) / 1000);
+      const elapsed = Math.floor(Math.abs(now - startedAt) / 1000);
 
       const hours = Math.floor(elapsed / 3600);
       const minutes = Math.floor((elapsed % 3600) / 60);
@@ -17,7 +17,11 @@ document.addEventListener('DOMContentLoaded', function() {
       const minutesStr = minutes == 0 ? '' : `${String(minutes)}m `;
       const secondsStr = seconds == 0 ? '' : `${String(seconds)}s`;
 
-      displayElement.textContent = hoursStr + minutesStr + secondsStr;
+      if (now > startedAt) {
+        displayElement.textContent = hoursStr + minutesStr + secondsStr;
+      } else {
+        displayElement.textContent = 'In ' + hoursStr + minutesStr + secondsStr;
+      }
     }
 
     updateTimer();

data/app/views/ductwork/pipelines/show.html.erb

@@ -27,16 +27,22 @@
       </div>
 
       <div>
-        <div class="metric-heading">
-          <label>Triggered At</label>
+        <div>
+          <label>Runtime</label>
           <span>
-            <%= @pipeline.triggered_at.iso8601(3) %>
+            <% if @pipeline.completed_at.nil? %>
+              <div data-started-at=<%= @pipeline.started_at.iso8601 %>>
+                <span id="elapsed-timer">0h 0m 0s</span>
+              </div>
+            <% else %>
+              <%= formatted_time_distance(@pipeline.started_at, @pipeline.completed_at) %>
+            <% end %>
           </span>
         </div>
-        <div>
-          <label>Step Count</label>
+        <div class="metric-heading">
+          <label>Triggered At</label>
           <span>
-            <%= @pipeline.steps.count %>
+            <%= @pipeline.triggered_at.iso8601(3) %>
           </span>
         </div>
       </div>
@@ -59,13 +65,13 @@
 
     <div style="margin-top: 1rem;">
       <label>Definition</label>
-      <code>
+      <code style="width: 100%;">
         <%= @pipeline.parsed_definition %>
       </code>
     </div>
   </article>
 
-  <h3>Steps</h3>
+  <h3>Steps (<%= @pipeline.steps.count %>)</h3>
 
   <div class="separate-content">
     <div class="filter-group">
@@ -128,6 +134,13 @@
                 </div>
               </div>
 
+              <div>
+                <label>Class</label>
+                <code>
+                  <%= step.klass %>
+                </code>
+              </div>
+
               <div>
                 <label>Transition</label>
                 <code>
@@ -163,16 +176,31 @@
           <div>
             <div class="grid">
               <div>
-                <label>Node</label>
+                <label>Input</label>
                 <code>
-                  <%= step.node %>
+                  <% if step.job.input_args.present? %>
+                    <%= JSON.parse(step.job.input_args).dig("args", 0).inspect %>
+                  <% else %>
+                    no value
+                  <% end %>
                 </code>
               </div>
 
               <div>
-                <label>Class</label>
+                <label>Output</label>
                 <code>
-                  <%= step.klass %>
+                  <% if step.job.output_payload.present? %>
+                    <%= JSON.parse(step.job.output_payload)["payload"].inspect %>
+                  <% else %>
+                    no value
+                  <% end %>
+                </code>
+              </div>
+
+              <div>
+                <label>Node</label>
+                <code>
+                  <%= step.node %>
                 </code>
               </div>
 
@@ -189,7 +217,6 @@
                   <%= step.completed_at&.iso8601(3) %>
                 </div>
               </div>
-
             </div>
 
             <div style="margin-top: 1rem;">

data/lib/ductwork/cli.rb

@@ -17,7 +17,7 @@ module Ductwork
       option_parser.parse!(args)
       auto_configure
       puts banner
-      supervisor_runner.start!
+      launch_processes
     end
 
     private
@@ -45,6 +45,7 @@ module Ductwork
     end
 
     def auto_configure
+      options[:role] = ENV.fetch("DUCTWORK_ROLE", nil)
       Ductwork.configuration = Configuration.new(**options)
       Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
                           Rails.logger
@@ -63,17 +64,17 @@ module Ductwork
   ██║  ██║██║   ██║██║        ██║   ██║███╗██║██║   ██║██╔══██╗██╔═██╗
   ██████╔╝╚██████╔╝╚██████╗   ██║   ╚███╔███╔╝╚██████╔╝██║  ██║██║  ██╗
   ╚═════╝  ╚═════╝  ╚═════╝   ╚═╝    ╚══╝╚══╝  ╚═════╝ ╚═╝  ╚═╝╚═╝  ╚═╝
-   ▒▒▓  ▒ ░▒▓▒ ▒ ▒ ░ ░▒ ▒  ░  ▒ ░░   ░ ▓░▒ ▒  ░ ▒░▒░▒░ ░ ▒▓ ░▒▓░▒ ▒▒ ▓▒
-    ░ ▒  ▒ ░░▒░ ░ ░   ░  ▒       ░      ▒ ░ ░    ░ ▒ ▒░   ░▒ ░ ▒░░ ░▒ ▒░
-     ░ ░  ░  ░░░ ░ ░ ░          ░        ░   ░  ░ ░ ░ ▒    ░░   ░ ░ ░░ ░
-        ░       ░     ░ ░                    ░        ░ ░     ░     ░  ░
-      ░               ░
+  ▒▒▓  ▒ ░▒▓▒ ▒ ▒ ░ ░▒ ▒  ░  ▒ ░░   ░ ▓░▒ ▒  ░ ▒░▒░▒░ ░ ▒▓ ░▒▓░▒ ▒▒ ▓▒
+   ░ ▒  ▒ ░░▒░ ░ ░   ░  ▒       ░      ▒ ░ ░    ░ ▒ ▒░   ░▒ ░ ▒░░ ░▒ ▒░
+    ░ ░  ░  ░░░ ░ ░ ░          ░        ░   ░  ░ ░ ░ ▒    ░░   ░ ░ ░░ ░
+       ░       ░     ░ ░                    ░        ░ ░     ░     ░  ░
+     ░               ░
   \e[0m
       BANNER
     end
 
-    def supervisor_runner
-      Ductwork::Processes::SupervisorRunner
+    def launch_processes
+      Ductwork::Processes::Launcher.start_processes!
     end
   end
 end

data/lib/ductwork/configuration.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Ductwork
-  class Configuration
+  class Configuration # rubocop:todo Metrics/ClassLength
     DEFAULT_ENV = :default
     DEFAULT_FILE_PATH = "config/ductwork.yml"
     DEFAULT_JOB_WORKER_COUNT = 5 # threads
@@ -12,11 +12,15 @@ module Ductwork
     DEFAULT_LOGGER_SOURCE = "default" # `Logger` instance writing to STDOUT
     DEFAULT_PIPELINE_POLLING_TIMEOUT = 1 # second
     DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT = 20 # seconds
+    DEFAULT_ROLE = "all" # supervisor, pipeline advancer, and job workers
     DEFAULT_STEPS_MAX_DEPTH = -1 # unlimited count
     DEFAULT_SUPERVISOR_POLLING_TIMEOUT = 1 # second
     DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT = 30 # seconds
     DEFAULT_LOGGER = ::Logger.new($stdout)
     PIPELINES_WILDCARD = "*"
+    VALID_ROLES = %w[all advancer worker].freeze
+
+    class InvalidRoleError < StandardError; end
 
     attr_writer :job_worker_count, :job_worker_polling_timeout,
                 :job_worker_shutdown_timeout, :job_worker_max_retry,
@@ -25,13 +29,23 @@ module Ductwork
                 :steps_max_depth,
                 :supervisor_polling_timeout, :supervisor_shutdown_timeout
 
-    def initialize(path: DEFAULT_FILE_PATH)
+    def initialize(path: DEFAULT_FILE_PATH, role: nil)
       full_path = Pathname.new(path)
       data = ActiveSupport::ConfigurationFile.parse(full_path).deep_symbolize_keys
       env = defined?(Rails) ? Rails.env.to_sym : DEFAULT_ENV
-      @config = data[env]
+      @config = if role.present?
+                  data[env].merge(role:)
+                else
+                  data[env]
+                end
     rescue Errno::ENOENT
-      @config = {}
+      @config = { role: }.compact
+    end
+
+    def role
+      r = config[:role] || DEFAULT_ROLE
+      validate_role!(r)
+      r
     end
 
     def pipelines
@@ -185,5 +199,11 @@ module Ductwork
       config.dig(:supervisor, :shutdown_timeout) ||
         DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT
     end
+
+    def validate_role!(role)
+      if VALID_ROLES.exclude?(role)
+        raise InvalidRoleError, "Must use a valid role"
+      end
+    end
   end
 end

data/lib/ductwork/processes/job_worker.rb

@@ -3,7 +3,7 @@
 module Ductwork
   module Processes
     class JobWorker
-      attr_reader :thread, :last_heartbeat_at, :job
+      attr_reader :thread, :last_heartbeat_at, :job, :pipeline
 
       def initialize(pipeline, id)
         @pipeline = pipeline
@@ -30,7 +30,7 @@ module Ductwork
 
       private
 
-      attr_reader :pipeline, :id, :running_context
+      attr_reader :id, :running_context
 
       def work_loop
         run_hooks_for(:start)

data/lib/ductwork/processes/job_worker_runner.rb

@@ -3,8 +3,8 @@
 module Ductwork
   module Processes
     class JobWorkerRunner
-      def initialize(pipeline)
-        @pipeline = pipeline
+      def initialize(*pipelines)
+        @pipelines = pipelines
         @running_context = Ductwork::RunningContext.new
         @job_workers = []
 
@@ -30,7 +30,7 @@ module Ductwork
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker_runner,
-          pipeline: pipeline
+          pipelines: pipelines
         )
 
         while running?
@@ -45,7 +45,7 @@ module Ductwork
 
       private
 
-      attr_reader :pipeline, :running_context, :job_workers
+      attr_reader :pipelines, :running_context, :job_workers
 
       def create_process_record!
         Ductwork.wrap_with_app_executor do
@@ -58,16 +58,18 @@ module Ductwork
       end
 
       def start_job_workers
-        Ductwork.configuration.job_worker_count(pipeline).times do |i|
-          job_worker = Ductwork::Processes::JobWorker.new(pipeline, i)
-          job_workers.push(job_worker)
-          job_worker.start
-
-          Ductwork.logger.debug(
-            msg: "Created new job worker",
-            role: :job_worker_runner,
-            pipeline: pipeline
-          )
+        pipelines.each do |pipeline|
+          Ductwork.configuration.job_worker_count(pipeline).times do |i|
+            job_worker = Ductwork::Processes::JobWorker.new(pipeline, i)
+            job_workers.push(job_worker)
+            job_worker.start
+
+            Ductwork.logger.debug(
+              msg: "Created new job worker",
+              role: :job_worker_runner,
+              pipeline: pipeline
+            )
+          end
         end
       end
 
@@ -79,23 +81,23 @@ module Ductwork
         Ductwork.logger.debug(
           msg: "Checking thread health",
           role: :job_worker_runner,
-          pipeline: pipeline
+          pipelines: pipelines
         )
         job_workers.each do |job_worker|
           if !job_worker.alive?
             job_worker.restart
 
             Ductwork.logger.info(
-              msg: "Restarted thread",
+              msg: "Restarted job worker",
               role: :job_worker_runner,
-              pipeline: pipeline
+              pipeline: job_worker.pipeline
             )
           end
         end
         Ductwork.logger.debug(
           msg: "Checked thread health",
           role: :job_worker_runner,
-          pipeline: pipeline
+          pipelines: pipelines
         )
       end
 

data/lib/ductwork/processes/launcher.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module Processes
+    class Launcher
+      def self.start_processes!
+        new.start_processes!
+      end
+
+      def initialize
+        @pipelines = Ductwork.configuration.pipelines
+        @runner_klass = case Ductwork.configuration.role
+                        when "all"
+                          supervisor_runner
+                        when "advancer"
+                          pipeline_advancer_runner
+                        when "worker"
+                          job_worker_runner
+                        end
+      end
+
+      def start_processes!
+        runner_klass
+          .new(*pipelines)
+          .run
+      end
+
+      private
+
+      attr_reader :pipelines, :runner_klass
+
+      def supervisor_runner
+        Ductwork::Processes::SupervisorRunner
+      end
+
+      def pipeline_advancer_runner
+        Ductwork::Processes::PipelineAdvancerRunner
+      end
+
+      def job_worker_runner
+        Ductwork::Processes::JobWorkerRunner
+      end
+    end
+  end
+end

data/lib/ductwork/processes/supervisor_runner.rb

@@ -3,32 +3,37 @@
 module Ductwork
   module Processes
     class SupervisorRunner
-      def self.start!
-        supervisor = Ductwork::Processes::Supervisor.new
-        pipelines_to_advance = Ductwork.configuration.pipelines
+      def initialize(*pipelines)
+        @pipelines = pipelines
+        @supervisor = Ductwork::Processes::Supervisor.new
+      end
 
-        supervisor.add_worker(metadata: { pipelines: pipelines_to_advance }) do
+      def run
+        supervisor.add_worker(metadata: { pipelines: }) do
           Ductwork.logger.debug(
             msg: "Starting Pipeline Advancer process",
             role: :supervisor_runner
           )
-          Ductwork::Processes::PipelineAdvancerRunner
-            .new(*pipelines_to_advance).run
+          Ductwork::Processes::PipelineAdvancerRunner.new(*pipelines).run
         end
 
-        pipelines_to_advance.each do |pipeline|
+        pipelines.each do |pipeline|
           supervisor.add_worker(metadata: { pipeline: }) do
             Ductwork.logger.debug(
               msg: "Starting Job Worker Runner process",
               role: :supervisor_runner,
               pipeline: pipeline
             )
-            Ductwork::Processes::JobWorkerRunner.new(pipeline).run
+            Ductwork::Processes::JobWorkerRunner.new(*pipeline).run
           end
         end
 
         supervisor.run
       end
+
+      private
+
+      attr_reader :pipelines, :supervisor
     end
   end
 end

data/lib/ductwork/testing/rspec.rb

@@ -62,6 +62,27 @@ RSpec::Matchers.define(:have_triggered_pipelines) do |*expected|
   end
 end
 
+RSpec::Matchers.define(:have_set_context) do |expected|
+  supports_block_expectations
+
+  match do |block|
+    ctx = Ductwork::Context.new(pipeline_id)
+    before_values = expected.map { |k, _| ctx.get(k.to_s) }
+
+    block.call
+
+    after_context = expected.to_h { |k, _| [k, ctx.get(k.to_s)] }
+
+    before_values.all?(&:nil?) && after_context == expected
+  end
+
+  chain :for_pipeline, :pipeline_id
+
+  failure_message do
+    "Context does not match expected result"
+  end
+end
+
 module Ductwork
   module Testing
     module RSpec

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.17.0"
+  VERSION = "0.19.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.19.0
 platform: ruby
 authors:
 - Tyler Ewing
@@ -18,7 +18,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,7 +28,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -48,7 +48,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -68,7 +68,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -78,7 +78,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -88,7 +88,7 @@ dependencies:
         version: '7.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8.2'
+        version: '8.3'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -153,6 +153,7 @@ files:
 - lib/ductwork/models/tuple.rb
 - lib/ductwork/processes/job_worker.rb
 - lib/ductwork/processes/job_worker_runner.rb
+- lib/ductwork/processes/launcher.rb
 - lib/ductwork/processes/pipeline_advancer.rb
 - lib/ductwork/processes/pipeline_advancer_runner.rb
 - lib/ductwork/processes/supervisor.rb