ductwork 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f64abfa6d358cd7b225baae1b801894032936bef4de86ff0843e3163552aa530
-  data.tar.gz: aedd9daa71379091503375926ea4863a7834193a1701ee265a9b0b99a7fbfe50
+  metadata.gz: 9edf0e9d07be90e39d099d4f2f9a65677b4b37144c8fa5c7d3d0a88ab81ed1a3
+  data.tar.gz: 91b2177670ee8333062ed6182512f4c5b2cd3ecfc3db9c36a77fcdf54397f65a
 SHA512:
-  metadata.gz: 85598e36b5edb64f33217d19cb2f2af66e6991c68f74c0a8ede9c445d88884bd804da2b381b2dd9cdeaa6a6936a340628af25c45ff259632f36d1c4429665bcb
-  data.tar.gz: 4f59ad232e7a00c8e34403dfb8412af7e7351f0d921b3fd7fc6c27ee10524a1bd37e57bd18aa020c9e7daf1c3374127927e54474302364f3ee6962d157b9e468
+  metadata.gz: 7583ac23a38875a63ac31b333348ddc9ec4e35b431ec90b335eb2580a241c508acbfcfdc462957a2ce39f5c7cb6094d146f60b8b4e49b753bfc0ead13967bcc4
+  data.tar.gz: 466d021eb4635d0d7ba284569100905069598f5c58edbdc3480e9db3d2de2ba09485b72935280598ac8223374cefebfa16b5d3c60a17709df9d2455a8c1273c5

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Ductwork Changelog
 
+## [0.9.0]
+
+- feat: add health check to job worker runner process - this is a basic check if a thread is healthy via `Thread#alive?` and restarts the thread if it is dead
+
+## [0.8.1]
+
+- fix: properly wrap "units of work" in rails application executor in pipeline advancer
+- fix: remove wrapping thread creation with the rails application executor - these threads are long-running so they should not be wrapped; later commits will wrap each individual "unit of work" with the executor as recommended
+- fix: move pipeline advancer creation into thread initialization block - this effectively doesn't change anything but is useful in case we need to do something on the advancer thread in the initializer
+- fix: move job worker creation into thread initialization block - this effectively doesn't change anything but is useful in case we need to do something on the worker thread in the initializer
+
 ## [0.8.0]
 
 - chore: re-organize `Ductwork::CLI` class

data/LICENSE.txt

@@ -4,7 +4,7 @@ the LGPLv3.0 license. Please see below for license text:
                    GNU LESSER GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
- Copyright (c) 2024-2025 Tyler Ewing
+ Copyright (c) 2024-2025 Pen and Paper Solutions LLC
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 

data/lib/ductwork/processes/job_worker.rb

@@ -3,18 +3,44 @@
 module Ductwork
   module Processes
     class JobWorker
-      def initialize(pipeline, running_context)
+      attr_reader :thread, :last_hearthbeat_at
+
+      def initialize(pipeline, id)
         @pipeline = pipeline
-        @running_context = running_context
+        @id = id
+        @running_context = Ductwork::RunningContext.new
+        @thread = nil
+        @last_hearthbeat_at = Time.current
+      end
+
+      def start
+        @thread = Thread.new { work_loop }
+        @thread.name = "ductwork.job_worker.#{id}"
+      end
+
+      alias restart start
+
+      def alive?
+        thread&.alive? || false
+      end
+
+      def stop
+        running_context.shutdown!
       end
 
-      def run
+      private
+
+      attr_reader :pipeline, :id, :running_context
+
+      def work_loop
         run_hooks_for(:start)
+
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker,
           pipeline: pipeline
         )
+
         while running_context.running?
           Ductwork.logger.debug(
             msg: "Attempting to claim job",
@@ -37,21 +63,16 @@ module Ductwork
             )
             sleep(polling_timeout)
           end
-        end
-
-        shutdown
-      end
-
-      private
 
-      attr_reader :pipeline, :running_context
+          @last_hearthbeat_at = Time.current
+        end
 
-      def shutdown
         Ductwork.logger.debug(
           msg: "Shutting down",
           role: :job_worker,
           pipeline: pipeline
         )
+
         run_hooks_for(:stop)
       end
 

data/lib/ductwork/processes/job_worker_runner.rb

@@ -6,7 +6,7 @@ module Ductwork
       def initialize(pipeline)
         @pipeline = pipeline
         @running_context = Ductwork::RunningContext.new
-        @threads = create_threads
+        @job_workers = []
 
         Signal.trap(:INT) { running_context.shutdown! }
         Signal.trap(:TERM) { running_context.shutdown! }
@@ -24,7 +24,9 @@ module Ductwork
       end
 
       def run
-        create_process!
+        create_process_record!
+        start_job_workers
+
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker_runner,
@@ -34,7 +36,7 @@ module Ductwork
         while running?
           # TODO: Increase or make configurable
           sleep(5)
-          attempt_synchronize_threads
+          check_thread_health
           report_heartbeat!
         end
 
@@ -43,45 +45,29 @@ module Ductwork
 
       private
 
-      attr_reader :pipeline, :running_context, :threads
+      attr_reader :pipeline, :running_context, :job_workers
 
-      def worker_count
-        Ductwork.configuration.job_worker_count(pipeline)
+      def create_process_record!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.create!(
+            pid: ::Process.pid,
+            machine_identifier: Ductwork::MachineIdentifier.fetch,
+            last_heartbeat_at: Time.current
+          )
+        end
       end
 
-      def create_threads
-        worker_count.times.map do |i|
-          job_worker = Ductwork::Processes::JobWorker.new(
-            pipeline,
-            running_context
-          )
-          Ductwork.logger.debug(
-            msg: "Creating new thread",
-            role: :job_worker_runner,
-            pipeline: pipeline
-          )
-          thread = Thread.new do
-            job_worker.run
-          end
-          thread.name = "ductwork.job_worker.#{i}"
+      def start_job_workers
+        Ductwork.configuration.job_worker_count(pipeline).times do |i|
+          job_worker = Ductwork::Processes::JobWorker.new(pipeline, i)
+          job_workers.push(job_worker)
+          job_worker.start
 
           Ductwork.logger.debug(
-            msg: "Created new thread",
+            msg: "Created new job worker",
             role: :job_worker_runner,
             pipeline: pipeline
           )
-
-          thread
-        end
-      end
-
-      def create_process!
-        Ductwork.wrap_with_app_executor do
-          Ductwork::Process.create!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch,
-            last_heartbeat_at: Time.current
-          )
         end
       end
 
@@ -89,13 +75,17 @@ module Ductwork
         running_context.running?
       end
 
-      def attempt_synchronize_threads
+      def check_thread_health
         Ductwork.logger.debug(
           msg: "Attempting to synchronize threads",
           role: :job_worker_runner,
           pipeline: pipeline
         )
-        threads.each { |thread| thread.join(0.1) }
+        job_workers.each do |job_worker|
+          if !job_worker.alive?
+            job_worker.restart
+          end
+        end
         Ductwork.logger.debug(
           msg: "Synchronizing threads timed out",
           role: :job_worker_runner,
@@ -113,31 +103,36 @@ module Ductwork
 
       def shutdown!
         running_context.shutdown!
+        job_workers.each(&:stop)
         await_threads_graceful_shutdown
-        kill_remaining_threads
-        delete_process
+        kill_remaining_job_workers
+        delete_process_record!
       end
 
       def await_threads_graceful_shutdown
         timeout = Ductwork.configuration.job_worker_shutdown_timeout
         deadline = Time.current + timeout
 
-        Ductwork.logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
-        while Time.current < deadline && threads.any?(&:alive?)
-          threads.each do |thread|
+        Ductwork.logger.debug(
+          msg: "Attempting graceful shutdown",
+          role: :job_worker_runner
+        )
+
+        while Time.current < deadline && job_workers.any?(&:alive?)
+          job_workers.each do |job_worker|
             break if Time.current < deadline
 
             # TODO: Maybe make this configurable. If there's a ton of workers
             # it may not even get to the "later" ones depending on the timeout
-            thread.join(1)
+            job_worker.thread.join(1)
           end
         end
       end
 
-      def kill_remaining_threads
-        threads.each do |thread|
-          if thread.alive?
-            thread.kill
+      def kill_remaining_job_workers
+        job_workers.each do |job_worker|
+          if job_worker.alive?
+            job_worker.thread.kill
             Ductwork.logger.debug(
               msg: "Killed thread",
               role: :job_worker_runner,
@@ -147,7 +142,7 @@ module Ductwork
         end
       end
 
-      def delete_process
+      def delete_process_record!
         Ductwork.wrap_with_app_executor do
           Ductwork::Process.find_by!(
             pid: ::Process.pid,

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -10,8 +10,9 @@ module Ductwork
 
       def run # rubocop:todo Metrics/AbcSize, Metrics/MethodLength
         run_hooks_for(:start)
+
         while running_context.running?
-          id = Ductwork::Record.uncached do
+          id = Ductwork.wrap_with_app_executor do
             Ductwork::Pipeline
               .in_progress
               .where(klass: klass, claimed_for_advancing_at: nil)
@@ -24,12 +25,14 @@ module Ductwork
           end
 
           if id.present?
-            rows_updated = Ductwork::Pipeline
-                           .where(id: id, claimed_for_advancing_at: nil)
-                           .update_all(
-                             claimed_for_advancing_at: Time.current,
-                             status: "advancing"
-                           )
+            rows_updated = Ductwork.wrap_with_app_executor do
+              Ductwork::Pipeline
+                .where(id: id, claimed_for_advancing_at: nil)
+                .update_all(
+                  claimed_for_advancing_at: Time.current,
+                  status: "advancing"
+                )
+            end
 
             if rows_updated == 1
               Ductwork.logger.debug(
@@ -39,23 +42,31 @@ module Ductwork
                 role: :pipeline_advancer
               )
 
-              pipeline = Ductwork::Pipeline.find(id)
-              pipeline.advance!
+              pipeline = Ductwork.wrap_with_app_executor do
+                pipeline = Ductwork::Pipeline.find(id)
+                pipeline.advance!
 
-              Ductwork.logger.debug(
-                msg: "Pipeline advanced",
-                pipeline_id: id,
-                pipeline: klass,
-                role: :pipeline_advancer
-              )
+                Ductwork.logger.debug(
+                  msg: "Pipeline advanced",
+                  pipeline_id: id,
+                  pipeline: klass,
+                  role: :pipeline_advancer
+                )
 
-              # release the pipeline and set last advanced at so it doesnt block.
-              # we're not using a queue so we have to use a db timestamp
-              pipeline.update!(
-                claimed_for_advancing_at: nil,
-                last_advanced_at: Time.current,
-                status: "in_progress"
-              )
+                # rubocop:todo Metrics/BlockNesting
+                status = pipeline.completed? ? "completed" : "in_progress"
+                # rubocop:enable Metrics/BlockNesting
+
+                # release the pipeline and set last advanced at so it doesn't
+                # block. we're not using a queue so we have to use a db
+                # timestamp
+              ensure
+                pipeline.update!(
+                  claimed_for_advancing_at: nil,
+                  last_advanced_at: Time.current,
+                  status: status
+                )
+              end
             else
               Ductwork.logger.debug(
                 msg: "Did not claim pipeline, avoided race condition",

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -46,20 +46,15 @@ module Ductwork
 
       def create_threads
         klasses.map do |klass|
-          pipeline_advancer = Ductwork::Processes::PipelineAdvancer.new(
-            running_context,
-            klass
-          )
-
           Ductwork.logger.debug(
             msg: "Creating new thread",
             role: :pipeline_advancer_runner,
             pipeline: klass
           )
           thread = Thread.new do
-            Ductwork.wrap_with_app_executor do
-              pipeline_advancer.run
-            end
+            Ductwork::Processes::PipelineAdvancer
+              .new(running_context, klass)
+              .run
           end
           thread.name = "ductwork.pipeline_advancer.#{klass}"
 

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.8.0"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Tyler Ewing