ductwork 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6fde482283602dcf363baed46b5d8468757ddb8f6cc6fabb9b9d5e37497e10f4
-  data.tar.gz: 3aa0f1a9aa6f77b47997b64b246ed3dc7371e63554943110de8e06f98135f9d3
+  metadata.gz: 9edf0e9d07be90e39d099d4f2f9a65677b4b37144c8fa5c7d3d0a88ab81ed1a3
+  data.tar.gz: 91b2177670ee8333062ed6182512f4c5b2cd3ecfc3db9c36a77fcdf54397f65a
 SHA512:
-  metadata.gz: bc475be1df97dada6099f8210b5694cbc4d90e5466d565a50674bac796fc252923f19a18e6beae3cf673db8d25fccf4bfb8178622d8574fd0e2b911e27dc8a0f
-  data.tar.gz: e1caa07d54fe23a2f2ea1b90809371a53bfd791eba6751bab9b65dac254c7f7935cf54f4b217adb68e084abd478192000090c91005d644188f64e2d7930264fd
+  metadata.gz: 7583ac23a38875a63ac31b333348ddc9ec4e35b431ec90b335eb2580a241c508acbfcfdc462957a2ce39f5c7cb6094d146f60b8b4e49b753bfc0ead13967bcc4
+  data.tar.gz: 466d021eb4635d0d7ba284569100905069598f5c58edbdc3480e9db3d2de2ba09485b72935280598ac8223374cefebfa16b5d3c60a17709df9d2455a8c1273c5

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Ductwork Changelog
 
+## [0.9.0]
+
+- feat: add health check to job worker runner process - this is a basic check if a thread is healthy via `Thread#alive?` and restarts the thread if it is dead
+
 ## [0.8.1]
 
 - fix: properly wrap "units of work" in rails application executor in pipeline advancer

data/lib/ductwork/processes/job_worker.rb

@@ -3,18 +3,44 @@
 module Ductwork
   module Processes
     class JobWorker
-      def initialize(pipeline, running_context)
+      attr_reader :thread, :last_hearthbeat_at
+
+      def initialize(pipeline, id)
         @pipeline = pipeline
-        @running_context = running_context
+        @id = id
+        @running_context = Ductwork::RunningContext.new
+        @thread = nil
+        @last_hearthbeat_at = Time.current
+      end
+
+      def start
+        @thread = Thread.new { work_loop }
+        @thread.name = "ductwork.job_worker.#{id}"
+      end
+
+      alias restart start
+
+      def alive?
+        thread&.alive? || false
+      end
+
+      def stop
+        running_context.shutdown!
       end
 
-      def run
+      private
+
+      attr_reader :pipeline, :id, :running_context
+
+      def work_loop
         run_hooks_for(:start)
+
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker,
           pipeline: pipeline
         )
+
         while running_context.running?
           Ductwork.logger.debug(
             msg: "Attempting to claim job",
@@ -37,21 +63,16 @@ module Ductwork
             )
             sleep(polling_timeout)
           end
-        end
-
-        shutdown
-      end
-
-      private
 
-      attr_reader :pipeline, :running_context
+          @last_hearthbeat_at = Time.current
+        end
 
-      def shutdown
         Ductwork.logger.debug(
           msg: "Shutting down",
           role: :job_worker,
           pipeline: pipeline
         )
+
         run_hooks_for(:stop)
       end
 

data/lib/ductwork/processes/job_worker_runner.rb

@@ -6,7 +6,7 @@ module Ductwork
       def initialize(pipeline)
         @pipeline = pipeline
         @running_context = Ductwork::RunningContext.new
-        @threads = create_threads
+        @job_workers = []
 
         Signal.trap(:INT) { running_context.shutdown! }
         Signal.trap(:TERM) { running_context.shutdown! }
@@ -24,7 +24,9 @@ module Ductwork
       end
 
       def run
-        create_process!
+        create_process_record!
+        start_job_workers
+
         Ductwork.logger.debug(
           msg: "Entering main work loop",
           role: :job_worker_runner,
@@ -34,7 +36,7 @@ module Ductwork
         while running?
           # TODO: Increase or make configurable
           sleep(5)
-          attempt_synchronize_threads
+          check_thread_health
           report_heartbeat!
         end
 
@@ -43,43 +45,29 @@ module Ductwork
 
       private
 
-      attr_reader :pipeline, :running_context, :threads
+      attr_reader :pipeline, :running_context, :job_workers
 
-      def worker_count
-        Ductwork.configuration.job_worker_count(pipeline)
+      def create_process_record!
+        Ductwork.wrap_with_app_executor do
+          Ductwork::Process.create!(
+            pid: ::Process.pid,
+            machine_identifier: Ductwork::MachineIdentifier.fetch,
+            last_heartbeat_at: Time.current
+          )
+        end
       end
 
-      def create_threads
-        worker_count.times.map do |i|
-          Ductwork.logger.debug(
-            msg: "Creating new thread",
-            role: :job_worker_runner,
-            pipeline: pipeline
-          )
-          thread = Thread.new do
-            Ductwork::Processes::JobWorker
-              .new(pipeline, running_context)
-              .run
-          end
-          thread.name = "ductwork.job_worker.#{i}"
+      def start_job_workers
+        Ductwork.configuration.job_worker_count(pipeline).times do |i|
+          job_worker = Ductwork::Processes::JobWorker.new(pipeline, i)
+          job_workers.push(job_worker)
+          job_worker.start
 
           Ductwork.logger.debug(
-            msg: "Created new thread",
+            msg: "Created new job worker",
             role: :job_worker_runner,
             pipeline: pipeline
           )
-
-          thread
-        end
-      end
-
-      def create_process!
-        Ductwork.wrap_with_app_executor do
-          Ductwork::Process.create!(
-            pid: ::Process.pid,
-            machine_identifier: Ductwork::MachineIdentifier.fetch,
-            last_heartbeat_at: Time.current
-          )
         end
       end
 
@@ -87,13 +75,17 @@ module Ductwork
         running_context.running?
       end
 
-      def attempt_synchronize_threads
+      def check_thread_health
         Ductwork.logger.debug(
           msg: "Attempting to synchronize threads",
           role: :job_worker_runner,
           pipeline: pipeline
         )
-        threads.each { |thread| thread.join(0.1) }
+        job_workers.each do |job_worker|
+          if !job_worker.alive?
+            job_worker.restart
+          end
+        end
         Ductwork.logger.debug(
           msg: "Synchronizing threads timed out",
           role: :job_worker_runner,
@@ -111,31 +103,36 @@ module Ductwork
 
       def shutdown!
         running_context.shutdown!
+        job_workers.each(&:stop)
         await_threads_graceful_shutdown
-        kill_remaining_threads
-        delete_process
+        kill_remaining_job_workers
+        delete_process_record!
       end
 
       def await_threads_graceful_shutdown
         timeout = Ductwork.configuration.job_worker_shutdown_timeout
         deadline = Time.current + timeout
 
-        Ductwork.logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
-        while Time.current < deadline && threads.any?(&:alive?)
-          threads.each do |thread|
+        Ductwork.logger.debug(
+          msg: "Attempting graceful shutdown",
+          role: :job_worker_runner
+        )
+
+        while Time.current < deadline && job_workers.any?(&:alive?)
+          job_workers.each do |job_worker|
             break if Time.current < deadline
 
             # TODO: Maybe make this configurable. If there's a ton of workers
             # it may not even get to the "later" ones depending on the timeout
-            thread.join(1)
+            job_worker.thread.join(1)
           end
         end
       end
 
-      def kill_remaining_threads
-        threads.each do |thread|
-          if thread.alive?
-            thread.kill
+      def kill_remaining_job_workers
+        job_workers.each do |job_worker|
+          if job_worker.alive?
+            job_worker.thread.kill
             Ductwork.logger.debug(
               msg: "Killed thread",
               role: :job_worker_runner,
@@ -145,7 +142,7 @@ module Ductwork
         end
       end
 
-      def delete_process
+      def delete_process_record!
         Ductwork.wrap_with_app_executor do
           Ductwork::Process.find_by!(
             pid: ::Process.pid,

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.8.1"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Tyler Ewing