ductwork 0.11.1 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b31caaa51c1b1d2003cc6315f741e714b68df32472e017bb483f69b6d532072
-  data.tar.gz: bdc8fac9c21c4acf03249a25f60e36980d6313560be33cdae45fe74f3fff1b83
+  metadata.gz: f31c4b527238da8b5025cad05246c99ad197fa2dd3302ebc5523e58ecbddaf5e
+  data.tar.gz: 111a5c0e4a4b45ec0c6bed50698e25eeaca9947601220f45378f1e2eb9a2f7e2
 SHA512:
-  metadata.gz: 581a949ba0b798e3f5e7ee78a7d28f4c58aa72d59387f98538ca6553ccdbd7e030987ee0df26ba94124446d5329210cfc02839ea2cf5ff2695be1315001eb7da
-  data.tar.gz: ee32d3c2bbf9dd6fbc430145b7af4a4feee8ad6b922936f8f1b9687135172d52a064c5588b60d3520c8fc96efe7dedb01a5767eccb9132f180f42b96def79c8e
+  metadata.gz: '095e1453247f76378f960080fa87031a307da13b01de64e78da4d07598ae0f4da997c49f7801c4fbe6101512c235d4eb1208922e09d3acbd29ec8f2817fa197f'
+  data.tar.gz: 8f6606e82a50cbc6d847d3406d30ce18db1bf586a46dd1a44f86e80866cd9aed5ce444aa994933062e7af1d6d174c6d9c489f9cd82f6e62d96ede8357bba42cf

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Ductwork Changelog
 
+## [0.12.0]
+
+- chore: replace usages of `Pipeline#halted!` with `Pipeline#halt!`
+- feat: create `Ductwork::Pipeline#halt!` method
+- chore: extract `Ductwork::Pipeline#complete!` method
+
+## [0.11.2]
+
+- fix: join error backtrace lines before persisting in `text` column
+- fix: use computed `max_retry` when determining job retry
+- fix: protect against a possible null `status` when advancing a pipeline
+- fix: correct spelling of `last_heartbeat_at` variable
+- fix: flip inverted deadline check in job worker runner
+- fix: flip inverted deadline check in pipeline advancer runner
+- fix: reference `thread` variable from `job_worker`, not top-level
+- fix: remove unnecessary, erroneous line in definition builder
+
 ## [0.11.1]
 
 - fix: use info-level for job worker restart log messages

data/lib/ductwork/dsl/definition_builder.rb

@@ -180,7 +180,6 @@ module Ductwork
           to = klasses.map { |klass| "#{klass.name}.#{stages.length}" }
           definition[:edges][last_node][:to] = to
           definition[:edges][last_node][:type] = type
-          definition[:edges][last_node][:klass] ||= klass
         end
       end
 

data/lib/ductwork/models/job.rb

@@ -163,7 +163,7 @@ module Ductwork
           result_type: "failure",
           error_klass: error.class.to_s,
           error_message: error.message,
-          error_backtrace: error.backtrace
+          error_backtrace: error.backtrace.join("\n")
         )
 
         if execution.retry_count < max_retry
@@ -174,11 +174,11 @@ module Ductwork
           new_execution.create_availability!(
             started_at: FAILED_EXECUTION_TIMEOUT.from_now
           )
-        elsif execution.retry_count >= Ductwork.configuration.job_worker_max_retry
+        elsif execution.retry_count >= max_retry
           halted = true
 
           step.update!(status: :failed)
-          pipeline.halted!
+          pipeline.halt!
         end
       end
 

data/lib/ductwork/models/pipeline.rb

@@ -114,6 +114,26 @@ module Ductwork
       @parsed_definition ||= JSON.parse(definition).with_indifferent_access
     end
 
+    def complete!
+      update!(status: :completed, completed_at: Time.current)
+
+      Ductwork.logger.info(
+        msg: "Pipeline completed",
+        pipeline_id: id,
+        role: :pipeline_advancer
+      )
+    end
+
+    def halt!
+      halted!
+
+      Ductwork.logger.info(
+        msg: "Pipeline halted",
+        pipeline_id: id,
+        pipeline_klass: klass
+      )
+    end
+
     private
 
     def create_step_and_enqueue_job(edge:, input_arg:, node: nil)
@@ -148,13 +168,7 @@ module Ductwork
                   .exists?
 
       if !remaining
-        update!(status: :completed, completed_at: Time.current)
-
-        Ductwork.logger.info(
-          msg: "Pipeline completed",
-          pipeline_id: id,
-          role: :pipeline_advancer
-        )
+        complete!
       end
     end
 
@@ -205,7 +219,7 @@ module Ductwork
       end
 
       if too_many
-        halted!
+        halt!
       else
         edge[:to].each do |node|
           input_arg = Ductwork::Job.find_by(step_id:).return_value
@@ -251,7 +265,7 @@ module Ductwork
       max_depth = Ductwork.configuration.steps_max_depth(pipeline: klass, step: next_klass)
 
       if max_depth != -1 && return_value.count > max_depth
-        halted!
+        halt!
       else
         Array(return_value).each do |input_arg|
           create_step_and_enqueue_job(edge:, input_arg:)

data/lib/ductwork/processes/job_worker.rb

@@ -3,14 +3,14 @@
 module Ductwork
   module Processes
     class JobWorker
-      attr_reader :thread, :last_hearthbeat_at, :job
+      attr_reader :thread, :last_heartbeat_at, :job
 
       def initialize(pipeline, id)
         @pipeline = pipeline
         @id = id
         @running_context = Ductwork::RunningContext.new
         @thread = nil
-        @last_hearthbeat_at = Time.current
+        @last_heartbeat_at = Time.current
       end
 
       def start
@@ -67,7 +67,7 @@ module Ductwork
             sleep(polling_timeout)
           end
 
-          @last_hearthbeat_at = Time.current
+          @last_heartbeat_at = Time.current
         end
 
         Ductwork.logger.debug(

data/lib/ductwork/processes/job_worker_runner.rb

@@ -126,7 +126,7 @@ module Ductwork
 
         while Time.current < deadline && job_workers.any?(&:alive?)
           job_workers.each do |job_worker|
-            break if Time.current < deadline
+            break if Time.current > deadline
 
             # TODO: Maybe make this configurable. If there's a ton of workers
             # it may not even get to the "later" ones depending on the timeout
@@ -142,7 +142,7 @@ module Ductwork
             Ductwork.logger.debug(
               msg: "Killed thread",
               role: :job_worker_runner,
-              thread: thread.name
+              thread: job_worker.thread.name
             )
           end
         end

data/lib/ductwork/processes/pipeline_advancer.rb

@@ -56,15 +56,14 @@ module Ductwork
                 # rubocop:todo Metrics/BlockNesting
                 status = pipeline.completed? ? "completed" : "in_progress"
                 # rubocop:enable Metrics/BlockNesting
-
+              ensure
                 # release the pipeline and set last advanced at so it doesn't
                 # block. we're not using a queue so we have to use a db
                 # timestamp
-              ensure
                 pipeline.update!(
                   claimed_for_advancing_at: nil,
                   last_advanced_at: Time.current,
-                  status: status
+                  status: status || "in_progress"
                 )
               end
             else

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -125,7 +125,7 @@ module Ductwork
         )
         while Time.current < deadline && threads.any?(&:alive?)
           threads.each do |thread|
-            break if Time.current < deadline
+            break if Time.current > deadline
 
             # TODO: Maybe make this configurable. If there's a ton of workers
             # it may not even get to the "later" ones depending on the timeout

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.11.1"
+  VERSION = "0.12.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.0
 platform: ruby
 authors:
 - Tyler Ewing