rocketjob 5.2.0.beta2 → 5.2.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be987bf17c7505ce19a06f0f15bd612b0d9588a7aed842250b4f094b98b026ef
-  data.tar.gz: afb3b94b5f47c4fd462da9732f6b94d87b53f96e9afe22f66592c89eeed3c521
+  metadata.gz: 0cece68774d1d549fb219daa5f78f3def0e5b3c51d0659fea8812da200b6b268
+  data.tar.gz: 69ca23691dcbe2c8ba1b7b5a60a4d18cb8b033cf37510c2a0f1f0a02075357a7
 SHA512:
-  metadata.gz: 0f71a2a1aa604c1db1babf43ea5865724617515b7f0ea299e438d8fb4bb845d2a4ef21c4bbd5994034cb070c72c6b6f53dfbf45d2b52e576c8a2cb3202e1549a
-  data.tar.gz: 969bcffcb4aa38277b99f88188334e28393ed6fa92e5e470214879ec2d169e81a586947daaf3bcaea446ac8766e49e9c7dd328a4131bde0275808eff241a9437
+  metadata.gz: ff6d1c42eca584b91dbf27312b962ed4615168616a77531d70ca9a252145aac47f1c0a08655433aa956b143bab6a517e0b973b63474ba25cb718eb018df3950d
+  data.tar.gz: 9add07cf69e3bf6cfc0d1aa62902ce00deb07ba41f922491afbe531e00b364fddee70f99f86497b7ec45e97637e715d355e55a6c30dbebcde2a8811b907e4d03

data/lib/rocket_job/batch/throttle_running_workers.rb

@@ -63,7 +63,7 @@ module RocketJob
 
         # Cannot use this class since it will include instances of parent job classes.
         RocketJob::Job.with(read: {mode: :primary}) do |conn|
-          conn.running.where("_type" => self.class.name, :id.ne => id, :priority.lt => priority).count >= throttle_running_jobs
+          conn.running.where("_type" => self.class.name, :id.ne => id, :priority.lte => priority).count >= throttle_running_jobs
         end
       end
     end

data/lib/rocket_job/batch/worker.rb

@@ -130,45 +130,56 @@ module RocketJob
       # Once the slice has been successfully processed it will be removed from the input collection
       # Returns [Integer] the number of records successfully processed
       def rocket_job_process_slice(slice, &block)
-        # TODO: Skip records already processed
-        @rocket_job_record_number = slice.first_record_number || 0
-        @rocket_job_slice         = slice
+        @rocket_job_slice = slice
+        count             = 0
 
-        processed_records = 0
         run_callbacks(:slice) do
           # Allow before_slice callbacks to fail, complete or abort this slice.
           return 0 unless running?
 
-          RocketJob::Sliced::Writer::Output.collect(self, slice) do |writer|
-            slice.each do |record|
-              SemanticLogger.named_tagged(record: @rocket_job_record_number) do
-                writer << rocket_job_batch_perform(slice, record, &block)
-                processed_records += 1
-              end
-              # JRuby thinks self.rocket_job_record_number= is private and cannot be accessed
-              @rocket_job_record_number += 1
-            end
-          end
-          @rocket_job_slice         = nil
-          @rocket_job_record_number = nil
+          count = rocket_job_perform_slice(slice, &block)
         end
+        @rocket_job_slice = nil
 
         # On successful completion remove the slice from the input queue
         # TODO: Add option to complete slice instead of destroying it to retain input data.
         slice.destroy
-        processed_records
+        count
+      end
+
+      # Perform individual slice without callbacks
+      def rocket_job_perform_slice(slice, &block)
+        count = 0
+        RocketJob::Sliced::Writer::Output.collect(self, slice) do |writer|
+          records = slice.records
+          slice.processing_record_number ||= 0
+
+          # Skip records already processed, if any.
+          # TODO: Must append to existing output slices before this can be enabled.
+          # if !collect_output && (slice.processing_record_number > 1)
+          #   records = records[slice.processing_record_number - 1..-1]
+          # end
+
+          records.each do |record|
+            slice.processing_record_number += 1
+            SemanticLogger.named_tagged(record: slice.current_record_number) do
+              writer << rocket_job_batch_perform(slice, record, &block)
+              count += 1
+            end
+          end
+        end
+        count
       end
 
       # Perform a single record within the current slice.
       def rocket_job_batch_perform(slice, record)
-        slice.processing_record_number ||= 0
-        slice.processing_record_number += 1
+        @rocket_job_record_number = slice.current_record_number
 
         return block_given? ? yield(record) : perform(record) if _perform_callbacks.empty?
 
         # @rocket_job_input and @rocket_job_output can be modified by before/around callbacks
-        @rocket_job_input  = record
-        @rocket_job_output = nil
+        @rocket_job_input         = record
+        @rocket_job_output        = nil
 
         run_callbacks(:perform) do
           @rocket_job_output =
@@ -179,9 +190,9 @@ module RocketJob
             end
         end
 
-        @rocket_job_input  = nil
-        result             = @rocket_job_output
-        @rocket_job_output = nil
+        @rocket_job_input         = nil
+        result                    = @rocket_job_output
+        @rocket_job_output        = nil
         result
       end
 
@@ -197,24 +208,7 @@ module RocketJob
         if failed_count.positive? && (input_count == failed_count)
           # Reload to pull in any counters or other data that was modified.
           reload unless new_record?
-          if may_fail?
-            fail_job = true
-            unless new_record?
-              # Fail job iff no other worker has already finished it
-              # Must set write concern to at least 1 since we need the nModified back
-              result = self.class.with(write: {w: 1}) do |query|
-                query.
-                  where(id: id, state: :running, sub_state: :processing).
-                  update({"$set" => {state: :failed, worker_name: worker_name}})
-              end
-              fail_job = false unless result.modified_count.positive?
-            end
-            if fail_job
-              message        = "#{failed_count} slices failed to process"
-              self.exception = JobException.new(message: message)
-              fail!(worker_name, message)
-            end
-          end
+          rocket_job_batch_fail!(worker_name) if may_fail?
           return true
         end
 
@@ -237,15 +231,38 @@ module RocketJob
 
         # Reload to pull in any counters or other data that was modified.
         reload
+
         if result.modified_count.positive?
           rocket_job_batch_run_after_callbacks(false)
-        else
+        elsif aborted?
           # Repeat cleanup in case this worker was still running when the job was aborted
-          cleanup! if aborted?
+          cleanup!
         end
         true
       end
 
+      # Fail the job
+      def rocket_job_batch_fail!(worker_name)
+        fail_job = true
+
+        unless new_record?
+          # Fail job iff no other worker has already finished it
+          # Must set write concern to at least 1 since we need the nModified back
+          result = self.class.with(write: {w: 1}) do |query|
+            query.
+              where(id: id, state: :running, sub_state: :processing).
+              update({"$set" => {state: :failed, worker_name: worker_name}})
+          end
+          fail_job = false unless result.modified_count.positive?
+        end
+
+        return unless fail_job
+
+        message        = "#{input.failed.count} slices failed to process"
+        self.exception = JobException.new(message: message)
+        new_record? ? fail(worker_name, message) : fail!(worker_name, message)
+      end
+
       # Run the before_batch callbacks
       # Saves the current state before and after running callbacks if callbacks present
       def rocket_job_batch_run_before_callbacks

data/lib/rocket_job/sliced/slice.rb

@@ -25,8 +25,8 @@ module RocketJob
 
       # The record number of the first record in this slice.
       # Useful in knowing the line number of each record in this slice
-      # relative to the original file that was uploaded.
-      field :first_record_number, type: Integer
+      # relative to the original file that was uploaded. (1-based index)
+      field :first_record_number, type: Integer, default: 1
 
       #
       # Read-only attributes
@@ -41,7 +41,7 @@ module RocketJob
       # Number of times that this job has failed to process
       field :failure_count, type: Integer
 
-      # Number of the record within this slice (not the entire file/job) currently being processed. (One based index)
+      # Number of the record within this slice (not the entire file/job) currently being processed. (1-based index)
       field :processing_record_number, type: Integer
 
       # This name of the worker that this job is being processed by, or was processed by
@@ -112,7 +112,7 @@ module RocketJob
 
       # Returns [Integer] the record number of the record currently being processed relative to the entire file.
       def current_record_number
-        first_record_number.to_i + processing_record_number.to_i
+        first_record_number + (processing_record_number || 1) - 1
       end
 
       # Before Fail save the exception to this slice.

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "5.2.0.beta2".freeze
+  VERSION = "5.2.0.beta3".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 5.2.0.beta2
+  version: 5.2.0.beta3
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-06 00:00:00.000000000 Z
+date: 2020-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm