rocketjob 5.1.1 → 5.2.0.beta1

data/lib/rocket_job/batch/tabular.rb

@@ -13,8 +13,8 @@ module RocketJob
     #
     # tabular.render(row)
     class Tabular
-      autoload :Input, 'rocket_job/batch/tabular/input'
-      autoload :Output, 'rocket_job/batch/tabular/output'
+      autoload :Input, "rocket_job/batch/tabular/input"
+      autoload :Output, "rocket_job/batch/tabular/output"
 
       def initialize(map)
         @map = map

data/lib/rocket_job/batch/tabular/input.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch
@@ -53,7 +53,7 @@ module RocketJob
           input_stream = stream.nil? ? nil : IOStreams.new(stream)
 
           if stream && (tabular_input_type == :text)
-            input_stream.option_or_stream(:encode, encoding: 'UTF-8', cleaner: :printable, replace: '')
+            input_stream.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: "")
           end
 
           # If an input header is not required, then we don't extract it'
@@ -67,14 +67,14 @@ module RocketJob
 
           case tabular_input_mode
           when :line
-            parse_header = -> (line) do
+            parse_header = lambda do |line|
               tabular_input.parse_header(line)
               tabular_input_cleanse_header
               self.tabular_input_header = tabular_input.header.columns
             end
             super(input_stream, on_first: parse_header, stream_mode: :line, **args, &block)
           when :array, :row
-            set_header = -> (row) do
+            set_header = lambda do |row|
               tabular_input.header.columns = row
               tabular_input_cleanse_header
               self.tabular_input_header = tabular_input.header.columns
@@ -101,19 +101,23 @@ module RocketJob
         end
 
         def tabular_input_render
-          @rocket_job_input = tabular_input.record_parse(@rocket_job_input) unless tabular_input_header.blank? && tabular_input.header?
+          unless tabular_input_header.blank? && tabular_input.header?
+            @rocket_job_input = tabular_input.record_parse(@rocket_job_input)
+          end
         end
 
         # Cleanse custom input header if supplied.
         def tabular_input_cleanse_header
           ignored_columns = tabular_input.header.cleanse!
-          logger.warn('Stripped out invalid columns from custom header', ignored_columns) unless ignored_columns.empty?
+          logger.warn("Stripped out invalid columns from custom header", ignored_columns) unless ignored_columns.empty?
 
           self.tabular_input_header = tabular_input.header.columns
         end
 
         def tabular_input_header_present
-          return if tabular_input_header.present? || !tabular_input.header? || (tabular_input_mode == :hash || tabular_input_mode == :record)
+          if tabular_input_header.present? || !tabular_input.header? || (tabular_input_mode == :hash || tabular_input_mode == :record)
+            return
+          end
 
           errors.add(:tabular_input_header, "is required when tabular_input_format is #{tabular_input_format.inspect}")
         end

data/lib/rocket_job/batch/tabular/output.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch

data/lib/rocket_job/batch/throttle.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch
@@ -31,7 +31,6 @@ module RocketJob
 
       included do
         class_attribute :rocket_job_batch_throttles
-        self.rocket_job_batch_throttles = []
       end
 
       module ClassMethods
@@ -48,44 +47,26 @@ module RocketJob
         #
         # Note: Throttles are executed in the order they are defined.
         def define_batch_throttle(method_name, filter: :throttle_filter_class)
-          unless filter.is_a?(Symbol) || filter.is_a?(Proc)
-            raise(ArgumentError, "Filter for #{method_name} must be a Symbol or Proc")
-          end
-          if batch_throttle?(method_name)
-            raise(ArgumentError, "Cannot define #{method_name} twice, undefine previous throttle first")
-          end
-
-          self.rocket_job_batch_throttles += [ThrottleDefinition.new(method_name, filter)]
+          # Duplicate to prevent modifying parent class throttles
+          definitions = rocket_job_batch_throttles ? rocket_job_batch_throttles.dup : ThrottleDefinitions.new
+          definitions.add(method_name, filter)
+          self.rocket_job_batch_throttles = definitions
         end
 
         # Undefine a previously defined throttle
         def undefine_batch_throttle(method_name)
-          rocket_job_batch_throttles.delete_if { |throttle| throttle.method_name == method_name }
+          return unless rocket_job_batch_throttles
+
+          definitions = rocket_job_batch_throttles.dup
+          definitions.remove(method_name)
+          self.rocket_job_batch_throttles = definitions
         end
 
         # Has a throttle been defined?
         def batch_throttle?(method_name)
-          rocket_job_batch_throttles.any? { |throttle| throttle.method_name == method_name }
+          rocket_job_batch_throttles.exist?(method_name)
         end
       end
-
-      private
-
-      ThrottleDefinition = Struct.new(:method_name, :filter)
-
-      # Returns the matching filter, or nil if no throttles were triggered.
-      def rocket_job_batch_evaluate_throttles(slice)
-        rocket_job_batch_throttles.each do |throttle|
-          throttle_exceeded = method(throttle.method_name).arity == 0 ? send(throttle.method_name) : send(throttle.method_name, slice)
-          next unless throttle_exceeded
-
-          logger.debug { "Batch Throttle: #{throttle.method_name} has been exceeded. #{self.class.name}:#{id}" }
-          filter = throttle.filter
-          return filter.is_a?(Proc) ? filter.call(self) : send(filter)
-        end
-        nil
-      end
-
     end
   end
 end

data/lib/rocket_job/batch/{throttle_running_slices.rb → throttle_running_workers.rb}

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch
@@ -9,7 +9,7 @@ module RocketJob
     #     include RocketJob::Batch
     #
     #     # Maximum number of slices to process at the same time for each running instance.
-    #     self.throttle_running_slices = 25
+    #     self.throttle_running_workers = 25
     #
     #     def perform(record)
     #       # ....
@@ -28,28 +28,31 @@ module RocketJob
     # 0 or nil : No limits in place
     #
     # Default: nil
-    module ThrottleRunningSlices
+    module ThrottleRunningWorkers
       extend ActiveSupport::Concern
 
       included do
-        field :throttle_running_slices, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
+        field :throttle_running_workers, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
 
-        validates :throttle_running_slices, numericality: {greater_than_or_equal_to: 0}, allow_nil: true
+        validates :throttle_running_workers, numericality: {greater_than_or_equal_to: 0}, allow_nil: true
 
-        define_batch_throttle :throttle_running_slices_exceeded?, filter: :throttle_filter_id
+        define_batch_throttle :throttle_running_workers_exceeded?, filter: :throttle_filter_id
+
+        # Deprecated. For backward compatibility.
+        alias_method :throttle_running_slices, :throttle_running_workers
+        alias_method :throttle_running_slices=, :throttle_running_workers=
       end
 
       private
 
       # Returns [Boolean] whether the throttle for this job has been exceeded
-      def throttle_running_slices_exceeded?(slice)
-        return unless throttle_running_slices&.positive?
+      def throttle_running_workers_exceeded?(slice)
+        return unless throttle_running_workers&.positive?
 
         input.running.with(read: {mode: :primary}) do |conn|
-          conn.where(:id.ne => slice.id).count >= throttle_running_slices
+          conn.where(:id.ne => slice.id).count >= throttle_running_workers
         end
       end
-
     end
   end
 end

data/lib/rocket_job/batch/worker.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch
@@ -16,10 +16,10 @@ module RocketJob
 
       # Processes records in each available slice for this job. Slices are processed
       # one at a time to allow for concurrent calls to this method to increase
-      # throughput. Processing will continue until there are no more jobs available
+      # throughput. Processing will continue until there are no more slices available
       # for this job.
       #
-      # Returns [true|false] whether this job should be excluded from the next lookup
+      # Returns [true|false] whether any work was performed.
       #
       # Slices are destroyed after their records are successfully processed
       #
@@ -33,36 +33,34 @@ module RocketJob
       # Mongo connection failure occurs.
       #
       # Thread-safe, can be called by multiple threads at the same time
-      def rocket_job_work(worker, re_raise_exceptions = false, filter = {})
-        raise 'Job must be started before calling #rocket_job_work' unless running?
+      def rocket_job_work(worker, re_raise_exceptions = false)
+        raise "Job must be started before calling #rocket_job_work" unless running?
+
         start_time = Time.now
         if sub_state != :processing
-          rocket_job_handle_callbacks(worker, re_raise_exceptions)
+          fail_on_exception!(re_raise_exceptions) { rocket_job_batch_callbacks(worker) }
           return false unless running?
         end
 
-        while !worker.shutdown?
-          if slice = input.next_slice(worker.name)
-            # Grab a slice before checking the throttle to reduce concurrency race condition.
-            if new_filter = rocket_job_batch_evaluate_throttles(slice)
-              # Restore retrieved slice so that other workers can process it later.
-              slice.set(worker_name: nil, state: :queued, started_at: nil)
-              self.class.send(:rocket_job_merge_filter, filter, new_filter)
+        SemanticLogger.named_tagged(job: id.to_s) do
+          until worker.shutdown?
+            if slice = input.next_slice(worker.name)
+              # Grab a slice before checking the throttle to reduce concurrency race condition.
+              return true if slice.fail_on_exception!(re_raise_exceptions) { rocket_job_batch_throttled?(slice, worker) }
+              next if slice.failed?
+
+              slice.fail_on_exception!(re_raise_exceptions) { rocket_job_process_slice(slice) }
+            elsif record_count && rocket_job_batch_complete?(worker.name)
+              return false
+            else
+              logger.debug "No more work available for this job"
+              worker.add_to_current_filter(throttle_filter_id)
               return true
             end
 
-            SemanticLogger.named_tagged(slice: slice.id.to_s) do
-              rocket_job_process_slice(slice, re_raise_exceptions)
-            end
-          else
-            break if record_count && rocket_job_batch_complete?(worker.name)
-            logger.debug 'No more work available for this job'
-            self.class.send(:rocket_job_merge_filter, filter, throttle_filter_id)
-            return true
+            # Allow new jobs with a higher priority to interrupt this job
+            break if (Time.now - start_time) >= Config.re_check_seconds
           end
-
-          # Allow new jobs with a higher priority to interrupt this job
-          break if (Time.now - start_time) >= Config.re_check_seconds
         end
         false
       end
@@ -76,27 +74,26 @@ module RocketJob
       #
       # Note: The slice will be removed from processing when this method completes
       def work_first_slice(&block)
-        raise '#work_first_slice can only be called from within before_batch callbacks' unless sub_state == :before
-        # TODO Make these settings configurable
+        raise "#work_first_slice can only be called from within before_batch callbacks" unless sub_state == :before
+
+        # TODO: Make these settings configurable
         count        = 0
         wait_seconds = 5
-        while (slice = input.first).nil?
+        while input.first.nil?
           break if count > 10
+
           logger.info "First slice has not arrived yet, sleeping for #{wait_seconds} seconds"
           sleep wait_seconds
           count += 1
         end
 
-        if slice = input.first
-          SemanticLogger.named_tagged(slice: slice.id.to_s) do
-            # TODO Persist that the first slice is being processed by this worker
-            slice.start
-            rocket_job_process_slice(slice, true, &block)
-          end
-        else
-          # No records processed
-          0
-        end
+        slice = input.first
+        # No records processed
+        return 0 unless slice
+
+        # TODO: Persist that the first slice is being processed by this worker
+        slice.start
+        rocket_job_process_slice(slice, &block)
       end
 
       # Returns [Array<ActiveWorker>] All workers actively working on this job
@@ -119,50 +116,73 @@ module RocketJob
 
       private
 
+      def rocket_job_batch_throttled?(slice, worker)
+        filter = self.class.rocket_job_batch_throttles.matching_filter(self, slice)
+        return false unless filter
+
+        # Restore retrieved slice so that other workers can process it later.
+        slice.set(worker_name: nil, state: :queued, started_at: nil)
+        worker.add_to_current_filter(filter)
+        true
+      end
+
       # Process a single slice from Mongo
       # Once the slice has been successfully processed it will be removed from the input collection
       # Returns [Integer] the number of records successfully processed
-      def rocket_job_process_slice(slice, re_raise_exceptions)
-        slice_record_number       = 0
+      def rocket_job_process_slice(slice)
+        # TODO: Skip records already processed
         @rocket_job_record_number = slice.first_record_number || 0
         @rocket_job_slice         = slice
-        run_callbacks :slice do
+
+        processed_records = 0
+        run_callbacks(:slice) do
+          # Allow before_slice callbacks to fail, complete or abort this slice.
+          return 0 unless running?
+
           RocketJob::Sliced::Writer::Output.collect(self, slice) do |writer|
             slice.each do |record|
-              slice_record_number += 1
               SemanticLogger.named_tagged(record: @rocket_job_record_number) do
-                if _perform_callbacks.empty?
-                  @rocket_job_output = block_given? ? yield(record) : perform(record)
-                else
-                  # Allows @rocket_job_input to be modified by before/around callbacks
-                  @rocket_job_input = record
-                  # Allow callbacks to fail, complete or abort the job
-                  if running?
-                    if block_given?
-                      run_callbacks(:perform) { @rocket_job_output = yield(@rocket_job_input) }
-                    else
-                      # Allows @rocket_job_output to be modified by after/around callbacks
-                      run_callbacks(:perform) { @rocket_job_output = perform(@rocket_job_input) }
-                    end
-                  end
-                end
-                writer << @rocket_job_output
+                writer << rocket_job_batch_perform(slice, record)
+                processed_records += 1
               end
-              # JRuby says self.rocket_job_record_number= is private and cannot be accessed
+              # JRuby thinks self.rocket_job_record_number= is private and cannot be accessed
               @rocket_job_record_number += 1
             end
           end
-          @rocket_job_input = @rocket_job_slice = @rocket_job_output = nil
+          @rocket_job_slice         = nil
+          @rocket_job_record_number = nil
         end
 
         # On successful completion remove the slice from the input queue
-        # TODO Option to complete slice instead of destroying it to retain input data
+        # TODO: Add option to complete slice instead of destroying it to retain input data.
         slice.destroy
-        slice_record_number
-      rescue Exception => exc
-        slice.fail!(exc, slice_record_number)
-        raise exc if re_raise_exceptions
-        slice_record_number > 0 ? slice_record_number - 1 : 0
+        processed_records
+      end
+
+      # Perform a single record within the current slice.
+      def rocket_job_batch_perform(slice, record)
+        slice.processing_record_number ||= 0
+        slice.processing_record_number += 1
+
+        return block_given? ? yield(record) : perform(record) if _perform_callbacks.empty?
+
+        # @rocket_job_input and @rocket_job_output can be modified by before/around callbacks
+        @rocket_job_input  = record
+        @rocket_job_output = nil
+
+        run_callbacks(:perform) do
+          @rocket_job_output =
+            if block_given?
+              yield(@rocket_job_input)
+            else
+              perform(@rocket_job_input)
+            end
+        end
+
+        @rocket_job_input  = nil
+        result             = @rocket_job_output
+        @rocket_job_output = nil
+        result
       end
 
       # Checks for completion and runs after_batch if defined
@@ -174,7 +194,7 @@ module RocketJob
         # Only failed slices left?
         input_count  = input.count
         failed_count = input.failed.count
-        if (failed_count > 0) && (input_count == failed_count)
+        if failed_count.positive? && (input_count == failed_count)
           # Reload to pull in any counters or other data that was modified.
           reload unless new_record?
           if may_fail?
@@ -185,9 +205,9 @@ module RocketJob
               result   = self.class.with(write: {w: 1}) do |query|
                 query.
                   where(id: id, state: :running, sub_state: :processing).
-                  update({'$set' => {state: :failed, worker_name: worker_name}})
+                  update({"$set" => {state: :failed, worker_name: worker_name}})
               end
-              fail_job = false unless result.modified_count > 0
+              fail_job = false unless result.modified_count.positive?
             end
             if fail_job
               message        = "#{failed_count} slices failed to process"
@@ -199,7 +219,7 @@ module RocketJob
         end
 
         # Any work left?
-        return false if input_count > 0
+        return false if input_count.positive?
 
         # If the job was not saved to the queue, do not save any changes
         if new_record?
@@ -212,12 +232,12 @@ module RocketJob
         result = self.class.with(write: {w: 1}) do |query|
           query.
             where(id: id, state: :running, sub_state: :processing).
-            update('$set' => {sub_state: :after, worker_name: worker_name})
+            update("$set" => {sub_state: :after, worker_name: worker_name})
         end
 
         # Reload to pull in any counters or other data that was modified.
         reload
-        if result.modified_count > 0
+        if result.modified_count.positive?
           rocket_job_batch_run_after_callbacks(false)
         else
           # Repeat cleanup in case this worker was still running when the job was aborted
@@ -233,7 +253,7 @@ module RocketJob
           self.sub_state = :before
           save! unless new_record? || destroyed?
           logger.measure_info(
-            'before_batch',
+            "before_batch",
             metric:             "#{self.class.name}/before_batch",
             log_exception:      :full,
             on_exception_level: :error,
@@ -253,7 +273,7 @@ module RocketJob
           self.sub_state = :after
           save! if save_before && !new_record? && !destroyed?
           logger.measure_info(
-            'after_batch',
+            "after_batch",
             metric:             "#{self.class.name}/after_batch",
             log_exception:      :full,
             on_exception_level: :error,
@@ -269,20 +289,17 @@ module RocketJob
         end
       end
 
-      # Handle before and after callbacks
-      def rocket_job_handle_callbacks(worker, re_raise_exceptions)
-        rocket_job_fail_on_exception!(worker.name, re_raise_exceptions) do
-          # If this is the first worker to pickup this job
-          if sub_state == :before
-            rocket_job_batch_run_before_callbacks
-            # Check for 0 record jobs
-            rocket_job_batch_complete?(worker.name) if running?
-          elsif sub_state == :after
-            rocket_job_batch_run_after_callbacks
-          end
+      # Run Batch before and after callbacks
+      def rocket_job_batch_callbacks(worker)
+        # If this is the first worker to pickup this job
+        if sub_state == :before
+          rocket_job_batch_run_before_callbacks
+          # Check for 0 record jobs
+          rocket_job_batch_complete?(worker.name) if running?
+        elsif sub_state == :after
+          rocket_job_batch_run_after_callbacks
         end
       end
-
     end
   end
 end