rocketjob 5.3.3 → 6.0.0.rc1

data/lib/rocket_job/batch/model.rb

@@ -11,46 +11,6 @@ module RocketJob
         #
         # The following attributes are set when the job is created
 
-        # Number of records to include in each slice that is processed
-        # Note:
-        #   slice_size is only used by SlicedJob#upload & Sliced::Input#upload
-        #   When slices are supplied directly, their size is not modified to match this number
-        field :slice_size, type: Integer, default: 100, class_attribute: true, user_editable: true, copy_on_restart: true
-
-        # Whether to retain nil results.
-        #
-        # Only applicable if `collect_output` is `true`
-        # Set to `false` to prevent collecting output from the perform
-        # method when it returns `nil`.
-        field :collect_nil_output, type: Boolean, default: true, class_attribute: true
-
-        # Optional Array<Symbol> list of categories that this job can output to
-        #
-        # By using categories the output from #perform can be placed in different
-        # output collections, and therefore different output files
-        #
-        # Categories must be declared in advance to avoid a #perform method
-        # accidentally writing its results to an unknown category
-        field :output_categories, type: Array, default: [:main], class_attribute: true
-
-        # Optional Array<Symbol> list of categories that this job can load input data into
-        field :input_categories, type: Array, default: [:main], class_attribute: true
-
-        # The file name of the uploaded file, if any.
-        # Set by #upload if a file name was supplied, but can also be set explicitly.
-        # May or may not include the fully qualified path name.
-        field :upload_file_name, type: String
-
-        # Compress uploaded records.
-        # The fields are not affected in any way, only the data stored in the
-        # records and results collections will compressed
-        field :compress, type: Boolean, default: false, class_attribute: true
-
-        # Encrypt uploaded records.
-        # The fields are not affected in any way, only the data stored in the
-        # records and results collections will be encrypted
-        field :encrypt, type: Boolean, default: false, class_attribute: true
-
         #
         # Values that jobs can also update during processing
         #
@@ -69,30 +29,7 @@ module RocketJob
 
         # Breaks the :running state up into multiple sub-states:
         #   :running -> :before -> :processing -> :after -> :complete
-        field :sub_state, type: Symbol
-
-        validates_presence_of :slice_size
-
-        validates_each :output_categories, :input_categories do |record, attr, value|
-          # Under some circumstances ActiveModel is passing in a nil value even though the
-          # attributes have default values
-          Array(value).each do |category|
-            record.errors.add(attr, "must only contain Symbol values") unless category.is_a?(Symbol)
-            unless category.to_s =~ /\A[a-z_0-9]+\Z/
-              record.errors.add(attr, "must only consist of lowercase characters, digits, and _")
-            end
-          end
-        end
-      end
-
-      # Returns [true|false] whether the slices for this job are encrypted
-      def encrypted?
-        encrypt == true
-      end
-
-      # Returns [true|false] whether the slices for this job are compressed
-      def compressed?
-        compress == true
+        field :sub_state, type: Mongoid::StringifiedSymbol
       end
 
       # Returns [Integer] percent of records completed so far
@@ -102,10 +39,10 @@ module RocketJob
         return 0 unless record_count.to_i.positive?
 
         # Approximate number of input records
-        input_records = input.count.to_f * slice_size
+        input_records = input.count.to_f * input_category.slice_size
         if input_records > record_count
           # Sanity check in case slice_size is not being adhered to
-          99
+          0
         else
           ((1.0 - (input_records.to_f / record_count)) * 100).to_i
         end
@@ -120,6 +57,10 @@ module RocketJob
           h["active_slices"] = worker_count
           h["failed_slices"] = input.failed.count
           h["queued_slices"] = input.queued.count
+          output_categories.each do |category|
+            name_str                      = category.name == :main ? "" : "_#{category.name}"
+            h["output_slices#{name_str}"] = output(category).count
+          end
           # Very high level estimated time left
           if record_count && running? && record_count.positive?
             percent = percent_complete
@@ -129,10 +70,9 @@ module RocketJob
             end
           end
         elsif completed?
-          secs = seconds.to_f
+          secs                  = seconds.to_f
           h["records_per_hour"] = ((record_count.to_f / secs) * 60 * 60).round if record_count&.positive? && (secs > 0.0)
         end
-        h["output_slices"] = output.count if collect_output? && !completed?
         h.merge!(super(time_zone))
         h.delete("result")
         # Worker name should be retrieved from the slices when processing
@@ -172,6 +112,18 @@ module RocketJob
         @worker_count_last = Time.now.to_i
         @worker_count
       end
+
+      # @deprecated
+      # For backward compatibility
+      def upload_file_name
+        input_category.file_name
+      end
+
+      # @deprecated
+      # For backward compatibility
+      def upload_file_name=(upload_file_name)
+        input_category.file_name = upload_file_name
+      end
     end
   end
 end

data/lib/rocket_job/batch/performance.rb

@@ -22,12 +22,15 @@ module RocketJob
         count_running_workers
 
         puts "Loading job with #{count} records/lines"
-        args = {log_level: :warn, slice_size: slice_size}
-        if defined?(::RocketJob)
-          args[:compress] = compress
-          args[:encrypt]  = encrypt
+        job = RocketJob::Jobs::PerformanceJob.new(log_level: :warn)
+        job.input_category.slice_size = slice_size
+        if encrypt
+          job.input_category.serializer  = :encrypt
+          job.output_category.serializer = :encrypt
+        elsif !compress
+          job.input_category.serializer  = :none
+          job.output_category.serializer = :none
         end
-        job = RocketJob::Jobs::PerformanceJob.new(args)
         job.upload do |writer|
           count.times { |i| writer << i }
         end
@@ -37,7 +40,15 @@ module RocketJob
         sleep 3 until job.reload.completed?
 
         duration = job.completed_at - job.started_at
-        {count: count, duration: duration, records_per_second: (count.to_f / duration).round(3), workers: workers, servers: servers, compress: compress, encrypt: encrypt}
+        {
+          count:              count,
+          duration:           duration,
+          records_per_second: (count.to_f / duration).round(3),
+          workers:            workers,
+          servers:            servers,
+          compress:           compress,
+          encrypt:            encrypt
+        }
       end
 
       # Export the Results hash to a CSV file
@@ -53,14 +64,15 @@ module RocketJob
 
       # Parse command line options
       def parse(argv)
-        parser = OptionParser.new do |o|
+        parser        = OptionParser.new do |o|
           o.on("-c", "--count COUNT", "Count of records to enqueue") do |arg|
             self.count = arg.to_i
           end
           o.on("-m", "--mongo MONGO_CONFIG_FILE_NAME", "Location of mongoid.yml config file") do |arg|
             self.mongo_config = arg
           end
-          o.on("-e", "--environment ENVIRONMENT", "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
+          o.on("-e", "--environment ENVIRONMENT",
+               "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
             self.environment = arg
           end
           o.on("-z", "--compress", "Turn on compression") do

data/lib/rocket_job/batch/statistics.rb

@@ -2,7 +2,11 @@ require "active_support/concern"
 
 module RocketJob
   module Batch
-    # Allow statistics to be gathered while a batch job is running
+    # Allow statistics to be gathered while a batch job is running.
+    #
+    # Notes:
+    # - Statistics for successfully processed records within a slice are saved.
+    # - Statistics gathered during a perform that then results in an exception are discarded.
     module Statistics
       extend ActiveSupport::Concern
 
@@ -45,39 +49,57 @@ module RocketJob
           last  = paths.pop
           return unless last
 
-          target = paths.inject(in_memory) { |target, key| target.key?(key) ? target[key] : target[key] = Hash.new(0) }
-          target[last] += increment
+          last_target       = paths.inject(in_memory) do |target, sub_key|
+            target.key?(sub_key) ? target[sub_key] : target[sub_key] = Hash.new(0)
+          end
+          last_target[last] += increment
         end
       end
 
       included do
         field :statistics, type: Hash, default: -> { Hash.new(0) }
 
-        around_slice :statistics_capture
+        around_slice :rocket_job_statistics_capture
+        after_perform :rocket_job_statistics_commit
       end
 
       # Increment a statistic
       def statistics_inc(key, increment = 1)
         return if key.nil? || key == ""
 
-        # Being called within tests outside of a perform
-        @slice_statistics ||= Stats.new(new_record? ? statistics : nil)
-        key.is_a?(Hash) ? @slice_statistics.inc(key) : @slice_statistics.inc_key(key, increment)
+        (@rocket_job_perform_statistics ||= []) << (key.is_a?(Hash) ? key : [key, increment])
       end
 
       private
 
-      # Capture the number of successful and failed tradelines
-      # as well as those with notices and alerts.
-      def statistics_capture
-        @slice_statistics = Stats.new(new_record? ? statistics : nil)
+      def rocket_job_statistics_capture
+        @rocket_job_perform_statistics = nil
+        @rocket_job_slice_statistics   = nil
         yield
-        collection.update_one({_id: id}, {"$inc" => @slice_statistics.stats}) unless @slice_statistics.empty?
+      ensure
+        if @rocket_job_slice_statistics && !@rocket_job_slice_statistics.empty?
+          collection.update_one({_id: id}, {"$inc" => @rocket_job_slice_statistics.stats})
+        end
+      end
+
+      def rocket_job_slice_statistics
+        @rocket_job_slice_statistics ||= Stats.new(new_record? ? statistics : nil)
+      end
+
+      # Apply stats gathered during the perform to the slice level stats
+      def rocket_job_statistics_commit
+        return unless @rocket_job_perform_statistics
+
+        @rocket_job_perform_statistics.each do |key|
+          key.is_a?(Hash) ? rocket_job_slice_statistics.inc(key) : rocket_job_slice_statistics.inc_key(*key)
+        end
+
+        @rocket_job_perform_statistics = nil
       end
 
       # Overrides RocketJob::Batch::Logger#rocket_job_batch_log_payload
       def rocket_job_batch_log_payload
-        h = {
+        h              = {
           from:  aasm.from_state,
           to:    aasm.to_state,
           event: aasm.current_event

data/lib/rocket_job/batch/tabular.rb

@@ -12,6 +12,8 @@ module RocketJob
     # )
     #
     # tabular.render(row)
+    #
+    # @deprecated
     class Tabular
       autoload :Input, "rocket_job/batch/tabular/input"
       autoload :Output, "rocket_job/batch/tabular/output"

data/lib/rocket_job/batch/tabular/input.rb

@@ -3,15 +3,15 @@ require "active_support/concern"
 module RocketJob
   module Batch
     class Tabular
-      # For the simple case where all `input_categories` have the same format,
-      # If multiple input categories are used with different formats, then use IOStreams::Tabular directly
-      # instead of this plugin.
+      # @deprecated
       module Input
         extend ActiveSupport::Concern
 
         included do
+          warn "#{name} is using RocketJob::Batch::Tabular::Input which is deprecated"
+
           field :tabular_input_header, type: Array, class_attribute: true, user_editable: true
-          field :tabular_input_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true
+          field :tabular_input_format, type: Mongoid::StringifiedSymbol, default: :csv, class_attribute: true, user_editable: true
           field :tabular_input_options, type: Hash, class_attribute: true
 
           # tabular_input_mode: [:line | :array | :hash]
@@ -22,7 +22,7 @@ module RocketJob
           #   :hash
           #     Parses each line from the file into a Hash and uploads each hash for processing by workers.
           #   See IOStreams#each.
-          field :tabular_input_mode, type: Symbol, default: :line, class_attribute: true, user_editable: true, copy_on_restart: true
+          field :tabular_input_mode, type: Mongoid::StringifiedSymbol, default: :line, class_attribute: true, user_editable: true, copy_on_restart: true
 
           validates_inclusion_of :tabular_input_format, in: IOStreams::Tabular.registered_formats
           validates_inclusion_of :tabular_input_mode, in: %i[line array hash row record]
@@ -119,7 +119,9 @@ module RocketJob
         end
 
         def tabular_input_header_present
-          if tabular_input_header.present? || !tabular_input.header? || (tabular_input_mode == :hash || tabular_input_mode == :record)
+          if tabular_input_header.present? ||
+             !tabular_input.header? ||
+             (tabular_input_mode == :hash || tabular_input_mode == :record)
             return
           end
 

data/lib/rocket_job/batch/tabular/output.rb

@@ -10,8 +10,10 @@ module RocketJob
         extend ActiveSupport::Concern
 
         included do
+          warn "#{name} is using RocketJob::Batch::Tabular::Output which is deprecated"
+
           field :tabular_output_header, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
-          field :tabular_output_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true, copy_on_restart: true
+          field :tabular_output_format, type: Mongoid::StringifiedSymbol, default: :csv, class_attribute: true, user_editable: true, copy_on_restart: true
           field :tabular_output_options, type: Hash, class_attribute: true
 
           validates_inclusion_of :tabular_output_format, in: IOStreams::Tabular.registered_formats
@@ -55,7 +57,7 @@ module RocketJob
 
         # Render the output from the perform.
         def tabular_output_render
-          return unless collect_output?
+          return unless output_categories.present?
 
           @rocket_job_output = tabular_output.render(@rocket_job_output)
         end

data/lib/rocket_job/batch/throttle_running_workers.rb

@@ -37,34 +37,25 @@ module RocketJob
         validates :throttle_running_workers, numericality: {greater_than_or_equal_to: 0}, allow_nil: true
 
         define_batch_throttle :throttle_running_workers_exceeded?, filter: :throttle_filter_id
-
-        # Deprecated. For backward compatibility.
-        alias_method :throttle_running_slices, :throttle_running_workers
-        alias_method :throttle_running_slices=, :throttle_running_workers=
       end
 
       private
 
-      # Returns [Boolean] whether the throttle for this job has been exceeded
+      # Returns [true|false] whether the throttle for this job has been exceeded
       def throttle_running_workers_exceeded?(slice)
-        return unless throttle_running_workers&.positive?
+        return false unless throttle_running_workers&.positive?
 
         input.running.with(read: {mode: :primary}) do |conn|
           conn.where(:id.ne => slice.id).count >= throttle_running_workers
         end
       end
 
-      # Returns [Boolean] whether the throttle for this job has been exceeded
-      #
-      # With a Batch job, allow a higher priority queued job to replace a running one with
-      # a lower priority.
-      def throttle_running_jobs_exceeded?
-        return unless throttle_running_jobs&.positive?
-
-        # Cannot use this class since it will include instances of parent job classes.
-        RocketJob::Job.with(read: {mode: :primary}) do |conn|
-          conn.running.where("_type" => self.class.name, :id.ne => id, :priority.lte => priority).count >= throttle_running_jobs
-        end
+      # Allows another job with a higher priority to start even though this one is running already
+      # @overrides RocketJob::Plugins::Job::ThrottleRunningJobs#throttle_running_jobs_base_query
+      def throttle_running_jobs_base_query
+        query = super
+        query[:priority.lte] = priority if throttle_running_workers&.positive?
+        query
       end
     end
   end

data/lib/rocket_job/batch/worker.rb

@@ -23,9 +23,6 @@ module RocketJob
       #
       # Slices are destroyed after their records are successfully processed
       #
-      # Results are stored in the output collection if `collect_output?`
-      # `nil` results from workers are kept if `collect_nil_output`
-      #
       # If an exception was thrown the entire slice of records is marked as failed.
       #
       # Thread-safe, can be called by multiple threads at the same time
@@ -40,7 +37,8 @@ module RocketJob
 
         SemanticLogger.named_tagged(job: id.to_s) do
           until worker.shutdown?
-            if slice = input.next_slice(worker.name)
+            slice = input.next_slice(worker.name)
+            if slice
               # Grab a slice before checking the throttle to reduce concurrency race condition.
               return true if slice.fail_on_exception!(re_raise_exceptions) { rocket_job_batch_throttled?(slice, worker) }
               next if slice.failed?
@@ -97,7 +95,7 @@ module RocketJob
         servers = []
         case sub_state
         when :before, :after
-          unless server_name && !worker_on_server?(server_name)
+          if running? && (server_name.nil? || worker_on_server?(server_name))
             servers << ActiveWorker.new(worker_name, started_at, self) if running?
           end
         when :processing
@@ -143,19 +141,23 @@ module RocketJob
 
       # Perform individual slice without callbacks
       def rocket_job_perform_slice(slice, &block)
-        count = 0
-        RocketJob::Sliced::Writer::Output.collect(self, slice) do |writer|
-          records = slice.records
-
-          # Skip records already processed, if any.
-          # slice.processing_record_number ||= 0
-          # TODO: Must append to existing output slices before this can be enabled.
-          # if !collect_output && (slice.processing_record_number > 1)
-          #   records = records[slice.processing_record_number - 1..-1]
-          # end
-          # Until the changes above have been implemented, reprocess all records in the slice.
+        slice.processing_record_number ||= 0
+        records                        = []
+        append                         = false
+
+        # Skip processed records in this slice if it has no output categpries.
+        if slice.processing_record_number > 1
+          records = slice.records[slice.processing_record_number - 1..-1]
+          append  = true
+          logger.info("Resuming previously incomplete slice from record number #{slice.processing_record_number}")
+        else
+          # Reprocess all records in this slice.
           slice.processing_record_number = 0
+          records                        = slice.records
+        end
 
+        count = 0
+        RocketJob::Sliced::Writer::Output.collect(self, input_slice: slice, append: append) do |writer|
           records.each do |record|
             slice.processing_record_number += 1
             SemanticLogger.named_tagged(record: slice.current_record_number) do
@@ -174,8 +176,8 @@ module RocketJob
         return block_given? ? yield(record) : perform(record) if _perform_callbacks.empty?
 
         # @rocket_job_input and @rocket_job_output can be modified by before/around callbacks
-        @rocket_job_input         = record
-        @rocket_job_output        = nil
+        @rocket_job_input  = record
+        @rocket_job_output = nil
 
         run_callbacks(:perform) do
           @rocket_job_output =
@@ -186,9 +188,9 @@ module RocketJob
             end
         end
 
-        @rocket_job_input         = nil
-        result                    = @rocket_job_output
-        @rocket_job_output        = nil
+        @rocket_job_input  = nil
+        result             = @rocket_job_output
+        @rocket_job_output = nil
         result
       end
 
@@ -244,7 +246,7 @@ module RocketJob
         unless new_record?
           # Fail job iff no other worker has already finished it
           # Must set write concern to at least 1 since we need the nModified back
-          result = self.class.with(write: {w: 1}) do |query|
+          result   = self.class.with(write: {w: 1}) do |query|
             query.
               where(id: id, state: :running, sub_state: :processing).
               update({"$set" => {state: :failed, worker_name: worker_name}})
@@ -305,11 +307,12 @@ module RocketJob
       # Run Batch before and after callbacks
       def rocket_job_batch_callbacks(worker)
         # If this is the first worker to pickup this job
-        if sub_state == :before
+        case sub_state
+        when :before
           rocket_job_batch_run_before_callbacks
           # Check for 0 record jobs
           rocket_job_batch_complete?(worker.name) if running?
-        elsif sub_state == :after
+        when sub_state == :after
           rocket_job_batch_run_after_callbacks
         end
       end