rocketjob 5.1.1 → 5.2.0.beta1

data/lib/rocket_job/plugins/processing_window.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -49,8 +49,8 @@ module RocketJob
         validates_each :processing_schedule do |record, attr, value|
           begin
             RocketJob::Plugins::Rufus::CronLine.new(value)
-          rescue ArgumentError => exc
-            record.errors.add(attr, exc.message)
+          rescue ArgumentError => e
+            record.errors.add(attr, e.message)
           end
         end
       end
@@ -68,8 +68,9 @@ module RocketJob
       # Only process this job if it is still in its processing window
       def rocket_job_processing_window_check
         return if rocket_job_processing_window_active?
+
         logger.warn("Processing window closed before job was processed. Job is re-scheduled to run at: #{rocket_job_processing_schedule.next_time}")
-        self.worker_name ||= 'inline'
+        self.worker_name ||= "inline"
         requeue!(worker_name)
       end
 

data/lib/rocket_job/plugins/restart.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -88,7 +88,7 @@ module RocketJob
       # Run again in the future, even if this run fails with an exception
       def rocket_job_restart_new_instance
         if expired?
-          logger.info('Job has expired. Not creating a new instance.')
+          logger.info("Job has expired. Not creating a new instance.")
           return
         end
         attributes = rocket_job_restart_attributes.each_with_object({}) { |attr, attrs| attrs[attr] = send(attr) }
@@ -109,7 +109,7 @@ module RocketJob
             logger.info("Created a new job instance: #{job.id}")
             return true
           else
-            logger.info('Job already active, retrying after a short sleep')
+            logger.info("Job already active, retrying after a short sleep")
             sleep(sleep_interval)
           end
           count += 1

data/lib/rocket_job/plugins/retry.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -76,7 +76,7 @@ module RocketJob
       # Returns [Time] when to retry this job at
       # Same basic formula as Delayed Job
       def rocket_job_retry_seconds_to_delay
-        (rocket_job_failure_count ** 4) + 15 + (rand(30) * (rocket_job_failure_count + 1))
+        (rocket_job_failure_count**4) + 15 + (rand(30) * (rocket_job_failure_count + 1))
       end
     end
   end

data/lib/rocket_job/plugins/singleton.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -25,7 +25,6 @@ module RocketJob
 
         errors.add(:state, "Another instance of #{self.class.name} is already running, queued, or paused")
       end
-
     end
   end
 end

data/lib/rocket_job/plugins/state_machine.rb

@@ -1,5 +1,5 @@
-require 'active_support/concern'
-require 'aasm'
+require "active_support/concern"
+require "aasm"
 
 module RocketJob
   module Plugins
@@ -33,8 +33,8 @@ module RocketJob
         # Adds a :before or :after callback to an event
         #  state_machine_add_event_callback(:start, :before, :my_method)
         def self.state_machine_add_event_callback(event_name, action, *methods, &block)
-          raise(ArgumentError, 'Cannot supply both a method name and a block') if methods.size.positive? && block
-          raise(ArgumentError, 'Must supply either a method name or a block') unless methods.size.positive? || block
+          raise(ArgumentError, "Cannot supply both a method name and a block") if methods.size.positive? && block
+          raise(ArgumentError, "Must supply either a method name or a block") unless methods.size.positive? || block
 
           # TODO: Somehow get AASM to support options such as :if and :unless to be consistent with other callbacks
           # For example:

data/lib/rocket_job/plugins/transaction.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins

data/lib/rocket_job/rocket_job.rb

@@ -26,20 +26,21 @@ module RocketJob
   # Returns a human readable duration from the supplied [Float] number of seconds
   def self.seconds_as_duration(seconds)
     return nil unless seconds
+
     if seconds >= 86_400.0 # 1 day
       "#{(seconds / 86_400).to_i}d #{Time.at(seconds).strftime('%-Hh %-Mm')}"
     elsif seconds >= 3600.0 # 1 hour
-      Time.at(seconds).strftime('%-Hh %-Mm')
+      Time.at(seconds).strftime("%-Hh %-Mm")
     elsif seconds >= 60.0 # 1 minute
-      Time.at(seconds).strftime('%-Mm %-Ss')
+      Time.at(seconds).strftime("%-Mm %-Ss")
     elsif seconds >= 1.0 # 1 second
-      format('%.3fs', seconds)
+      format("%.3fs", seconds)
     else
       duration = seconds * 1000
       if defined? JRuby
         "#{duration.to_i}ms"
       else
-        duration < 10.0 ? format('%.3fms', duration) : format('%.1fms', duration)
+        duration < 10.0 ? format("%.3fms", duration) : format("%.1fms", duration)
       end
     end
   end

data/lib/rocket_job/server.rb

@@ -1,5 +1,5 @@
-require 'rocket_job/server/model'
-require 'rocket_job/server/state_machine'
+require "rocket_job/server/model"
+require "rocket_job/server/state_machine"
 
 module RocketJob
   # Server

data/lib/rocket_job/server/model.rb

@@ -1,5 +1,5 @@
-require 'yaml'
-require 'active_support/concern'
+require "yaml"
+require "active_support/concern"
 
 module RocketJob
   class Server
@@ -8,7 +8,7 @@ module RocketJob
       extend ActiveSupport::Concern
 
       included do
-        store_in collection: 'rocket_job.servers'
+        store_in collection: "rocket_job.servers"
 
         # Unique Name of this server instance
         #   Default: `host name:PID`
@@ -24,13 +24,13 @@ module RocketJob
         field :started_at, type: Time
 
         # The heartbeat information for this server
-        embeds_one :heartbeat, class_name: 'RocketJob::Heartbeat'
+        embeds_one :heartbeat, class_name: "RocketJob::Heartbeat"
 
         # Current state
         #   Internal use only. Do not set this field directly
         field :state, type: Symbol, default: :starting
 
-        index({name: 1}, background: true, unique: true, drop_dups: true)
+        index({name: 1}, background: true, unique: true)
 
         validates_presence_of :state, :name, :max_workers
 
@@ -58,8 +58,8 @@ module RocketJob
         #   # => {}
         def self.counts_by_state
           counts = {}
-          collection.aggregate([{'$group' => {_id: '$state', count: {'$sum' => 1}}}]).each do |result|
-            counts[result['_id'].to_sym] = result['count']
+          collection.aggregate([{"$group" => {_id: "$state", count: {"$sum" => 1}}}]).each do |result|
+            counts[result["_id"].to_sym] = result["count"]
           end
           counts
         end
@@ -70,6 +70,7 @@ module RocketJob
           count = 0
           each do |server|
             next unless server.zombie?
+
             logger.warn "Destroying zombie server #{server.name}, and requeueing its jobs"
             server.destroy
             count += 1
@@ -83,9 +84,9 @@ module RocketJob
           last_heartbeat_time = Time.now - dead_seconds
           where(
             :state.in => %i[stopping running paused],
-            '$or'     => [
-              {'heartbeat.updated_at' => {'$exists' => false}},
-              {'heartbeat.updated_at' => {'$lte' => last_heartbeat_time}}
+            "$or"     => [
+              {"heartbeat.updated_at" => {"$exists" => false}},
+              {"heartbeat.updated_at" => {"$lte" => last_heartbeat_time}}
             ]
           )
         end
@@ -100,6 +101,7 @@ module RocketJob
       def zombie?(missed = 4)
         return false unless running? || stopping? || paused?
         return true if heartbeat.nil? || heartbeat.updated_at.nil?
+
         dead_seconds = Config.heartbeat_seconds * missed
         (Time.now - heartbeat.updated_at) >= dead_seconds
       end
@@ -108,8 +110,8 @@ module RocketJob
       def refresh(worker_count)
         SemanticLogger.silence(:info) do
           find_and_update(
-            'heartbeat.updated_at' => Time.now,
-            'heartbeat.workers'    => worker_count
+            "heartbeat.updated_at" => Time.now,
+            "heartbeat.workers"    => worker_count
           )
         end
       end
@@ -120,7 +122,6 @@ module RocketJob
       def requeue_jobs
         RocketJob::Job.requeue_dead_server(name)
       end
-
     end
   end
 end

data/lib/rocket_job/server/state_machine.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   class Server
@@ -54,7 +54,6 @@ module RocketJob
           paused.each(&:resume!)
         end
       end
-
     end
   end
 end

data/lib/rocket_job/sliced/compressed_slice.rb

@@ -1,4 +1,4 @@
-require 'zlib'
+require "zlib"
 module RocketJob
   module Sliced
     # Compress the records within a slice
@@ -6,20 +6,20 @@ module RocketJob
       private
 
       def parse_records
-        records = attributes.delete('records')
+        records = attributes.delete("records")
 
         # Convert BSON::Binary to a string
         binary_str = records.data
 
         str      = Zlib::Inflate.inflate(binary_str)
-        @records = Hash.from_bson(BSON::ByteBuffer.new(str))['r']
+        @records = Hash.from_bson(BSON::ByteBuffer.new(str))["r"]
       end
 
       def serialize_records
         return [] if @records.nil? || @records.empty?
 
         # Convert slice of records into a single string
-        str = {'r' => records.to_a}.to_bson.to_s
+        str = {"r" => records.to_a}.to_bson.to_s
 
         data = Zlib::Deflate.deflate(str)
         BSON::Binary.new(data)

data/lib/rocket_job/sliced/encrypted_slice.rb

@@ -1,4 +1,4 @@
-require 'symmetric-encryption'
+require "symmetric-encryption"
 module RocketJob
   module Sliced
     # Compress the records within a slice
@@ -6,7 +6,7 @@ module RocketJob
       private
 
       def parse_records
-        records = attributes.delete('records')
+        records = attributes.delete("records")
 
         # Convert BSON::Binary to a string
         binary_str = records.data
@@ -16,14 +16,14 @@ module RocketJob
         # Use the header that is present to decrypt the data, since its version could be different
         str = header.cipher.binary_decrypt(binary_str, header: header)
 
-        @records = Hash.from_bson(BSON::ByteBuffer.new(str))['r']
+        @records = Hash.from_bson(BSON::ByteBuffer.new(str))["r"]
       end
 
       def serialize_records
         return [] if @records.nil? || @records.empty?
 
         # Convert slice of records into a single string
-        str = {'r' => to_a}.to_bson.to_s
+        str = {"r" => to_a}.to_bson.to_s
 
         # Encrypt to binary without applying an encoding such as Base64
         # Use a random_iv with each encryption for better security

data/lib/rocket_job/sliced/input.rb

@@ -5,9 +5,9 @@ module RocketJob
         # Create indexes before uploading
         create_indexes
         Writer::Input.collect(self, on_first: on_first, &block)
-      rescue StandardError => exc
+      rescue StandardError => e
         drop
-        raise(exc)
+        raise(e)
       end
 
       def upload_mongo_query(criteria, *column_names, &block)
@@ -19,7 +19,7 @@ module RocketJob
           options[:projection] = options.delete(:fields) if options.key?(:fields)
         else
           column_names = column_names.collect(&:to_s)
-          column_names << '_id' if column_names.size.zero?
+          column_names << "_id" if column_names.size.zero?
 
           fields = options.delete(:fields) || {}
           column_names.each { |col| fields[col] = 1 }
@@ -73,9 +73,9 @@ module RocketJob
           count    += 1
         end
         count
-      rescue StandardError => exc
+      rescue StandardError => e
         drop
-        raise(exc)
+        raise(e)
       end
 
       def upload_integer_range_in_reverse_order(start_id, last_id)
@@ -91,9 +91,9 @@ module RocketJob
           count  += 1
         end
         count
-      rescue StandardError => exc
+      rescue StandardError => e
         drop
-        raise(exc)
+        raise(e)
       end
 
       # Iterate over each failed record, if any
@@ -116,16 +116,16 @@ module RocketJob
       # Requeue all failed slices
       def requeue_failed
         failed.update_all(
-          '$unset' => {worker_name: nil, started_at: nil},
-          '$set'   => {state: :queued}
+          "$unset" => {worker_name: nil, started_at: nil},
+          "$set"   => {state: :queued}
         )
       end
 
       # Requeue all running slices for a server or worker that is no longer available
       def requeue_running(worker_name)
         running.where(worker_name: /\A#{worker_name}/).update_all(
-          '$unset' => {worker_name: nil, started_at: nil},
-          '$set'   => {state: :queued}
+          "$unset" => {worker_name: nil, started_at: nil},
+          "$set"   => {state: :queued}
         )
       end
 
@@ -137,11 +137,11 @@ module RocketJob
         # TODO: Will it perform faster without the id sort?
         # I.e. Just process on a FIFO basis?
         document                 = all.queued.
-          sort('_id' => 1).
-          find_one_and_update(
-            {'$set' => {worker_name: worker_name, state: :running, started_at: Time.now}},
-            return_document: :after
-          )
+                                   sort("_id" => 1).
+                                   find_one_and_update(
+                                     {"$set" => {worker_name: worker_name, state: :running, started_at: Time.now}},
+                                     return_document: :after
+                                   )
         document.collection_name = collection_name if document
         document
       end

data/lib/rocket_job/sliced/output.rb

@@ -1,10 +1,10 @@
-require 'tempfile'
+require "tempfile"
 
 module RocketJob
   module Sliced
     class Output < Slices
       def download(header_line: nil)
-        raise(ArgumentError, 'Block is mandatory') unless block_given?
+        raise(ArgumentError, "Block is mandatory") unless block_given?
 
         # Write the header line
         yield(header_line) if header_line

data/lib/rocket_job/sliced/slice.rb

@@ -1,4 +1,4 @@
-require 'forwardable'
+require "forwardable"
 module RocketJob
   module Sliced
     # A slice is an Array of Records, along with meta-data that is used
@@ -21,12 +21,11 @@ module RocketJob
       include RocketJob::Plugins::StateMachine
       extend Forwardable
 
-      store_in client: 'rocketjob_slices'
+      store_in client: "rocketjob_slices"
 
       # The record number of the first record in this slice.
-      #
-      # Optional: If present the record_number is set while the job
-      #           is being processed.
+      # Useful in knowing the line number of each record in this slice
+      # relative to the original file that was uploaded.
       field :first_record_number, type: Integer
 
       #
@@ -42,11 +41,14 @@ module RocketJob
       # Number of times that this job has failed to process
       field :failure_count, type: Integer
 
+      # Number of the record within this slice (not the entire file/job) currently being processed. (One based index)
+      field :processing_record_number, type: Integer
+
       # This name of the worker that this job is being processed by, or was processed by
       field :worker_name, type: String
 
       # The last exception for this slice if any
-      embeds_one :exception, class_name: 'RocketJob::JobException'
+      embeds_one :exception, class_name: "RocketJob::JobException"
 
       after_find :parse_records
 
@@ -108,12 +110,16 @@ module RocketJob
       def_instance_delegators :records, :each, :<<, :size, :concat, :at
       def_instance_delegators :records, *(Enumerable.instance_methods - Module.methods)
 
-      # Fail this slice, along with the exception that caused the failure
-      def set_exception(exc = nil, record_number = nil)
+      # Returns [Integer] the record number of the record currently being processed relative to the entire file.
+      def current_record_number
+        first_record_number.to_i + processing_record_number.to_i
+      end
+
+      # Before Fail save the exception to this slice.
+      def set_exception(exc = nil)
         if exc
-          self.exception          = JobException.from_exception(exc)
-          exception.worker_name   = worker_name
-          exception.record_number = record_number
+          self.exception        = JobException.from_exception(exc)
+          exception.worker_name = worker_name
         end
         self.failure_count = failure_count.to_i + 1
         self.worker_name   = nil
@@ -122,8 +128,8 @@ module RocketJob
       # Returns the failed record.
       # Returns [nil] if there is no failed record
       def failed_record
-        if exception && (record_number = exception.record_number)
-          at(record_number - 1)
+        if exception && processing_record_number
+          at(processing_record_number - 1)
         end
       end
 
@@ -132,13 +138,13 @@ module RocketJob
       if ::Mongoid::VERSION.to_i >= 6
         def as_attributes
           attrs            = super
-          attrs['records'] = serialize_records if @records
+          attrs["records"] = serialize_records if @records
           attrs
         end
       else
         def as_document
           attrs            = super
-          attrs['records'] = serialize_records if @records
+          attrs["records"] = serialize_records if @records
           attrs
         end
       end
@@ -147,25 +153,42 @@ module RocketJob
         "#{super[0...-1]}, records: #{@records.inspect}, collection_name: #{collection_name.inspect}>"
       end
 
+      # Fail this slice if an exception occurs during processing.
+      def fail_on_exception!(re_raise_exceptions = false, &block)
+        SemanticLogger.named_tagged(slice: id.to_s, &block)
+      rescue Exception => e
+        SemanticLogger.named_tagged(slice: id.to_s) do
+          if failed? || !may_fail?
+            exception             = JobException.from_exception(e)
+            exception.worker_name = worker_name
+            save! unless new_record? || destroyed?
+          elsif new_record? || destroyed?
+            fail(e)
+          else
+            fail!(e)
+          end
+          raise e if re_raise_exceptions
+        end
+      end
+
       private
 
       # Always add records to any updates.
       def atomic_updates(*args)
-        r = super(*args)
-        if @records
-          (r['$set'] ||= {})['records'] = serialize_records
-        end
+        r                             = super(*args)
+        (r["$set"] ||= {})["records"] = serialize_records if @records
         r
       end
 
       def parse_records
-        @records = attributes.delete('records')
+        @records = attributes.delete("records")
       end
 
       def serialize_records
         records.mongoize
       end
 
+      # Before Start
       def set_started_at
         self.started_at = Time.now
       end