rocketjob 5.1.1 → 5.2.0.beta1

data/lib/rocket_job/plugins/document.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -8,27 +8,29 @@ module RocketJob
       include ::Mongoid::Document
 
       included do
-        store_in client: 'rocketjob'
+        store_in client: "rocketjob"
       end
 
+      # rubocop:disable Style/RedundantSort
       module ClassMethods
         # Mongoid does not apply ordering, add sort
         def first
-          all.sort('_id' => 1).first
+          all.sort("_id" => 1).first
         end
 
         # Mongoid does not apply ordering, add sort
         def last
-          all.sort('_id' => -1).first
+          all.sort("_id" => -1).first
         end
       end
+      # rubocop:enable Style/RedundantSort
 
       private
 
       # Apply changes to this document returning the updated document from the database.
       # Allows other changes to be made on the server that will be loaded.
       def find_and_update(attrs)
-        doc = collection.find(_id: id).find_one_and_update({'$set' => attrs}, return_document: :after)
+        doc = collection.find(_id: id).find_one_and_update({"$set" => attrs}, return_document: :after)
         raise(::Mongoid::Errors::DocumentNotFound.new(self.class, id)) unless doc
 
         # Clear out keys that are not returned during the reload from MongoDB

data/lib/rocket_job/plugins/job/callbacks.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins

data/lib/rocket_job/plugins/job/logger.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -18,9 +18,9 @@ module RocketJob
         #   - on_exception_level changes log level from info to error on exception
         #   - silence noisy jobs by raising log level
         def rocket_job_around_logger(&block)
-          logger.info('Start #perform')
+          logger.info("Start #perform")
           logger.measure_info(
-            'Completed #perform',
+            "Completed #perform",
             metric:             "#{self.class.name}/perform",
             log_exception:      :full,
             on_exception_level: :error,

data/lib/rocket_job/plugins/job/model.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -87,7 +87,7 @@ module RocketJob
           field :percent_complete, type: Integer, default: 0
 
           # Store the last exception for this job
-          embeds_one :exception, class_name: 'RocketJob::JobException'
+          embeds_one :exception, class_name: "RocketJob::JobException"
 
           # Store the Hash result from this job if collect_output is true,
           # and the job returned actually returned a Hash, otherwise nil
@@ -110,7 +110,7 @@ module RocketJob
           #   job.underscore_name
           #   # => "data_study"
           def underscore_name
-            @underscore_name ||= name.sub(/Job$/, '').underscore
+            @underscore_name ||= name.sub(/Job$/, "").underscore
           end
 
           # Allow the collective name for this job class to be overridden
@@ -125,7 +125,7 @@ module RocketJob
           #   job.human_name
           #   # => "Data Study"
           def human_name
-            @human_name ||= name.sub(/Job$/, '').titleize
+            @human_name ||= name.sub(/Job$/, "").titleize
           end
 
           # Allow the human readable job name for this job class to be overridden
@@ -140,7 +140,7 @@ module RocketJob
           #   job.collective_name
           #   # => "data_studies"
           def collective_name
-            @collective_name ||= name.sub(/Job$/, '').pluralize.underscore
+            @collective_name ||= name.sub(/Job$/, "").pluralize.underscore
           end
 
           # Allow the collective name for this job class to be overridden
@@ -155,8 +155,14 @@ module RocketJob
 
           # Scope for queued jobs that can run now
           # I.e. Queued jobs excluding scheduled jobs
-          def queued_now
-            queued.or({run_at: nil}, :run_at.lte => Time.now)
+          if Mongoid::VERSION.to_f >= 7.1
+            def queued_now
+              queued.and(RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now))
+            end
+          else
+            def queued_now
+              queued.or({run_at: nil}, :run_at.lte => Time.now)
+            end
           end
 
           # Defines all the fields that are accessible on the Document
@@ -193,7 +199,7 @@ module RocketJob
 
           # DEPRECATED
           def rocket_job
-            warn 'Replace calls to .rocket_job with calls to set class instance variables. For example: self.priority = 50'
+            warn "Replace calls to .rocket_job with calls to set class instance variables. For example: self.priority = 50"
             yield(self)
           end
 
@@ -276,42 +282,42 @@ module RocketJob
         # Returns [Hash] status of this job
         def as_json
           attrs = serializable_hash(methods: %i[seconds duration])
-          attrs.delete('result') unless collect_output?
-          attrs.delete('failure_count') unless failure_count.positive?
+          attrs.delete("result") unless collect_output?
+          attrs.delete("failure_count") unless failure_count.positive?
           if queued?
-            attrs.delete('started_at')
-            attrs.delete('completed_at')
-            attrs.delete('result')
+            attrs.delete("started_at")
+            attrs.delete("completed_at")
+            attrs.delete("result")
             attrs
           elsif running?
-            attrs.delete('completed_at')
-            attrs.delete('result')
+            attrs.delete("completed_at")
+            attrs.delete("result")
             attrs
           elsif completed?
-            attrs.delete('percent_complete')
+            attrs.delete("percent_complete")
             attrs
           elsif paused?
-            attrs.delete('completed_at')
-            attrs.delete('result')
+            attrs.delete("completed_at")
+            attrs.delete("result")
             # Ensure 'paused_at' appears first in the hash
-            {'paused_at' => completed_at}.merge(attrs)
+            {"paused_at" => completed_at}.merge(attrs)
           elsif aborted?
-            attrs.delete('completed_at')
-            attrs.delete('result')
-            {'aborted_at' => completed_at}.merge(attrs)
+            attrs.delete("completed_at")
+            attrs.delete("result")
+            {"aborted_at" => completed_at}.merge(attrs)
           elsif failed?
-            attrs.delete('completed_at')
-            attrs.delete('result')
-            {'failed_at' => completed_at}.merge(attrs)
+            attrs.delete("completed_at")
+            attrs.delete("result")
+            {"failed_at" => completed_at}.merge(attrs)
           else
             attrs
           end
         end
 
         # Returns [Hash] the status of this job
-        def status(time_zone = 'Eastern Time (US & Canada)')
+        def status(time_zone = "Eastern Time (US & Canada)")
           h = as_json
-          h.delete('seconds')
+          h.delete("seconds")
           h.dup.each_pair do |k, v|
             if v.is_a?(Time)
               h[k] = v.in_time_zone(time_zone).to_s
@@ -325,6 +331,7 @@ module RocketJob
         # Returns [Boolean] whether the worker runs on a particular server.
         def worker_on_server?(server_name)
           return false unless worker_name.present? && server_name.present?
+
           worker_name.start_with?(server_name)
         end
       end

data/lib/rocket_job/plugins/job/persistence.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -9,38 +9,10 @@ module RocketJob
 
         included do
           # Store all job types in this collection
-          store_in collection: 'rocket_job.jobs'
+          store_in collection: "rocket_job.jobs"
         end
 
         module ClassMethods
-          # Retrieves the next job to work on in priority based order
-          # and assigns it to this worker
-          #
-          # Returns nil if no jobs are available for processing
-          #
-          # Parameters
-          #   worker_name: [String]
-          #     Name of the worker that will be processing this job
-          #
-          #   filter: [Hash]
-          #     Filter to apply to the query.
-          #     For example: to exclude jobs from being returned.
-          #
-          # Example:
-          #   # Skip any job ids from the job_ids_list
-          #   filter = {:id.nin => job_ids_list}
-          #   job    = RocketJob::Job.rocket_job_retrieve('host:pid:worker', filter)
-          def rocket_job_retrieve(worker_name, filter)
-            SemanticLogger.silence(:info) do
-              scheduled = {'$or' => [{run_at: nil}, {:run_at.lte => Time.now}]}
-              working   = {'$or' => [{state: :queued}, {state: :running, sub_state: :processing}]}
-              query     = self.and(working, scheduled)
-              query     = query.where(filter) unless filter.blank?
-              update    = {'$set' => {'worker_name' => worker_name, 'state' => 'running'}}
-              query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
-            end
-          end
-
           # Returns [Hash<String:Integer>] of the number of jobs in each state
           # Queued jobs are separated into :queued_now and :scheduled
           #   :queued_now are jobs that are awaiting processing and can be processed now.
@@ -73,14 +45,14 @@ module RocketJob
             collection.aggregate(
               [
                 {
-                  '$group' => {
-                    _id:   '$state',
-                    count: {'$sum' => 1}
+                  "$group" => {
+                    _id:   "$state",
+                    count: {"$sum" => 1}
                   }
                 }
               ]
             ).each do |result|
-              counts[result['_id'].to_sym] = result['count']
+              counts[result["_id"].to_sym] = result["count"]
             end
 
             # Calculate :queued_now and :scheduled if there are queued jobs
@@ -101,6 +73,7 @@ module RocketJob
         # Set in-memory job to complete if `destroy_on_complete` and the job has been destroyed
         def reload
           return super unless destroy_on_complete
+
           begin
             super
           rescue ::Mongoid::Errors::DocumentNotFound

data/lib/rocket_job/plugins/job/state_machine.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -126,7 +126,7 @@ module RocketJob
             exception.worker_name = worker_name
           else
             build_exception(
-              class_name:  'RocketJob::JobException',
+              class_name:  "RocketJob::JobException",
               message:     exc_or_message,
               backtrace:   [],
               worker_name: worker_name
@@ -134,8 +134,9 @@ module RocketJob
           end
         end
 
-        def rocket_job_set_started_at
-          self.started_at = Time.now
+        def rocket_job_set_started_at(worker_name = nil)
+          self.started_at  = Time.now
+          self.worker_name = worker_name if worker_name
         end
 
         def rocket_job_mark_complete

data/lib/rocket_job/plugins/job/throttle.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -30,7 +30,6 @@ module RocketJob
 
         included do
           class_attribute :rocket_job_throttles
-          self.rocket_job_throttles = []
         end
 
         module ClassMethods
@@ -40,6 +39,7 @@ module RocketJob
           #   method_name: [Symbol]
           #     Name of method to call to evaluate whether a throttle has been exceeded.
           #     Note: Must return true or false.
+          #
           #   filter: [Symbol|Proc]
           #     Name of method to call to return the filter when the throttle has been exceeded.
           #     Or, a block that will return the filter.
@@ -47,24 +47,24 @@ module RocketJob
           #
           # Note: Throttles are executed in the order they are defined.
           def define_throttle(method_name, filter: :throttle_filter_class)
-            unless filter.is_a?(Symbol) || filter.is_a?(Proc)
-              raise(ArgumentError, "Filter for #{method_name} must be a Symbol or Proc")
-            end
-            if throttle?(method_name)
-              raise(ArgumentError, "Cannot define #{method_name} twice, undefine previous throttle first")
-            end
-
-            self.rocket_job_throttles += [ThrottleDefinition.new(method_name, filter)]
+            # Duplicate to prevent modifying parent class throttles
+            definitions = rocket_job_throttles ? rocket_job_throttles.dup : ThrottleDefinitions.new
+            definitions.add(method_name, filter)
+            self.rocket_job_throttles = definitions
           end
 
           # Undefine a previously defined throttle
           def undefine_throttle(method_name)
-            rocket_job_throttles.delete_if { |throttle| throttle.method_name == method_name }
+            return unless rocket_job_throttles
+
+            definitions = rocket_job_throttles.dup
+            definitions.remove(method_name)
+            self.rocket_job_throttles = definitions
           end
 
           # Has a throttle been defined?
           def throttle?(method_name)
-            rocket_job_throttles.any? { |throttle| throttle.method_name == method_name }
+            rocket_job_throttles&.exist?(method_name)
           end
         end
 
@@ -81,22 +81,6 @@ module RocketJob
         def throttle_filter_id
           {:id.nin => [id]}
         end
-
-        private
-
-        ThrottleDefinition = Struct.new(:method_name, :filter)
-
-        # Returns the matching filter, or nil if no throttles were triggered.
-        def rocket_job_evaluate_throttles
-          rocket_job_throttles.each do |throttle|
-            # Throttle exceeded?
-            next unless send(throttle.method_name)
-            logger.debug { "Throttle: #{throttle.method_name} has been exceeded. #{self.class.name}:#{id}" }
-            filter = throttle.filter
-            return filter.is_a?(Proc) ? filter.call(self) : send(filter)
-          end
-          nil
-        end
       end
     end
   end

data/lib/rocket_job/plugins/job/throttle_running_jobs.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Plugins
@@ -40,7 +40,7 @@ module RocketJob
 
           # Cannot use this class since it will include instances of parent job classes.
           RocketJob::Job.with(read: {mode: :primary}) do |conn|
-            conn.running.where('_type' => self.class.name, :id.ne => id).count >= throttle_running_jobs
+            conn.running.where("_type" => self.class.name, :id.ne => id).count >= throttle_running_jobs
           end
         end
       end

data/lib/rocket_job/plugins/job/worker.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 # Worker behavior for a job
 module RocketJob
@@ -23,41 +23,6 @@ module RocketJob
             job
           end
 
-          # Returns the next job to work on in priority based order
-          # Returns nil if there are currently no queued jobs, or processing batch jobs
-          #   with records that require processing
-          #
-          # Parameters
-          #   worker_name [String]
-          #     Name of the worker that will be processing this job
-          #
-          #   skip_job_ids [Array<BSON::ObjectId>]
-          #     Job ids to exclude when looking for the next job
-          #
-          # Note:
-          #   If a job is in queued state it will be started
-          def rocket_job_next_job(worker_name, filter = {})
-            while (job = rocket_job_retrieve(worker_name, filter))
-              # Batch Job?
-              return job if job.running?
-
-              if job.expired?
-                job.rocket_job_fail_on_exception!(worker_name) { job.destroy }
-                logger.info "Destroyed expired job #{job.class.name}, id:#{job.id}"
-              elsif (new_filter = job.send(:rocket_job_evaluate_throttles))
-                rocket_job_merge_filter(filter, new_filter)
-                # Restore retrieved job so that other workers can process it later
-                job.set(worker_name: nil, state: :queued)
-              else
-                job.worker_name = worker_name
-                job.rocket_job_fail_on_exception!(worker_name) do
-                  job.start!
-                end
-                return job if job.running?
-              end
-            end
-          end
-
           # Requeues all jobs that were running on a server that died
           def requeue_dead_server(server_name)
             # Need to requeue paused, failed since user may have transitioned job before it finished
@@ -67,20 +32,6 @@ module RocketJob
               end
             end
           end
-
-          private
-
-          def rocket_job_merge_filter(target, source)
-            source.each_pair do |k, v|
-              target[k] =
-                if (previous = target[k])
-                  v.is_a?(Array) ? previous + v : v
-                else
-                  v
-                end
-            end
-            target
-          end
         end
 
         # Runs the job now in the current thread.
@@ -112,38 +63,38 @@ module RocketJob
         #
         # The job is automatically saved only if an exception is raised in the supplied block.
         #
-        # worker_name: [String]
-        #   Name of the server on which the exception has occurred
-        #
         # re_raise_exceptions: [true|false]
         #   Re-raise the exception after updating the job
         #   Default: false
-        def rocket_job_fail_on_exception!(worker_name, re_raise_exceptions = false)
-          yield
-        rescue Exception => exc
-          if failed? || !may_fail?
-            self.exception        = JobException.from_exception(exc)
-            exception.worker_name = worker_name
-            save! unless new_record? || destroyed?
-          elsif new_record? || destroyed?
-            fail(worker_name, exc)
-          else
-            fail!(worker_name, exc)
+        def fail_on_exception!(re_raise_exceptions = false, &block)
+          SemanticLogger.named_tagged(job: id.to_s, &block)
+        rescue Exception => e
+          SemanticLogger.named_tagged(job: id.to_s) do
+            if failed? || !may_fail?
+              self.exception        = JobException.from_exception(e)
+              exception.worker_name = worker_name
+              save! unless new_record? || destroyed?
+            elsif new_record? || destroyed?
+              fail(worker_name, e)
+            else
+              fail!(worker_name, e)
+            end
+            raise e if re_raise_exceptions
           end
-          raise exc if re_raise_exceptions
         end
 
         # Works on this job
         #
-        # Returns [true|false] whether this job should be excluded from the next lookup
+        # Returns [true|false] whether any work was performed.
         #
         # If an exception is thrown the job is marked as failed and the exception
         # is set in the job itself.
         #
         # Thread-safe, can be called by multiple threads at the same time
-        def rocket_job_work(worker, re_raise_exceptions = false, _filter = nil)
-          raise(ArgumentError, 'Job must be started before calling #rocket_job_work') unless running?
-          rocket_job_fail_on_exception!(worker.name, re_raise_exceptions) do
+        def rocket_job_work(_worker, re_raise_exceptions = false)
+          raise(ArgumentError, "Job must be started before calling #rocket_job_work") unless running?
+
+          fail_on_exception!(re_raise_exceptions) do
             if _perform_callbacks.empty?
               @rocket_job_output = perform
             else
@@ -156,7 +107,7 @@ module RocketJob
 
             if collect_output?
               # Result must be a Hash, if not put it in a Hash
-              self.result = @rocket_job_output.is_a?(Hash) ? @rocket_job_output : {'result' => @rocket_job_output}
+              self.result = @rocket_job_output.is_a?(Hash) ? @rocket_job_output : {"result" => @rocket_job_output}
             end
 
             if new_record? || destroyed?
@@ -171,6 +122,7 @@ module RocketJob
         # Returns [Hash<String:[Array<ActiveWorker>]>] All servers actively working on this job
         def rocket_job_active_workers(server_name = nil)
           return [] if !running? || (server_name && !worker_on_server?(server_name))
+
           [ActiveWorker.new(worker_name, started_at, self)]
         end
       end