rocketjob 5.1.1 → 5.2.0.beta1

data/lib/rocket_job/sliced/slices.rb

@@ -81,9 +81,10 @@ module RocketJob
 
         begin
           slice.save!
-        rescue Mongo::Error::OperationFailure => exc
+        rescue Mongo::Error::OperationFailure => e
           # Ignore duplicates since it means the job was restarted
-          raise(exc) unless exc.message.include?('E11000')
+          raise(e) unless e.message.include?("E11000")
+
           logger.warn "Skipped already processed slice# #{slice.id}"
         end
         slice
@@ -93,7 +94,7 @@ module RocketJob
 
       # Index for find_and_modify only if it is not already present
       def create_indexes
-        all.collection.indexes.create_one(state: 1, _id: 1) if all.collection.indexes.none? { |i| i['name'] == 'state_1__id_1' }
+        all.collection.indexes.create_one(state: 1, _id: 1) if all.collection.indexes.none? { |i| i["name"] == "state_1__id_1" }
       rescue Mongo::Error::OperationFailure
         all.collection.indexes.create_one(state: 1, _id: 1)
       end
@@ -124,13 +125,15 @@ module RocketJob
       end
 
       # Mongoid does not apply ordering, add sort
+      # rubocop:disable Style/RedundantSort
       def first
-        all.sort('_id' => 1).first
+        all.sort("_id" => 1).first
       end
 
       def last
-        all.sort('_id' => -1).first
+        all.sort("_id" => -1).first
       end
+      # rubocop:enable Style/RedundantSort
 
       # Returns [Array<Struct>] grouped exceptions by class name,
       # and unique exception messages by exception class.
@@ -149,19 +152,19 @@ module RocketJob
         result        = all.collection.aggregate(
           [
             {
-              '$match' => {state: 'failed'}
+              "$match" => {state: "failed"}
             },
             {
-              '$group' => {
-                _id:      {error_class: '$exception.class_name'},
-                messages: {'$addToSet' => '$exception.message'},
-                count:    {'$sum' => 1}
+              "$group" => {
+                _id:      {error_class: "$exception.class_name"},
+                messages: {"$addToSet" => "$exception.message"},
+                count:    {"$sum" => 1}
               }
             }
           ]
         )
         result.collect do |errors|
-          result_struct.new(errors['_id']['error_class'], errors['count'], errors['messages'])
+          result_struct.new(errors["_id"]["error_class"], errors["count"], errors["messages"])
         end
       end
     end

data/lib/rocket_job/subscriber.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   # Mix-in to publish and subscribe to events.
@@ -66,12 +66,12 @@ module RocketJob
         return
       end
 
-      args = (method(action).arity == 0) || parameters.nil? ? nil : parameters.symbolize_keys
+      args = method(action).arity.zero? || parameters.nil? ? nil : parameters.symbolize_keys
       args ? public_send(action, **args) : public_send(action)
-    rescue ArgumentError => exc
-      logger.error("##{action}: Invalid Arguments. Resuming..", exc)
-    rescue StandardError => exc
-      logger.error("##{action}: Exception caught. Resuming..", exc)
+    rescue ArgumentError => e
+      logger.error("##{action}: Invalid Arguments. Resuming..", e)
+    rescue StandardError => e
+      logger.error("##{action}: Exception caught. Resuming..", e)
     end
 
     def process_event(name, action, parameters)

data/lib/rocket_job/subscribers/logger.rb

@@ -1,4 +1,4 @@
-require 'socket'
+require "socket"
 
 module RocketJob
   module Subscribers
@@ -9,8 +9,8 @@ module RocketJob
         @host_name ||= Socket.gethostname
       end
 
-      def self.host_name=(host_name)
-        @host_name = host_name
+      class << self
+        attr_writer :host_name
       end
 
       # Change the log level

data/lib/rocket_job/supervisor.rb

@@ -1,4 +1,4 @@
-require 'rocket_job/supervisor/shutdown'
+require "rocket_job/supervisor/shutdown"
 
 module RocketJob
   # Starts a server instance, along with the workers and ensures workers remain running until they need to shutdown.
@@ -11,7 +11,7 @@ module RocketJob
 
     # Start the Supervisor, using the supplied attributes to create a new Server instance.
     def self.run
-      Thread.current.name = 'rocketjob main'
+      Thread.current.name = "rocketjob main"
       RocketJob.create_indexes
       register_signal_handlers
 
@@ -29,9 +29,9 @@ module RocketJob
 
     def run
       logger.info "Using MongoDB Database: #{RocketJob::Job.collection.database.name}"
-      logger.info('Running with filter', Config.filter) if Config.filter
+      logger.info("Running with filter", Config.filter) if Config.filter
       server.started!
-      logger.info 'Rocket Job Server started'
+      logger.info "Rocket Job Server started"
 
       event_listener = Thread.new { Event.listener }
       Subscribers::Server.subscribe(self) do
@@ -43,21 +43,21 @@ module RocketJob
         end
       end
     rescue ::Mongoid::Errors::DocumentNotFound
-      logger.info('Server has been destroyed. Going down hard!')
-    rescue Exception => exc
-      logger.error('RocketJob::Server is stopping due to an exception', exc)
+      logger.info("Server has been destroyed. Going down hard!")
+    rescue Exception => e
+      logger.error("RocketJob::Server is stopping due to an exception", e)
     ensure
-      event_listener.kill if event_listener
+      event_listener&.kill
       # Logs the backtrace for each running worker
       worker_pool.log_backtraces
-      logger.info('Shutdown Complete')
+      logger.info("Shutdown Complete")
     end
 
     def stop!
       server.stop! if server.may_stop?
       worker_pool.stop
-      while !worker_pool.join
-        logger.info 'Waiting for workers to finish processing ...'
+      until worker_pool.join
+        logger.info "Waiting for workers to finish processing ..."
         # One or more workers still running so update heartbeat so that server reports "alive".
         server.refresh(worker_pool.living_count)
       end
@@ -65,7 +65,7 @@ module RocketJob
 
     def supervise_pool
       stagger = true
-      while !self.class.shutdown?
+      until self.class.shutdown?
         synchronize do
           if server.running?
             worker_pool.prune

data/lib/rocket_job/supervisor/shutdown.rb

@@ -1,5 +1,5 @@
-require 'active_support/concern'
-require 'concurrent'
+require "active_support/concern"
+require "concurrent"
 
 module RocketJob
   class Supervisor
@@ -37,23 +37,23 @@ module RocketJob
         #   Perform clean shutdown
         #
         def self.register_signal_handlers
-          Signal.trap 'SIGTERM' do
+          Signal.trap "SIGTERM" do
             Thread.new do
               shutdown!
-              message = 'Shutdown signal (SIGTERM) received. Will shutdown as soon as active jobs/slices have completed.'
+              message = "Shutdown signal (SIGTERM) received. Will shutdown as soon as active jobs/slices have completed."
               logger.info(message)
             end
           end
 
-          Signal.trap 'INT' do
+          Signal.trap "INT" do
             Thread.new do
               shutdown!
-              message = 'Shutdown signal (INT) received. Will shutdown as soon as active jobs/slices have completed.'
+              message = "Shutdown signal (INT) received. Will shutdown as soon as active jobs/slices have completed."
               logger.info(message)
             end
           end
         rescue StandardError
-          logger.warn 'SIGTERM handler not installed. Not able to shutdown gracefully'
+          logger.warn "SIGTERM handler not installed. Not able to shutdown gracefully"
         end
 
         private_class_method :register_signal_handlers

data/lib/rocket_job/throttle_definition.rb

@@ -0,0 +1,37 @@
+module RocketJob
+  class ThrottleDefinition
+    attr_reader :method_name, :filter
+
+    def initialize(method_name, filter)
+      @method_name = method_name.to_sym
+      @filter      = filter
+    end
+
+    # Returns [true|false] whether the throttle was triggered.
+    def throttled?(job, *args)
+      # Throttle exceeded?
+      # Throttle methods can be private.
+      throttled =
+        if args.size.positive?
+          job.method(method_name).arity.zero? ? job.send(method_name) : job.send(method_name, *args)
+        else
+          job.send(method_name)
+        end
+      return false unless throttled
+
+      job.logger.debug { "Throttle: #{method_name} has been exceeded." }
+      true
+    end
+
+    # Returns the filter to apply to the job when the above throttle returns true.
+    def extract_filter(job, *args)
+      return filter.call(job, *args) if filter.is_a?(Proc)
+
+      if args.size.positive?
+        job.method(filter).arity.zero? ? job.send(filter) : job.send(filter, *args)
+      else
+        job.send(filter)
+      end
+    end
+  end
+end

data/lib/rocket_job/throttle_definitions.rb

@@ -0,0 +1,39 @@
+module RocketJob
+  class ThrottleDefinitions
+    attr_reader :throttles
+
+    def initialize
+      @throttles = []
+    end
+
+    def add(method_name, filter)
+      unless filter.is_a?(Symbol) || filter.is_a?(Proc)
+        raise(ArgumentError, "Filter for #{method_name} must be a Symbol or Proc")
+      end
+      raise(ArgumentError, "Cannot define #{method_name} twice, undefine previous throttle first") if exist?(method_name)
+
+      @throttles += [ThrottleDefinition.new(method_name, filter)]
+    end
+
+    # Undefine a previously defined throttle
+    def remove(method_name)
+      throttles.delete_if { |throttle| throttle.method_name == method_name }
+    end
+
+    # Has a throttle been defined?
+    def exist?(method_name)
+      throttles.any? { |throttle| throttle.method_name == method_name }
+    end
+
+    # Returns the matching filter,
+    # or nil if no throttles were triggered.
+    def matching_filter(job, *args)
+      throttles.each do |throttle|
+        next unless throttle.throttled?(job, *args)
+
+        return throttle.extract_filter(job, *args)
+      end
+      nil
+    end
+  end
+end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = '5.1.1'.freeze
+  VERSION = "5.2.0.beta1".freeze
 end

data/lib/rocket_job/worker.rb

@@ -1,5 +1,5 @@
-require 'concurrent'
-require 'forwardable'
+require "concurrent"
+require "forwardable"
 module RocketJob
   # Worker
   #
@@ -12,13 +12,13 @@ module RocketJob
     define_callbacks :running
 
     attr_accessor :id, :current_filter
-    attr_reader :thread, :name, :inline
+    attr_reader :thread, :name, :inline, :server_name
 
     # Raised when a worker is killed so that it shutdown immediately, yet cleanly.
     #
     # Note:
     # - It is not recommended to catch this exception since it is to shutdown workers quickly.
-    class Shutdown < Interrupt
+    class Shutdown < RuntimeError
     end
 
     def self.before_running(*filters, &blk)
@@ -33,7 +33,7 @@ module RocketJob
       set_callback(:running, :around, *filters, &blk)
     end
 
-    def initialize(id: 0, server_name: 'inline:0', inline: false)
+    def initialize(id: 0, server_name: "inline:0", inline: false)
       @id             = id
       @server_name    = server_name
       @shutdown       = Concurrent::Event.new
@@ -76,49 +76,37 @@ module RocketJob
       @shutdown.wait(timeout)
     end
 
-    private
-
     # Process jobs until it shuts down
     #
     # Params
     #   worker_id [Integer]
     #     The number of this worker for logging purposes
     def run
-      Thread.current.name = format('rocketjob %03i', id)
-      logger.info 'Started'
-      until shutdown?
-        wait = Config.max_poll_seconds
-        if process_available_jobs
-          # Keeps workers staggered across the poll interval so that
-          # all workers don't poll at the same time
-          wait = rand(wait * 1000) / 1000
-        end
-        break if wait_for_shutdown?(wait)
-      end
-      logger.info 'Stopping'
-    rescue Exception => exc
-      logger.fatal('Unhandled exception in job processing thread', exc)
-    ensure
-      ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
-    end
+      Thread.current.name = format("rocketjob %03i", id)
+      logger.info "Started"
 
-    # Process the next available job
-    # Returns [Boolean] whether any job was actually processed
-    def process_available_jobs
-      processed = false
       until shutdown?
+        sleep_seconds = Config.max_poll_seconds
         reset_filter_if_expired
-        job = Job.rocket_job_next_job(name, current_filter)
-        break unless job
-
-        SemanticLogger.named_tagged(job: job.id.to_s) do
-          processed = true unless job.rocket_job_work(self, false, current_filter)
+        job = next_available_job
 
+        # Returns true when work was completed, but no other work is available
+        if job&.rocket_job_work(self, false)
           # Return the database connections for this thread back to the connection pool
           ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
+
+          # Stagger workers so that they don't all poll at the same time.
+          sleep_seconds = random_wait_interval
         end
+
+        wait_for_shutdown?(sleep_seconds)
       end
-      processed
+
+      logger.info "Stopping"
+    rescue Exception => e
+      logger.fatal("Unhandled exception in job processing thread", e)
+    ensure
+      ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
     end
 
     # Resets the current job filter if the relevant time interval has passed
@@ -130,5 +118,99 @@ module RocketJob
       @re_check_start     = time
       self.current_filter = Config.filter || {}
     end
+
+    # Returns [RocketJob::Job] the next job available for processing.
+    # Returns [nil] if no job is available for processing.
+    #
+    # Notes:
+    # - Destroys expired jobs
+    # - Runs job throttles and skips the job if it is throttled.
+    #   - Adding that filter to the current filter to exclude from subsequent polling.
+    def next_available_job
+      until shutdown?
+        job = find_and_assign_job
+        return unless job
+
+        if job.expired?
+          job.fail_on_exception! do
+            job.worker_name = name
+            job.destroy
+            logger.info("Destroyed expired job.")
+          end
+          next
+        end
+
+        # Batch Job that is already started?
+        # Batch has its own throttles for slices.
+        return job if job.running?
+
+        # Should this job be throttled?
+        next if job.fail_on_exception! { throttled_job?(job) }
+
+        # Start this job!
+        job.fail_on_exception! { job.start!(name) }
+        return job if job.running?
+      end
+    end
+
+    # Whether the supplied job has been throttled and should be ignored.
+    def throttled_job?(job)
+      # Evaluate job throttles, if any.
+      filter = job.rocket_job_throttles.matching_filter(job)
+      return false unless filter
+
+      add_to_current_filter(filter)
+      # Restore retrieved job so that other workers can process it later
+      job.set(worker_name: nil, state: :queued)
+      true
+    end
+
+    # Finds the next job to work on in priority based order
+    # and assigns it to this worker.
+    #
+    # Applies the current filter to exclude filtered jobs.
+    #
+    # Returns nil if no jobs are available for processing.
+    if Mongoid::VERSION.to_f >= 7.1
+      def find_and_assign_job
+        SemanticLogger.silence(:info) do
+          scheduled = RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now)
+          working   = RocketJob::Job.queued.or(state: :running, sub_state: :processing)
+          query     = RocketJob::Job.and(working, scheduled)
+          query     = query.and(current_filter) unless current_filter.blank?
+          update    = {"$set" => {"worker_name" => name, "state" => "running"}}
+          query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
+        end
+      end
+    else
+      def find_and_assign_job
+        SemanticLogger.silence(:info) do
+          scheduled = {"$or" => [{run_at: nil}, {:run_at.lte => Time.now}]}
+          working   = {"$or" => [{state: :queued}, {state: :running, sub_state: :processing}]}
+          query     = RocketJob::Job.and(working, scheduled)
+          query     = query.where(current_filter) unless current_filter.blank?
+          update    = {"$set" => {"worker_name" => name, "state" => "running"}}
+          query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
+        end
+      end
+    end
+
+    # Add the supplied filter to the current filter.
+    def add_to_current_filter(filter)
+      filter.each_pair do |k, v|
+        current_filter[k] =
+          if (previous = current_filter[k])
+            v.is_a?(Array) ? previous + v : v
+          else
+            v
+          end
+      end
+      current_filter
+    end
+
+    # Returns [Float] a randomized poll interval in seconds up to the maximum configured poll interval.
+    def random_wait_interval
+      rand(Config.max_poll_seconds * 1000) / 1000
+    end
   end
 end