rocketjob 6.3.2 → 7.1.0

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -94,11 +94,12 @@ module RocketJob
       end
 
       VALID_INSTANCE_METHODS = %i[upload upload_file_name= full_file_name=].freeze
+      private_constant :VALID_INSTANCE_METHODS
 
       # Validates job_class is a Rocket Job
       def job_implements_upload
         klass = job_class
-        return if klass.nil? || klass.instance_methods.any? { |m| VALID_INSTANCE_METHODS.include?(m) }
+        return if klass.nil? || klass.instance_methods.intersect?(VALID_INSTANCE_METHODS)
 
         errors.add(:job_class_name,
                    "#{job_class} must implement any one of: :#{VALID_INSTANCE_METHODS.join(' :')} instance methods")
@@ -120,14 +121,16 @@ module RocketJob
         klass = job_class
         return unless klass
 
-        properties.each_pair do |k, _v|
-          next if klass.public_method_defined?("#{k}=".to_sym)
+        # Mongoid 9 returns Hash field keys as Strings, earlier versions as Symbols.
+        properties.each_pair do |raw_key, value|
+          k = raw_key.to_sym
+          next if klass.public_method_defined?(:"#{k}=")
 
           if %i[output_categories input_categories].include?(k)
             category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
-            properties[k].each do |category|
+            value.each do |category|
               category.each_pair do |key, _value|
-                next if category_class.public_method_defined?("#{key}=".to_sym)
+                next if category_class.public_method_defined?(:"#{key}=")
 
                 errors.add(
                   :properties,

data/lib/rocket_job/lookup_collection.rb

@@ -12,8 +12,8 @@ module RocketJob
     #   end
     #
     #   input_category(:my_lookup).find(id: 123).first
-    def upload(batch_size: 10_000, &block)
-      BatchUploader.upload(batch_size: batch_size, &block)
+    def upload(batch_size: 10_000, &)
+      BatchUploader.upload(batch_size: batch_size, &)
     end
 
     # Looks up the value at the specified id.

data/lib/rocket_job/performance.rb

@@ -18,7 +18,7 @@ module RocketJob
     # Loads the queue with jobs to be processed once the queue is loaded.
     # Retain the first and last job for timings, all others are destroyed on completion.
     def run_test_case(count = self.count)
-      if RocketJob::Server.where(:state.in => %w[running paused]).count.zero?
+      if RocketJob::Server.where(:state.in => %w[running paused]).none?
         raise "Please start servers before starting the performance test"
       end
 

data/lib/rocket_job/plugins/cron.rb

@@ -18,7 +18,8 @@ module RocketJob
 
         # Whether to prevent another instance of this job from running with the exact _same_ cron schedule.
         # Another job instance with a different `cron_schedule` string is permitted.
-        field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+        field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true,
+copy_on_restart: true
 
         # Whether to re-schedule the next job occurrence when this job starts, or when it is complete.
         #
@@ -38,7 +39,8 @@ module RocketJob
         #     at any time until after the job has failed, or is aborted.
         #   - To prevent this job creating any new duplicate instances during subsequent processing,
         #     its `cron_schedule` is set to `nil` after it fails or is aborted.
-        field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+        field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true,
+copy_on_restart: true
 
         validates_each :cron_schedule do |record, attr, value|
           record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
@@ -88,7 +90,8 @@ module RocketJob
       def rocket_job_cron_singleton_check
         return if cron_schedule.nil? || completed? || aborted? || !rocket_job_cron_duplicate?
 
-        errors.add(:state, "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
+        errors.add(:state,
+                   "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
       end
     end
   end

data/lib/rocket_job/plugins/document.rb

@@ -11,7 +11,6 @@ module RocketJob
         store_in client: "rocketjob"
       end
 
-      # rubocop:disable Style/RedundantSort
       module ClassMethods
         # Mongoid does not apply ordering, add sort
         def first
@@ -23,7 +22,6 @@ module RocketJob
           all.sort("_id" => -1).first
         end
       end
-      # rubocop:enable Style/RedundantSort
 
       private
 

data/lib/rocket_job/plugins/job/logger.rb

@@ -17,7 +17,7 @@ module RocketJob
         #   - log_exception logs entire exception if raised
         #   - on_exception_level changes log level from info to error on exception
         #   - silence noisy jobs by raising log level
-        def rocket_job_around_logger(&block)
+        def rocket_job_around_logger(&)
           logger.info("Start #perform")
           logger.measure_info(
             "Completed #perform",
@@ -25,7 +25,7 @@ module RocketJob
             log_exception:      :full,
             on_exception_level: :error,
             silence:            log_level,
-            &block
+            &
           )
         end
       end

data/lib/rocket_job/plugins/job/model.rb

@@ -185,7 +185,7 @@ module RocketJob
               self.rocket_job_restart_attributes += [name.to_sym]
             end
 
-            super(name, options)
+            super
           end
 
           # Builds this job instance from the supplied properties hash.

data/lib/rocket_job/plugins/job/persistence.rb

@@ -79,8 +79,8 @@ module RocketJob
             return
           end
 
-          job_attrs = self.class.rocket_job_restart_attributes.each_with_object({}) do |attr, attrs|
-            attrs[attr] = send(attr)
+          job_attrs = self.class.rocket_job_restart_attributes.to_h do |attr|
+            [attr, send(attr)]
           end
           job_attrs.merge!(overrides)
 

data/lib/rocket_job/plugins/job/throttle.rb

@@ -30,6 +30,13 @@ module RocketJob
 
         included do
           class_attribute :rocket_job_throttles
+
+          # Human readable reason why this job is currently being throttled.
+          # Set when a throttle prevents the job from running, cleared when it starts.
+          # Surfaced in Rocket Job Mission Control.
+          field :throttled_by, type: String
+          # Time at which this job was last throttled.
+          field :throttled_at, type: Time
         end
 
         module ClassMethods
@@ -45,11 +52,16 @@ module RocketJob
           #     Or, a block that will return the filter.
           #     Default: :throttle_filter_class (Throttle all jobs of this class)
           #
+          #   description: [String|Proc]
+          #     Human readable reason why the job is throttled, persisted to `throttled_by`
+          #     and shown in Mission Control. A Proc is called with the job and must
+          #     return a String. Default: a humanized version of the method name.
+          #
           # Note: Throttles are executed in the order they are defined.
-          def define_throttle(method_name, filter: :throttle_filter_class)
+          def define_throttle(method_name, filter: :throttle_filter_class, description: nil)
             # Duplicate to prevent modifying parent class throttles
             definitions = rocket_job_throttles ? rocket_job_throttles.deep_dup : ThrottleDefinitions.new
-            definitions.add(method_name, filter)
+            definitions.add(method_name, filter, description)
             self.rocket_job_throttles = definitions
           end
 

data/lib/rocket_job/plugins/job/throttle_running_jobs.rb

@@ -32,7 +32,10 @@ module RocketJob
           # Allow jobs to be throttled by group name instance of the job class name.
           field :throttle_group, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
 
-          define_throttle :throttle_running_jobs_exceeded?
+          define_throttle :throttle_running_jobs_exceeded?,
+                          description: lambda { |job, *|
+                            "Throttled: maximum of #{job.throttle_running_jobs} running jobs reached"
+                          }
         end
 
         private

data/lib/rocket_job/plugins/job/worker.rb

@@ -66,10 +66,13 @@ module RocketJob
         # re_raise_exceptions: [true|false]
         #   Re-raise the exception after updating the job
         #   Default: false
-        def fail_on_exception!(re_raise_exceptions = false, &block)
-          SemanticLogger.named_tagged(job: id.to_s, &block)
+        def fail_on_exception!(re_raise_exceptions = false, &)
+          SemanticLogger.named_tagged(job: id.to_s, &)
         rescue Exception => e
           SemanticLogger.named_tagged(job: id.to_s) do
+            # Not a guard clause: the `fail` event clears `worker_name`, so the
+            # branches are not equivalent and must not be collapsed.
+            # rubocop:disable-next Style/GuardClause
             if failed? || !may_fail?
               self.exception        = JobException.from_exception(e)
               exception.worker_name = worker_name

data/lib/rocket_job/plugins/retry.rb

@@ -27,41 +27,42 @@ module RocketJob
     # MyCronJob.create!(retry_limit: 0)
     #
     # Below is a table of the delay for each retry attempt, as well as the _total_ duration spent retrying a job for
-    # the specified number of retries, excluding actual processing time.
+    # the specified number of retries, excluding actual processing time. The delay before retry `n` is
+    # `(n - 1)**4 + 5` seconds.
     #
     # |---------|------------|----------------|
     # | Attempt |      Delay | Total Duration |
     # |---------|------------|----------------|
-    # |      01 |     6.000s |         6.000s |
-    # |      02 |    21.000s |        27.000s |
-    # |      03 |     1m 26s |         1m 53s |
-    # |      04 |     4m 21s |         6m 14s |
-    # |      05 |    10m 30s |        16m 44s |
-    # |      06 |    21m 41s |        38m 25s |
-    # |      07 |     40m 6s |        20h 18m |
-    # |      08 |     20h 8m |        21h 26m |
-    # |      09 |    20h 49m |        23h 16m |
-    # |      10 |    21h 46m |          2h 3m |
-    # |      11 |     23h 4m |          6h 7m |
-    # |      12 |     0h 45m |        11h 52m |
-    # |      13 |     2h 56m |     1d 19h 48m |
-    # |      14 |     5h 40m |      1d 6h 29m |
-    # |      15 |      9h 3m |     2d 20h 33m |
-    # |      16 |    13h 12m |     2d 14h 45m |
-    # |      17 |    18h 12m |     3d 13h 57m |
-    # |      18 |   1d 0h 9m |      5d 19h 7m |
-    # |      19 |  1d 7h 12m |      6d 7h 19m |
-    # |      20 | 1d 15h 26m |      8d 3h 46m |
-    # |      21 |   2d 1h 1m |     10d 9h 47m |
-    # |      22 |  2d 12h 4m |     13d 2h 51m |
-    # |      23 |  3d 0h 44m |     16d 8h 35m |
-    # |      24 |  3d 15h 9m |     20d 4h 45m |
-    # |      25 |  4d 7h 30m |     24d 17h 16 |
-    # |      26 |  5d 1h 56m |     30d 0h 12m |
-    # |      27 | 6d 22h 37m |     36d 3h 49m |
-    # |      28 | 7d 21h 44m |     43d 6h 34m |
-    # |      29 | 8d 23h 28m |     51d 11h 2m |
-    # |      30 |   9d 4h 0m |     61d 20h 2m |
+    # |      01 |     5.000s |         5.000s |
+    # |      02 |     6.000s |        11.000s |
+    # |      03 |    21.000s |        32.000s |
+    # |      04 |     1m 26s |         1m 58s |
+    # |      05 |     4m 21s |         6m 19s |
+    # |      06 |    10m 30s |        16m 49s |
+    # |      07 |    21m 41s |        38m 30s |
+    # |      08 |     40m 6s |         1h 18m |
+    # |      09 |      1h 8m |         2h 26m |
+    # |      10 |     1h 49m |         4h 16m |
+    # |      11 |     2h 46m |          7h 3m |
+    # |      12 |      4h 4m |         11h 7m |
+    # |      13 |     5h 45m |        16h 52m |
+    # |      14 |     7h 56m |      1d 0h 49m |
+    # |      15 |    10h 40m |     1d 11h 29m |
+    # |      16 |     14h 3m |      2d 1h 33m |
+    # |      17 |    18h 12m |     2d 19h 45m |
+    # |      18 |    23h 12m |     3d 18h 57m |
+    # |      19 |   1d 5h 9m |       5d 0h 7m |
+    # |      20 | 1d 12h 12m |     6d 12h 19m |
+    # |      21 | 1d 20h 26m |      8d 8h 46m |
+    # |      22 |   2d 6h 1m |    10d 14h 47m |
+    # |      23 |  2d 17h 4m |     13d 7h 51m |
+    # |      24 |  3d 5h 44m |    16d 13h 36m |
+    # |      25 |  3d 20h 9m |     20d 9h 45m |
+    # |      26 | 4d 12h 30m |    24d 22h 16m |
+    # |      27 |  5d 6h 56m |     30d 5h 12m |
+    # |      28 |  6d 3h 37m |     36d 8h 50m |
+    # |      29 |  7d 2h 44m |    43d 11h 34m |
+    # |      30 |  8d 4h 28m |     51d 16h 2m |
     # |---------|------------|----------------|
     module Retry
       extend ActiveSupport::Concern
@@ -70,7 +71,7 @@ module RocketJob
         after_fail :rocket_job_retry
 
         # Maximum number of times to retry this job.
-        # The default of 25 is almost 25 days of retries.
+        # The default of 25 retries spans about 20 days, see the table above.
         field :retry_limit, type: Integer, default: 25, class_attribute: true, user_editable: true, copy_on_restart: true
 
         # List of times when this job failed
@@ -115,7 +116,7 @@ module RocketJob
       #
       # For example, to see the durations for the first 25 retries:
       #   count = 25
-      #   intervals = (1..count).map { |attempt| attempt**4 + 5 }
+      #   intervals = (0...count).map { |failures| (failures**4) + 5 }
       #
       # Display the above intervals as human readable durations:
       #   intervals.map { |seconds| RocketJob.seconds_as_duration(seconds) }
@@ -124,9 +125,9 @@ module RocketJob
       #   RocketJob.seconds_as_duration(intervals.sum)
       #
       # Or, to see the total durations based on the number of retries:
-      #   (0..count).map{|i| "#{i+1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}"}
+      #   (0...count).map { |i| "#{i + 1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}" }
       def rocket_job_retry_seconds_to_delay
-        rocket_job_failure_count**4 + 5
+        (rocket_job_failure_count**4) + 5
       end
     end
   end

data/lib/rocket_job/plugins/throttle_dependent_jobs.rb

@@ -13,8 +13,13 @@ module RocketJob
       included do
         field :dependent_jobs, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
 
-        define_throttle :dependent_jobs_running?
-        define_batch_throttle :dependent_jobs_running? if respond_to?(:define_batch_throttle)
+        dependent_jobs_description = lambda { |job, *|
+          "Throttled: dependent jobs are running (#{Array(job.dependent_jobs).join(', ')})"
+        }
+        define_throttle :dependent_jobs_running?, description: dependent_jobs_description
+        if respond_to?(:define_batch_throttle)
+          define_batch_throttle :dependent_jobs_running?, description: dependent_jobs_description
+        end
       end
 
       class_methods do

data/lib/rocket_job/plugins/transaction.rb

@@ -41,8 +41,8 @@ module RocketJob
 
       private
 
-      def rocket_job_transaction(&block)
-        ActiveRecord::Base.transaction(&block)
+      def rocket_job_transaction(&)
+        ActiveRecord::Base.transaction(&)
       end
     end
   end

data/lib/rocket_job/sliced/input.rb

@@ -1,10 +1,10 @@
 module RocketJob
   module Sliced
     class Input < Slices
-      def upload(**args, &block)
+      def upload(**args, &)
         # Create indexes before uploading
         create_indexes
-        Writer::Input.collect(self, **args, &block)
+        Writer::Input.collect(self, **args, &)
       rescue Exception => e
         drop
         raise(e)
@@ -117,19 +117,24 @@ module RocketJob
         )
       end
 
-      # Returns the next slice to work on in id order
-      # Returns nil if there are currently no queued slices
+      # Claims and returns the next queued slice for this worker.
+      # Returns nil if there are currently no queued slices.
       #
-      # If a slice is in queued state it will be started and assigned to this worker
+      # No explicit sort is applied. Forcing the global minimum `_id` (a
+      # `sort("_id" => 1)`) makes every worker target the same document, so under
+      # concurrency they collide on the atomic claim: one wins and the rest hit a
+      # WriteConflict ("Document no longer matches the predicate") and retry,
+      # which throttles a large batch job as workers are added. Without the sort,
+      # the `{state: 1, _id: 1}` index still yields queued slices in roughly `_id`
+      # (upload) order, but concurrent workers land on different documents instead
+      # of contending for one. Output ordering is unaffected: each output slice
+      # inherits its input slice's `_id` and downloads read in `_id` order.
       def next_slice(worker_name)
-        # TODO: Will it perform faster without the id sort?
-        # I.e. Just process on a FIFO basis?
         document                 = all.queued.
-          sort("_id" => 1).
-          find_one_and_update(
-            {"$set" => {worker_name: worker_name, state: "running", started_at: Time.now}},
-            return_document: :after
-          )
+                                   find_one_and_update(
+                                     {"$set" => {worker_name: worker_name, state: "running", started_at: Time.now}},
+                                     return_document: :after
+                                   )
         document.collection_name = collection_name if document
         document
       end

data/lib/rocket_job/sliced/slice.rb

@@ -154,8 +154,8 @@ module RocketJob
       end
 
       # Fail this slice if an exception occurs during processing.
-      def fail_on_exception!(re_raise_exceptions = false, &block)
-        SemanticLogger.named_tagged(slice: id.to_s, &block)
+      def fail_on_exception!(re_raise_exceptions = false, &)
+        SemanticLogger.named_tagged(slice: id.to_s, &)
       rescue Exception => e
         SemanticLogger.named_tagged(slice: id.to_s) do
           if failed? || !may_fail?
@@ -175,7 +175,7 @@ module RocketJob
 
       # Always add records to any updates.
       def atomic_updates(*args)
-        r                             = super(*args)
+        r = super
         (r["$set"] ||= {})["records"] = serialize_records if @records
         r
       end

data/lib/rocket_job/sliced/slices.rb

@@ -44,8 +44,8 @@ module RocketJob
 
       # Returns output slices in the order of their id
       # which is usually the order in which they were written.
-      def each(&block)
-        all.sort(id: 1).each(&block)
+      def each(&)
+        all.sort(id: 1).each(&)
       end
 
       # Insert a new slice into the collection
@@ -120,7 +120,7 @@ module RocketJob
       end
 
       # Forward additional methods.
-      def_instance_delegators :@all, :collection, :count, :delete_all, :first, :find, :last, :nor, :not, :or, :to_a, :where
+      def_instance_delegators :@all, :collection, :count, :delete_all, :find, :nor, :not, :or, :to_a, :where
 
       # Drop this collection when it is no longer needed
       def drop
@@ -145,7 +145,6 @@ module RocketJob
       end
 
       # Mongoid does not apply ordering, add sort
-      # rubocop:disable Style/RedundantSort
       def first
         all.sort("_id" => 1).first
       end
@@ -154,8 +153,6 @@ module RocketJob
         all.sort("_id" => -1).first
       end
 
-      # rubocop:enable Style/RedundantSort
-
       # Returns [Array<Struct>] grouped exceptions by class name,
       # and unique exception messages by exception class.
       #

data/lib/rocket_job/sliced/writer/input.rb

@@ -9,8 +9,9 @@ module RocketJob
         #
         # Parameters
         #   on_first: [Proc]
-        #     Block to call on the first line only, instead of storing in the slice.
-        #     Useful for extracting the header row
+        #     Called with each line before the first that is stored, instead of storing it, until it returns true.
+        #     Useful for extracting the header row, after any blank lines before it, see
+        #     Category::Input#extract_header_callback.
         #     Default: nil
         #
         #   slice_size: [Integer]
@@ -42,8 +43,7 @@ module RocketJob
 
         def <<(line)
           if @on_first
-            @on_first.call(line)
-            @on_first = nil
+            @on_first = nil if @on_first.call(line)
             return self
           end
           @slice << line

data/lib/rocket_job/supervisor.rb

@@ -92,8 +92,8 @@ module RocketJob
       end
     end
 
-    def synchronize(&block)
-      @mutex.synchronize(&block)
+    def synchronize(&)
+      @mutex.synchronize(&)
     end
   end
 end

data/lib/rocket_job/thread_worker.rb

@@ -8,7 +8,7 @@ module RocketJob
     attr_reader :thread
 
     def initialize(id:, server_name:)
-      super(id: id, server_name: server_name)
+      super
       @shutdown = Concurrent::Event.new
       @thread   = Thread.new { run }
     end
@@ -21,8 +21,8 @@ module RocketJob
       @thread.backtrace
     end
 
-    def join(*args)
-      @thread.join(*args)
+    def join(*)
+      @thread.join(*)
     end
 
     # Send each active worker the RocketJob::ShutdownException so that stops processing immediately.

data/lib/rocket_job/throttle_definition.rb

@@ -1,10 +1,18 @@
 module RocketJob
   class ThrottleDefinition
-    attr_reader :method_name, :filter
+    attr_reader :method_name, :filter, :description
 
-    def initialize(method_name, filter)
+    # Parameters:
+    #   description: [String|Proc|nil]
+    #     Human readable reason why the job is throttled, persisted to the job as
+    #     `throttled_by` and surfaced in Mission Control.
+    #     When a Proc, it is called with the same arguments as the throttle and must
+    #     return a String, allowing the reason to include runtime detail.
+    #     When nil, a humanized version of the method name is used.
+    def initialize(method_name, filter, description = nil)
       @method_name = method_name.to_sym
       @filter      = filter
+      @description = description
     end
 
     # Returns [true|false] whether the throttle was triggered.
@@ -33,5 +41,14 @@ module RocketJob
         job.send(filter)
       end
     end
+
+    # Returns [String] the human readable reason why the job is throttled.
+    def extract_description(job, *)
+      return description.call(job, *) if description.is_a?(Proc)
+      return description if description
+
+      # Default: humanize the method name, dropping a trailing `?` or `_exceeded`.
+      method_name.to_s.sub(/_exceeded\?\z/, "").sub(/\?\z/, "").tr("_", " ").capitalize
+    end
   end
 end

data/lib/rocket_job/throttle_definitions.rb

@@ -6,13 +6,13 @@ module RocketJob
       @throttles = []
     end
 
-    def add(method_name, filter)
+    def add(method_name, filter, description = nil)
       unless filter.is_a?(Symbol) || filter.is_a?(Proc)
         raise(ArgumentError, "Filter for #{method_name} must be a Symbol or Proc")
       end
       raise(ArgumentError, "Cannot define #{method_name} twice, undefine previous throttle first") if exist?(method_name)
 
-      @throttles += [ThrottleDefinition.new(method_name, filter)]
+      @throttles += [ThrottleDefinition.new(method_name, filter, description)]
     end
 
     # Undefine a previously defined throttle
@@ -25,15 +25,20 @@ module RocketJob
       throttles.any? { |throttle| throttle.method_name == method_name }
     end
 
-    # Returns the matching filter,
+    # Returns [ThrottleDefinition] the first throttle that was triggered,
     # or nil if no throttles were triggered.
-    def matching_filter(job, *args)
-      throttles.each do |throttle|
-        next unless throttle.throttled?(job, *args)
+    #
+    # The returned definition exposes both the filter (`extract_filter`) and the
+    # human readable reason (`extract_description`) so callers can apply the filter
+    # and persist why the job was throttled.
+    def matching_throttle(job, *args)
+      throttles.find { |throttle| throttle.throttled?(job, *args) }
+    end
 
-        return throttle.extract_filter(job, *args)
-      end
-      nil
+    # Returns the matching filter,
+    # or nil if no throttles were triggered.
+    def matching_filter(job, *)
+      matching_throttle(job, *)&.extract_filter(job, *)
     end
 
     def deep_dup

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "6.3.2".freeze
+  VERSION = "7.1.0".freeze
 end