rocketjob 5.4.1 → 6.0.0

data/lib/rocket_job/lookup_collection.rb

@@ -0,0 +1,69 @@
+module RocketJob
+  class LookupCollection < Mongo::Collection
+    # Rapidly upload individual records in batches.
+    #
+    # Operates directly on a Mongo Collection to avoid the overhead of creating Mongoid objects
+    # for each and every row.
+    #
+    # Example:
+    #   lookup_collection(:my_lookup).upload do |io|
+    #     io << {id: 123, data: "first record"}
+    #     io << {id: 124, data: "second record"}
+    #   end
+    #
+    #   input_category(:my_lookup).find(id: 123).first
+    def upload(batch_size: 10_000, &block)
+      BatchUploader.upload(batch_size: batch_size, &block)
+    end
+
+    # Looks up the value at the specified id.
+    # Returns [nil] if no record was found with the supplied id.
+    def lookup(id)
+      find(id: id).first
+    end
+
+    # Internal class for uploading records in batches
+    class BatchUploader
+      attr_reader :record_count
+
+      def self.upload(collection, **args)
+        writer = new(collection, **args)
+        yield(writer)
+        writer.record_count
+      ensure
+        writer&.close
+      end
+
+      def initialize(collection, batch_size:)
+        @batch_size   = batch_size
+        @record_count = 0
+        @batch_count  = 0
+        @documents    = []
+        @collection   = collection
+      end
+
+      def <<(record)
+        raise(ArgumentError, "Record must be a Hash") unless record.is_a?(Hash)
+
+        unless record.key?(:id) || record.key?("id") || record.key?("_id")
+          raise(ArgumentError, "Record must include an :id key")
+        end
+
+        @documents << record
+        @record_count += 1
+        @batch_count  += 1
+        if @batch_count >= @batch_size
+          @collection.insert_many(@documents)
+          @documents.clear
+          @batch_count = 0
+        end
+
+        self
+      end
+
+      def close
+        @collection.insert_many(@documents) unless @documents.empty?
+      end
+    end
+  end
+end

data/lib/rocket_job/plugins/cron.rb

@@ -14,41 +14,81 @@ module RocketJob
       extend ActiveSupport::Concern
 
       included do
-        include Restart
-
         field :cron_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
 
+        # Whether to prevent another instance of this job from running with the exact _same_ cron schedule.
+        # Another job instance with a different `cron_schedule` string is permitted.
+        field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+
+        # Whether to re-schedule the next job occurrence when this job starts, or when it is complete.
+        #
+        # `true`: Create a new scheduled instance of this job after it has started. (Default)
+        #   - Ensures that the next scheduled instance is not missed because the current instance is still running.
+        #   - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
+        #     _only_ if they were changed during an `after_start` callback.
+        #     Changes to these during other callbacks or during the `perform` will not be saved to the new scheduled
+        #     instance.
+        #   - To prevent this job creating any new duplicate instances during subsequent processing,
+        #     its `cron_schedule` is set to `nil`.
+        #
+        # `false`: Create a new scheduled instance of this job on `fail`, or `abort`.
+        #   - Prevents the next scheduled instance from running or being scheduled while the current instance is
+        #     still running.
+        #   - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
+        #     at any time until after the job has failed, or is aborted.
+        #   - To prevent this job creating any new duplicate instances during subsequent processing,
+        #     its `cron_schedule` is set to `nil` after it fails or is aborted.
+        field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+
         validates_each :cron_schedule do |record, attr, value|
           record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
         end
+        validate :rocket_job_cron_singleton_check
+
         before_save :rocket_job_cron_set_run_at
 
-        private
+        after_start :rocket_job_cron_on_start
+        after_abort :rocket_job_cron_end_state
+        after_complete :rocket_job_cron_end_state
+        after_fail :rocket_job_cron_end_state
+      end
 
-        # Prevent auto restart if this job does not have a cron schedule.
-        # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_new_instance
-        def rocket_job_restart_new_instance
-          return unless cron_schedule
+      def rocket_job_cron_set_run_at
+        return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
 
-          super
-        end
+        self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
+      end
 
-        # On failure:
-        # - create a new instance scheduled to run in the future.
-        # - clear out the `cron_schedule` so this instance will not schedule another instance to run on completion.
-        # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_abort
-        def rocket_job_restart_abort
-          return unless cron_schedule
+      private
 
-          rocket_job_restart_new_instance
-          update_attribute(:cron_schedule, nil)
+      def rocket_job_cron_on_start
+        return unless cron_schedule && cron_after_start
+
+        current_cron_schedule = cron_schedule
+        update_attribute(:cron_schedule, nil)
+        create_restart!(cron_schedule: current_cron_schedule)
+      end
+
+      def rocket_job_cron_end_state
+        return unless cron_schedule && !cron_after_start
+
+        current_cron_schedule = cron_schedule
+        update_attribute(:cron_schedule, nil)
+        create_restart!(cron_schedule: current_cron_schedule)
+      end
+
+      # Returns [true|false] whether another instance of this job with the same cron schedule is already active
+      def rocket_job_cron_duplicate?
+        self.class.with(read: {mode: :primary}) do |conn|
+          conn.where(:state.in => %i[queued running failed paused], :id.ne => id, cron_schedule: cron_schedule).exists?
         end
       end
 
-      def rocket_job_cron_set_run_at
-        return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
+      # Prevent creation of a new job when another is running with the same cron schedule.
+      def rocket_job_cron_singleton_check
+        return if cron_schedule.nil? || completed? || aborted? || !rocket_job_cron_duplicate?
 
-        self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
+        errors.add(:state, "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
       end
     end
   end

data/lib/rocket_job/plugins/job/model.rb

@@ -37,12 +37,10 @@ module RocketJob
           # arrives, then the current job will complete the current slices and process
           # the new higher priority job
           field :priority, type: Integer, default: 50, class_attribute: true, user_editable: true, copy_on_restart: true
+          validates_inclusion_of :priority, in: 1..100
 
           # When the job completes destroy it from both the database and the UI
-          field :destroy_on_complete, type: Boolean, default: true, class_attribute: true, copy_on_restart: true
-
-          # Whether to store the results from this job
-          field :collect_output, type: Boolean, default: false, class_attribute: true
+          field :destroy_on_complete, type: Mongoid::Boolean, default: true, class_attribute: true, copy_on_restart: true
 
           # Run this job no earlier than this time
           field :run_at, type: Time, user_editable: true
@@ -54,14 +52,15 @@ module RocketJob
           # Can be used to reduce log noise, especially during high volume calls
           # For debugging a single job can be logged at a low level such as :trace
           #   Levels supported: :trace, :debug, :info, :warn, :error, :fatal
-          field :log_level, type: Symbol, class_attribute: true, user_editable: true, copy_on_restart: true
+          field :log_level, type: Mongoid::StringifiedSymbol, class_attribute: true, user_editable: true, copy_on_restart: true
+          validates_inclusion_of :log_level, in: SemanticLogger::LEVELS + [nil]
 
           #
           # Read-only attributes
           #
 
           # Current state, as set by the state machine. Do not modify this value directly.
-          field :state, type: Symbol, default: :queued
+          field :state, type: Mongoid::StringifiedSymbol, default: :queued
 
           # When the job was created
           field :created_at, type: Time, default: -> { Time.now }
@@ -89,17 +88,12 @@ module RocketJob
           # Store the last exception for this job
           embeds_one :exception, class_name: "RocketJob::JobException"
 
-          # Store the Hash result from this job if collect_output is true,
-          # and the job returned actually returned a Hash, otherwise nil
-          # Not applicable to SlicedJob jobs, since its output is stored in a
-          # separate collection
-          field :result, type: Hash
-
+          # Used when workers fetch jobs to work on.
           index({state: 1, priority: 1, _id: 1}, background: true)
+          # Used by Mission Control to display completed jobs sorted by completion.
+          index({completed_at: 1}, background: true)
 
           validates_presence_of :state, :failure_count, :created_at
-          validates :priority, inclusion: 1..100
-          validates :log_level, inclusion: SemanticLogger::LEVELS + [nil]
         end
 
         module ClassMethods
@@ -155,14 +149,8 @@ module RocketJob
 
           # Scope for queued jobs that can run now
           # I.e. Queued jobs excluding scheduled jobs
-          if Mongoid::VERSION.to_f >= 7.1
-            def queued_now
-              queued.and(RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now))
-            end
-          else
-            def queued_now
-              queued.or({run_at: nil}, :run_at.lte => Time.now)
-            end
+          def queued_now
+            queued.and(RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now))
           end
 
           # Defines all the fields that are accessible on the Document
@@ -183,43 +171,30 @@ module RocketJob
           #
           # @return [ Field ] The generated field
           def field(name, options)
-            if options.delete(:user_editable) == true
-              self.user_editable_fields += [name.to_sym] unless user_editable_fields.include?(name.to_sym)
+            if (options.delete(:user_editable) == true) && !user_editable_fields.include?(name.to_sym)
+              self.user_editable_fields += [name.to_sym]
             end
+
             if options.delete(:class_attribute) == true
               class_attribute(name, instance_accessor: false)
               public_send("#{name}=", options[:default]) if options.key?(:default)
               options[:default] = -> { self.class.public_send(name) }
             end
-            if options.delete(:copy_on_restart) == true
-              self.rocket_job_restart_attributes += [name.to_sym] unless rocket_job_restart_attributes.include?(name.to_sym)
+
+            if (options.delete(:copy_on_restart) == true) && !rocket_job_restart_attributes.include?(name.to_sym)
+              self.rocket_job_restart_attributes += [name.to_sym]
             end
-            super(name, options)
-          end
 
-          # DEPRECATED
-          def rocket_job
-            warn "Replace calls to .rocket_job with calls to set class instance variables. For example: self.priority = 50"
-            yield(self)
+            super(name, options)
           end
 
-          # DEPRECATED
-          def public_rocket_job_properties(*args)
-            warn "Replace calls to .public_rocket_job_properties by adding `user_editable: true` option to the field declaration in #{name} for: #{args.inspect}"
-            self.user_editable_fields += args.collect(&:to_sym)
+          # Builds this job instance from the supplied properties hash.
+          # Overridden by batch to support child objects.
+          def from_properties(properties)
+            new(properties)
           end
         end
 
-        # Returns [true|false] whether to collect nil results from running this batch
-        def collect_nil_output?
-          collect_output? ? (collect_nil_output == true) : false
-        end
-
-        # Returns [true|false] whether to collect the results from running this batch
-        def collect_output?
-          collect_output == true
-        end
-
         # Returns [Float] the number of seconds the job has taken
         # - Elapsed seconds to process the job from when a worker first started working on it
         #   until now if still running, or until it was completed
@@ -282,7 +257,6 @@ module RocketJob
         # Returns [Hash] status of this job
         def as_json
           attrs = serializable_hash(methods: %i[seconds duration])
-          attrs.delete("result") unless collect_output?
           attrs.delete("failure_count") unless failure_count.positive?
           if queued?
             attrs.delete("started_at")
@@ -319,16 +293,17 @@ module RocketJob
           h = as_json
           h.delete("seconds")
           h.dup.each_pair do |k, v|
-            if v.is_a?(Time)
+            case v
+            when Time
               h[k] = v.in_time_zone(time_zone).to_s
-            elsif v.is_a?(BSON::ObjectId)
+            when BSON::ObjectId
               h[k] = v.to_s
             end
           end
           h
         end
 
-        # Returns [Boolean] whether the worker runs on a particular server.
+        # Returns [true|false] whether the worker runs on a particular server.
         def worker_on_server?(server_name)
           return false unless worker_name.present? && server_name.present?
 

data/lib/rocket_job/plugins/job/persistence.rb

@@ -70,6 +70,29 @@ module RocketJob
           end
         end
 
+        # Create a new instance of this job, copying across only the `copy_on_restart` attributes.
+        # Copy across input and output categories to new scheduled job so that all of the
+        # settings are remembered between instance. Example: slice_size
+        def create_restart!(**overrides)
+          if expired?
+            logger.info("Job has expired. Not creating a new instance.")
+            return
+          end
+
+          job_attrs = self.class.rocket_job_restart_attributes.each_with_object({}) do |attr, attrs|
+            attrs[attr] = send(attr)
+          end
+          job_attrs.merge!(overrides)
+
+          job                   = self.class.new(job_attrs)
+          job.input_categories  = input_categories if respond_to?(:input_categories)
+          job.output_categories = output_categories if respond_to?(:output_categories)
+
+          job.save_with_retry!
+
+          logger.info("Created a new job instance: #{job.id}")
+        end
+
         # Set in-memory job to complete if `destroy_on_complete` and the job has been destroyed
         def reload
           return super unless destroy_on_complete
@@ -85,6 +108,19 @@ module RocketJob
             self
           end
         end
+
+        # Save with retry in case persistence takes a moment.
+        def save_with_retry!(retry_limit = 10, sleep_interval = 0.5)
+          count = 0
+          while count < retry_limit
+            return true if save
+
+            logger.info("Retrying to persist new scheduled instance: #{errors.messages.inspect}")
+            sleep(sleep_interval)
+            count += 1
+          end
+          save!
+        end
       end
     end
   end

data/lib/rocket_job/plugins/job/throttle.rb

@@ -48,7 +48,7 @@ module RocketJob
           # Note: Throttles are executed in the order they are defined.
           def define_throttle(method_name, filter: :throttle_filter_class)
             # Duplicate to prevent modifying parent class throttles
-            definitions = rocket_job_throttles ? rocket_job_throttles.dup : ThrottleDefinitions.new
+            definitions = rocket_job_throttles ? rocket_job_throttles.deep_dup : ThrottleDefinitions.new
             definitions.add(method_name, filter)
             self.rocket_job_throttles = definitions
           end
@@ -57,7 +57,7 @@ module RocketJob
           def undefine_throttle(method_name)
             return unless rocket_job_throttles
 
-            definitions = rocket_job_throttles.dup
+            definitions = rocket_job_throttles.deep_dup
             definitions.remove(method_name)
             self.rocket_job_throttles = definitions
           end

data/lib/rocket_job/plugins/job/throttle_running_jobs.rb

@@ -37,7 +37,7 @@ module RocketJob
 
         private
 
-        # Returns [Boolean] whether the throttle for this job has been exceeded
+        # Returns [true|false] whether the throttle for this job has been exceeded
         def throttle_running_jobs_exceeded?
           return false unless throttle_running_jobs&.positive?
 

data/lib/rocket_job/plugins/job/worker.rb

@@ -48,11 +48,11 @@ module RocketJob
         def perform_now
           raise(::Mongoid::Errors::Validations, self) unless valid?
 
-          worker = RocketJob::Worker.new(inline: true)
+          worker = RocketJob::Worker.new
           start if may_start?
           # Re-Raise exceptions
           rocket_job_work(worker, true) if running?
-          result
+          @rocket_job_output
         end
 
         def perform(*)
@@ -106,11 +106,6 @@ module RocketJob
               end
             end
 
-            if collect_output?
-              # Result must be a Hash, if not put it in a Hash
-              self.result = @rocket_job_output.is_a?(Hash) ? @rocket_job_output : {"result" => @rocket_job_output}
-            end
-
             if new_record? || destroyed?
               complete if may_complete?
             else

data/lib/rocket_job/plugins/restart.rb

@@ -2,121 +2,21 @@ require "active_support/concern"
 
 module RocketJob
   module Plugins
-    # Automatically starts a new instance of this job anytime it fails, aborts, or completes.
-    #
-    # Notes:
-    # * Restartable jobs automatically abort if they fail. This prevents the failed job from being retried.
-    #   - To disable this behavior, add the following empty method:
-    #        def rocket_job_restart_abort
-    #        end
-    # * On destroy this job is destroyed without starting a new instance.
-    # * On Abort a new instance is created.
-    # * Include `RocketJob::Plugins::Singleton` to prevent multiple copies of a job from running at
-    #   the same time.
-    # * The job will not be restarted if:
-    #   - A validation fails after creating the new instance of this job.
-    #   - The job has expired.
-    # * Only the fields that have `copy_on_restart: true` will be passed onto the new instance of this job.
-    #
-    # Example:
-    #
-    # class RestartableJob < RocketJob::Job
-    #   include RocketJob::Plugins::Restart
-    #
-    #   # Retain the completed job under the completed tab in Rocket Job Web Interface.
-    #   self.destroy_on_complete = false
-    #
-    #   # Will be copied to the new job on restart.
-    #   field :limit, type: Integer, copy_on_restart: true
-    #
-    #   # Will _not_ be copied to the new job on restart.
-    #   field :list, type: Array, default: [1,2,3]
-    #
-    #   # Set run_at every time a new instance of the job is created.
-    #   after_initialize set_run_at, if: :new_record?
-    #
-    #   def perform
-    #     puts "The limit is #{limit}"
-    #     puts "The list is #{list}"
-    #     'DONE'
-    #   end
-    #
-    #   private
-    #
-    #   # Run this job in 30 minutes.
-    #   def set_run_at
-    #     self.run_at = 30.minutes.from_now
-    #   end
-    # end
-    #
-    # job = RestartableJob.create!(limit: 10, list: [4,5,6])
-    # job.reload.state
-    # # => :queued
-    #
-    # job.limit
-    # # => 10
-    #
-    # job.list
-    # # => [4,5,6]
-    #
-    # # Wait 30 minutes ...
-    #
-    # job.reload.state
-    # # => :completed
-    #
-    # # A new instance was automatically created.
-    # job2 = RestartableJob.last
-    # job2.state
-    # # => :queued
-    #
-    # job2.limit
-    # # => 10
-    #
-    # job2.list
-    # # => [1,2,3]
+    # @deprecated
     module Restart
       extend ActiveSupport::Concern
 
       included do
-        after_abort :rocket_job_restart_new_instance
-        after_complete :rocket_job_restart_new_instance
+        after_abort :create_restart!
+        after_complete :create_restart!
         after_fail :rocket_job_restart_abort
       end
 
       private
 
-      # Run again in the future, even if this run fails with an exception
-      def rocket_job_restart_new_instance
-        if expired?
-          logger.info("Job has expired. Not creating a new instance.")
-          return
-        end
-        attributes = rocket_job_restart_attributes.each_with_object({}) { |attr, attrs| attrs[attr] = send(attr) }
-        rocket_job_restart_create(attributes)
-      end
-
       def rocket_job_restart_abort
         new_record? ? abort : abort!
       end
-
-      # Allow Singleton to prevent the creation of a new job if one is already running
-      # Retry since the delete may not have persisted to disk yet.
-      def rocket_job_restart_create(attrs, retry_limit = 3, sleep_interval = 0.1)
-        count = 0
-        while count < retry_limit
-          job = self.class.create(attrs)
-          if job.persisted?
-            logger.info("Created a new job instance: #{job.id}")
-            return true
-          else
-            logger.info("Job already active, retrying after a short sleep")
-            sleep(sleep_interval)
-          end
-          count += 1
-        end
-        logger.error("New job instance not started: #{job.errors.messages.inspect}")
-        false
-      end
     end
   end
 end