rocketjob 5.3.3 → 6.0.0.rc1

data/lib/rocket_job/extensions/psych/yaml_tree.rb

@@ -0,0 +1,8 @@
+require "psych/visitors/yaml_tree"
+
+class Psych::Visitors::YAMLTree
+  # Serialize IOStream path as a string
+  def visit_IOStreams_Path(o)
+    visit_String(o.to_s)
+  end
+end

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -18,7 +18,7 @@ module RocketJob
     #   file name of the archived file is passed into the job as either
     #   `upload_file_name` or `full_file_name`.
 
-    # Note:
+    # Notes:
     # - Jobs that do not implement #upload _must_ have either `upload_file_name` or `full_file_name` as an attribute.
     #
     # With RocketJob Pro, the file is automatically uploaded into the job itself

data/lib/rocket_job/jobs/housekeeping_job.rb

@@ -35,7 +35,7 @@ module RocketJob
       self.cron_schedule = "*/15 * * * * UTC"
 
       # Whether to destroy zombie servers automatically
-      field :destroy_zombies, type: Boolean, default: true, user_editable: true, copy_on_restart: true
+      field :destroy_zombies, type: Mongoid::Boolean, default: true, user_editable: true, copy_on_restart: true
 
       # Retention intervals in seconds.
       # Set to nil to retain everything.
@@ -54,12 +54,12 @@ module RocketJob
         RocketJob::Job.paused.where(completed_at: {"$lte" => paused_retention.seconds.ago}).destroy_all if paused_retention
         RocketJob::Job.queued.where(created_at: {"$lte" => queued_retention.seconds.ago}).destroy_all if queued_retention
 
-        if destroy_zombies
-          # Cleanup zombie servers
-          RocketJob::Server.destroy_zombies
-          # Requeue jobs where the worker is in the zombie state and its server has gone away
-          RocketJob::ActiveWorker.requeue_zombies
-        end
+        return unless destroy_zombies
+
+        # Cleanup zombie servers
+        RocketJob::Server.destroy_zombies
+        # Requeue jobs where the worker is in the zombie state and its server has gone away
+        RocketJob::ActiveWorker.requeue_zombies
       end
     end
   end

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -31,16 +31,17 @@
 #   job.perform_now
 #   job.cleanup!
 #
-# By default output is not collected, add the option `collect_output: true` to collect output.
+# By default output is not collected, call the method `#collect_output` to collect output.
 #
 # Example:
 #   job = RocketJob::Jobs::OnDemandBatchJob(
 #     description:              'Fix data',
 #     code:                     code,
 #     throttle_running_workers: 5,
-#     priority:                 30,
-#     collect_output:           true
+#     priority:                 30
 #   )
+#   job.collect_output
+#   job.save!
 #
 # Example: Move the upload operation into a before_batch.
 #   upload_code = <<-CODE
@@ -95,10 +96,18 @@ module RocketJob
       before_batch :run_before_code
       after_batch :run_after_code
 
+      # Make this job collect its output
+      # :nils [true|false]
+      #   Whether to skip the output from `code` when it is nil
+      #   Default: false
+      def collect_output(nils: false)
+        self.output_categories = [RocketJob::Category::Output.new(nils: nils)]
+      end
+
       private
 
       def load_perform_code
-        instance_eval("def perform(row)\n#{code}\nend")
+        instance_eval("def perform(row)\n#{code}\nend", __FILE__, __LINE__)
       end
 
       def run_before_code
@@ -118,13 +127,13 @@ module RocketJob
       def validate_before_code
         return if before_code.nil?
 
-        validate_field(:before_code) { instance_eval("def __before_code\n#{before_code}\nend") }
+        validate_field(:before_code) { instance_eval("def __before_code\n#{before_code}\nend", __FILE__, __LINE__) }
       end
 
       def validate_after_code
         return if after_code.nil?
 
-        validate_field(:after_code) { instance_eval("def __after_code\n#{after_code}\nend") }
+        validate_field(:after_code) { instance_eval("def __after_code\n#{after_code}\nend", __FILE__, __LINE__) }
       end
 
       def validate_field(field)

data/lib/rocket_job/jobs/on_demand_job.rb

@@ -38,12 +38,11 @@
 #
 # Example: Retain output:
 #   code = <<~CODE
-#     {'value' => data['a'] * data['b']}
+#     data['result'] = data['a'] * data['b']
 #   CODE
 #
 #   RocketJob::Jobs::OnDemandJob.create!(
 #     code:           code,
-#     collect_output: true,
 #     data:           {'a' => 10, 'b' => 2}
 #   )
 #

data/lib/rocket_job/jobs/performance_job.rb

@@ -6,9 +6,11 @@ module RocketJob
       # Define the job's default attributes
       self.description         = "Performance Test"
       self.priority            = 5
-      self.slice_size          = 100
       self.destroy_on_complete = false
 
+      input_category slice_size: 100
+      output_category
+
       # No operation, just return the supplied line (record)
       def perform(line)
         line

data/lib/rocket_job/jobs/re_encrypt/relational_job.rb

@@ -17,13 +17,13 @@ module RocketJob
       class RelationalJob < RocketJob::Job
         include RocketJob::Batch
 
-        self.slice_size               = 1000
         self.priority                 = 30
         self.destroy_on_complete      = false
-        self.compress                 = true
         self.throttle_running_jobs    = 1
         self.throttle_running_workers = 10
 
+        input_category slice_size: 1_000
+
         # Name of the table being re-encrypted
         field :table_name, type: String
 
@@ -73,7 +73,7 @@ module RocketJob
 
           # Use AR to fetch all the records
           self.class.connection.select_rows(sql).each do |row|
-            row     = row.unshift(nil)
+            row.unshift(nil)
             index   = 1
             sql     = "update #{quoted_table_name} set "
             updates = []
@@ -120,7 +120,8 @@ module RocketJob
         def upload_records
           start_id          = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
           last_id           = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
-          self.record_count = last_id.positive? ? (input.upload_integer_range_in_reverse_order(start_id, last_id) * slice_size) : 0
+          self.record_count =
+            last_id.positive? ? (input.upload_integer_range_in_reverse_order(start_id, last_id) * input_category.slice_size) : 0
         end
       end
     end

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -19,7 +19,7 @@ module RocketJob
       field :properties, type: Hash, default: {}, user_editable: true
 
       # File to upload
-      field :upload_file_name, type: String, user_editable: true
+      field :upload_file_name, type: IOStreams::Path, user_editable: true
 
       # The original Input file name.
       # Used by #upload to extract the IOStreams when present.
@@ -33,14 +33,15 @@ module RocketJob
       validate :job_is_a_rocket_job
       validate :job_implements_upload
       validate :file_exists
+      validate :job_has_properties
 
       # Create the job and upload the file into it.
       def perform
-        job    = job_class.new(properties)
+        job    = job_class.from_properties(properties)
         job.id = job_id if job_id
         upload_file(job)
         job.save!
-      rescue StandardError => e
+      rescue Exception => e
         # Prevent partial uploads
         job&.cleanup! if job.respond_to?(:cleanup!)
         raise(e)
@@ -66,7 +67,10 @@ module RocketJob
         elsif job.respond_to?(:full_file_name=)
           job.full_file_name = upload_file_name
         else
-          raise(ArgumentError, "Model #{job_class_name} must implement '#upload', or have attribute 'upload_file_name' or 'full_file_name'")
+          raise(
+            ArgumentError,
+            "Model #{job_class_name} must implement '#upload', or have attribute 'upload_file_name' or 'full_file_name'"
+          )
         end
       end
 
@@ -85,17 +89,50 @@ module RocketJob
         klass = job_class
         return if klass.nil? || klass.instance_methods.any? { |m| VALID_INSTANCE_METHODS.include?(m) }
 
-        errors.add(:job_class_name, "#{job_class} must implement any one of: :#{VALID_INSTANCE_METHODS.join(' :')} instance methods")
+        errors.add(:job_class_name,
+                   "#{job_class} must implement any one of: :#{VALID_INSTANCE_METHODS.join(' :')} instance methods")
       end
 
       def file_exists
-        return if upload_file_name.nil?
-
-        uri = URI.parse(upload_file_name)
-        return unless uri.scheme.nil? || uri.scheme == "file"
-        return if File.exist?(upload_file_name)
+        # Only check for file existence when it is a local file
+        return unless upload_file_name.is_a?(IOStreams::Paths::File)
+        if upload_file_name.to_s == ""
+          return errors.add(:upload_file_name, "Upload file name can't be blank.")
+        end
 
+        return if upload_file_name.exist?
         errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
+      rescue NotImplementedError
+        nil
+      end
+
+      def job_has_properties
+        klass = job_class
+        return unless klass
+
+        properties.each_pair do |k, _v|
+          next if klass.public_method_defined?("#{k}=".to_sym)
+
+          if %i[output_categories input_categories].include?(k)
+            category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
+            properties[k].each do |category|
+              category.each_pair do |key, _value|
+                next if category_class.public_method_defined?("#{key}=".to_sym)
+
+                errors.add(
+                  :properties,
+                  "Unknown Property in #{k}: Attempted to set a value for #{key}.#{k} which is not allowed on the job #{job_class_name}"
+                )
+              end
+            end
+            next
+          end
+
+          errors.add(
+            :properties,
+            "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}"
+          )
+        end
       end
     end
   end

data/lib/rocket_job/lookup_collection.rb

@@ -0,0 +1,68 @@
+module RocketJob
+  class LookupCollection < Mongo::Collection
+    # Rapidly upload individual records in batches.
+    #
+    # Operates directly on a Mongo Collection to avoid the overhead of creating Mongoid objects
+    # for each and every row.
+    #
+    # Example:
+    #   lookup_collection(:my_lookup).upload do |io|
+    #     io << {id: 123, data: "first record"}
+    #     io << {id: 124, data: "second record"}
+    #   end
+    #
+    #   input_category(:my_lookup).find(id: 123).first
+    def upload(batch_size: 10_000, &block)
+      BatchUploader.upload(batch_size: batch_size, &block)
+    end
+
+    # Looks up the value at the specified id.
+    # Returns [nil] if no record was found with the supplied id.
+    def lookup(id)
+      find(id: id).first
+    end
+
+    private
+
+    # Internal class for uploading records in batches
+    class BatchUploader
+      attr_reader :record_count
+
+      def self.upload(collection, **args)
+        writer = new(collection, **args)
+        yield(writer)
+        writer.record_count
+      ensure
+        writer&.close
+      end
+
+      def initialize(collection, batch_size:)
+        @batch_size   = batch_size
+        @record_count = 0
+        @batch_count  = 0
+        @documents    = []
+        @collection   = collection
+      end
+
+      def <<(record)
+        raise(ArgumentError, "Record must be a Hash") unless record.is_a?(Hash)
+        raise(ArgumentError, "Record must include an :id key") unless record.key?(:id) || record.key?("id") || record.key?("_id")
+
+        @documents << record
+        @record_count += 1
+        @batch_count  += 1
+        if @batch_count >= @batch_size
+          @collection.insert_many(@documents)
+          @documents.clear
+          @batch_count = 0
+        end
+
+        self
+      end
+
+      def close
+        @collection.insert_many(@documents) unless @documents.empty?
+      end
+    end
+  end
+end

data/lib/rocket_job/plugins/job/model.rb

@@ -37,12 +37,10 @@ module RocketJob
           # arrives, then the current job will complete the current slices and process
           # the new higher priority job
           field :priority, type: Integer, default: 50, class_attribute: true, user_editable: true, copy_on_restart: true
+          validates_inclusion_of :priority, in: 1..100
 
           # When the job completes destroy it from both the database and the UI
-          field :destroy_on_complete, type: Boolean, default: true, class_attribute: true, copy_on_restart: true
-
-          # Whether to store the results from this job
-          field :collect_output, type: Boolean, default: false, class_attribute: true
+          field :destroy_on_complete, type: Mongoid::Boolean, default: true, class_attribute: true, copy_on_restart: true
 
           # Run this job no earlier than this time
           field :run_at, type: Time, user_editable: true
@@ -54,14 +52,15 @@ module RocketJob
           # Can be used to reduce log noise, especially during high volume calls
           # For debugging a single job can be logged at a low level such as :trace
           #   Levels supported: :trace, :debug, :info, :warn, :error, :fatal
-          field :log_level, type: Symbol, class_attribute: true, user_editable: true, copy_on_restart: true
+          field :log_level, type: Mongoid::StringifiedSymbol, class_attribute: true, user_editable: true, copy_on_restart: true
+          validates_inclusion_of :log_level, in: SemanticLogger::LEVELS + [nil]
 
           #
           # Read-only attributes
           #
 
           # Current state, as set by the state machine. Do not modify this value directly.
-          field :state, type: Symbol, default: :queued
+          field :state, type: Mongoid::StringifiedSymbol, default: :queued
 
           # When the job was created
           field :created_at, type: Time, default: -> { Time.now }
@@ -89,17 +88,12 @@ module RocketJob
           # Store the last exception for this job
           embeds_one :exception, class_name: "RocketJob::JobException"
 
-          # Store the Hash result from this job if collect_output is true,
-          # and the job returned actually returned a Hash, otherwise nil
-          # Not applicable to SlicedJob jobs, since its output is stored in a
-          # separate collection
-          field :result, type: Hash
-
+          # Used when workers fetch jobs to work on.
           index({state: 1, priority: 1, _id: 1}, background: true)
+          # Used by Mission Control to display completed jobs sorted by completion.
+          index({completed_at: 1}, background: true)
 
           validates_presence_of :state, :failure_count, :created_at
-          validates :priority, inclusion: 1..100
-          validates :log_level, inclusion: SemanticLogger::LEVELS + [nil]
         end
 
         module ClassMethods
@@ -155,14 +149,8 @@ module RocketJob
 
           # Scope for queued jobs that can run now
           # I.e. Queued jobs excluding scheduled jobs
-          if Mongoid::VERSION.to_f >= 7.1
-            def queued_now
-              queued.and(RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now))
-            end
-          else
-            def queued_now
-              queued.or({run_at: nil}, :run_at.lte => Time.now)
-            end
+          def queued_now
+            queued.and(RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now))
           end
 
           # Defines all the fields that are accessible on the Document
@@ -183,43 +171,30 @@ module RocketJob
           #
           # @return [ Field ] The generated field
           def field(name, options)
-            if options.delete(:user_editable) == true
-              self.user_editable_fields += [name.to_sym] unless user_editable_fields.include?(name.to_sym)
+            if (options.delete(:user_editable) == true) && !user_editable_fields.include?(name.to_sym)
+              self.user_editable_fields += [name.to_sym]
             end
+
             if options.delete(:class_attribute) == true
               class_attribute(name, instance_accessor: false)
               public_send("#{name}=", options[:default]) if options.key?(:default)
               options[:default] = -> { self.class.public_send(name) }
             end
-            if options.delete(:copy_on_restart) == true
-              self.rocket_job_restart_attributes += [name.to_sym] unless rocket_job_restart_attributes.include?(name.to_sym)
+
+            if (options.delete(:copy_on_restart) == true) && !rocket_job_restart_attributes.include?(name.to_sym)
+              self.rocket_job_restart_attributes += [name.to_sym]
             end
-            super(name, options)
-          end
 
-          # DEPRECATED
-          def rocket_job
-            warn "Replace calls to .rocket_job with calls to set class instance variables. For example: self.priority = 50"
-            yield(self)
+            super(name, options)
           end
 
-          # DEPRECATED
-          def public_rocket_job_properties(*args)
-            warn "Replace calls to .public_rocket_job_properties by adding `user_editable: true` option to the field declaration in #{name} for: #{args.inspect}"
-            self.user_editable_fields += args.collect(&:to_sym)
+          # Builds this job instance from the supplied properties hash.
+          # Overridden by batch to support child objects.
+          def from_properties(properties)
+            new(properties)
           end
         end
 
-        # Returns [true|false] whether to collect nil results from running this batch
-        def collect_nil_output?
-          collect_output? ? (collect_nil_output == true) : false
-        end
-
-        # Returns [true|false] whether to collect the results from running this batch
-        def collect_output?
-          collect_output == true
-        end
-
         # Returns [Float] the number of seconds the job has taken
         # - Elapsed seconds to process the job from when a worker first started working on it
         #   until now if still running, or until it was completed
@@ -282,7 +257,6 @@ module RocketJob
         # Returns [Hash] status of this job
         def as_json
           attrs = serializable_hash(methods: %i[seconds duration])
-          attrs.delete("result") unless collect_output?
           attrs.delete("failure_count") unless failure_count.positive?
           if queued?
             attrs.delete("started_at")
@@ -319,16 +293,17 @@ module RocketJob
           h = as_json
           h.delete("seconds")
           h.dup.each_pair do |k, v|
-            if v.is_a?(Time)
+            case v
+            when Time
               h[k] = v.in_time_zone(time_zone).to_s
-            elsif v.is_a?(BSON::ObjectId)
+            when BSON::ObjectId
               h[k] = v.to_s
             end
           end
           h
         end
 
-        # Returns [Boolean] whether the worker runs on a particular server.
+        # Returns [true|false] whether the worker runs on a particular server.
         def worker_on_server?(server_name)
           return false unless worker_name.present? && server_name.present?