rocketjob 6.3.2 → 7.1.0

data/lib/rocket_job/batch/io.rb

@@ -95,6 +95,9 @@ module RocketJob
       #       Parses each line from the file as an Array and uploads each array for processing by workers.
       #     :hash
       #       Parses each line from the file into a Hash and uploads each hash for processing by workers.
+      #     Default: :line
+      #     With a schema, the default is the mode of the input category, and the lines of the file are uploaded
+      #     in the :line and :hash modes, see RocketJob::Category::Input#mode.
       #     See IOStreams::Stream#each.
       #
       #   category [Symbol|RocketJob::Category::Input]
@@ -109,7 +112,7 @@ module RocketJob
       # Example:
       #   # Load plain text records from a file, stripping all non-printable characters,
       #   # as well as any characters that cannot be converted to UTF-8
-      #   path = IOStreams.path('hello.csv').option(:encode, cleaner: :printable, replace: '')
+      #   path = IOStreams.path('hello.csv').encoding(cleaner: :printable, replace: '')
       #   job.upload(path)
       #
       # Example: Zip
@@ -120,11 +123,11 @@ module RocketJob
       #   job.upload('myfile.csv.zip.enc')
       #
       # Example: Explicitly set the streams
-      #   path = IOStreams.path('myfile.ze').stream(:encode, encoding: 'UTF-8').stream(:zip).stream(:enc)
+      #   path = IOStreams.path('myfile.ze').stream(:zip).stream(:enc).encoding('UTF-8')
       #   job.upload(path)
       #
       # Example: Supply custom options
-      #   path = IOStreams.path('myfile.csv.enc').option(:enc, compress: false).option(:encode, encoding: 'UTF-8')
+      #   path = IOStreams.path('myfile.csv.enc').option(:enc, compress: false).encoding('UTF-8')
       #   job.upload(path)
       #
       # Example: Read from a tempfile and use the original file name to determine which streams to apply
@@ -269,13 +272,15 @@ module RocketJob
       # * If an exception is raised while uploading data, the input collection is cleared out
       #   so that if a job is retried during an upload failure, data is not duplicated.
 
-      def upload(object = nil, category: :main, file_name: nil, stream_mode: nil, on_first: nil, columns: nil, slice_batch_size: nil, **args, &block)
+      def upload(object = nil, category: :main, file_name: nil, stream_mode: nil, on_first: nil, columns: nil,
+                 slice_batch_size: nil, **args, &block)
         input_collection = input(category)
 
         if block
           raise(ArgumentError, "Cannot supply both an object to upload, and a block.") if object
-          if stream_mode || columns || slice_batch_size || args.size > 0
-            raise(ArgumentError, "Unknown keyword arguments when uploading a block. Only accepts :category, :file_name, or :on_first")
+          if stream_mode || columns || slice_batch_size || args.size.positive?
+            raise(ArgumentError,
+                  "Unknown keyword arguments when uploading a block. Only accepts :category, :file_name, or :on_first")
           end
 
           category           = input_category(category)
@@ -292,8 +297,9 @@ module RocketJob
         count =
           case object
           when Range
-            if file_name || stream_mode || on_first || args.size > 0
-              raise(ArgumentError, "Unknown keyword arguments when uploading a Range. Only accepts :category, :columns, or :slice_batch_size")
+            if file_name || stream_mode || on_first || args.size.positive?
+              raise(ArgumentError,
+                    "Unknown keyword arguments when uploading a Range. Only accepts :category, :columns, or :slice_batch_size")
             end
 
             first = object.first
@@ -304,14 +310,16 @@ module RocketJob
               input_collection.upload_integer_range_in_reverse_order(last, first, slice_batch_size: slice_batch_size || 1_000)
             end
           when Mongoid::Criteria
-            if file_name || stream_mode || on_first || args.size > 0
-              raise(ArgumentError, "Unknown keyword arguments when uploading a Mongoid::Criteria. Only accepts :category, :columns, or :slice_batch_size")
+            if file_name || stream_mode || on_first || args.size.positive?
+              raise(ArgumentError,
+                    "Unknown keyword arguments when uploading a Mongoid::Criteria. Only accepts :category, :columns, or :slice_batch_size")
             end
 
             input_collection.upload_mongo_query(object, columns: columns, slice_batch_size: slice_batch_size, &block)
           when defined?(ActiveRecord::Relation) ? ActiveRecord::Relation : false
-            if file_name || stream_mode || on_first || args.size > 0
-              raise(ArgumentError, "Unknown keyword arguments when uploading an ActiveRecord::Relation. Only accepts :category, :columns, or :slice_batch_size")
+            if file_name || stream_mode || on_first || args.size.positive?
+              raise(ArgumentError,
+                    "Unknown keyword arguments when uploading an ActiveRecord::Relation. Only accepts :category, :columns, or :slice_batch_size")
             end
 
             input_collection.upload_arel(object, columns: columns, slice_batch_size: slice_batch_size, &block)
@@ -319,14 +327,16 @@ module RocketJob
           else
             raise(ArgumentError, "Unknown keyword argument :columns when uploading a file") if columns
 
+            rocketjob_warn_deprecated_upload_options(args)
             category = input_category(category)
 
-            # Extract the header line during the upload when applicable.
+            # The path sets the format of a category whose format is :auto, which decides whether the file has a
+            # header row to extract during the upload.
+            path           = category.upload_path(object, original_file_name: file_name)
             extract_header = category.extract_header_callback(on_first)
-            path = category.upload_path(object, original_file_name: file_name)
 
             input_collection.upload(on_first: extract_header, slice_batch_size: slice_batch_size) do |io|
-              path.each(stream_mode || :line, **args) { |line| io << line }
+              category.each_record(path, mode: stream_mode, **args) { |record| io << record }
             end
 
           end
@@ -336,15 +346,15 @@ module RocketJob
       end
 
       # @deprecated
-      def upload_arel(arel, *column_names, category: :main, &block)
-        count             = input(category).upload_arel(arel, columns: column_names, &block)
+      def upload_arel(arel, *column_names, category: :main, &)
+        count             = input(category).upload_arel(arel, columns: column_names, &)
         self.record_count = (record_count || 0) + count
         count
       end
 
       # @deprecated
-      def upload_mongo_query(criteria, *column_names, category: :main, &block)
-        count             = input(category).upload_mongo_query(criteria, columns: column_names, &block)
+      def upload_mongo_query(criteria, *column_names, category: :main, &)
+        count             = input(category).upload_mongo_query(criteria, columns: column_names, &)
         self.record_count = (record_count || 0) + count
         count
       end
@@ -358,7 +368,8 @@ module RocketJob
 
       # @deprecated
       def upload_integer_range_in_reverse_order(start_id, last_id, category: :main, slice_batch_size: 1_000)
-        count             = input(category).upload_integer_range_in_reverse_order(start_id, last_id, slice_batch_size: slice_batch_size)
+        count             = input(category).upload_integer_range_in_reverse_order(start_id, last_id,
+                                                                                  slice_batch_size: slice_batch_size)
         self.record_count = (record_count || 0) + count
         count
       end
@@ -386,6 +397,11 @@ module RocketJob
         count
       end
 
+      # The options that IOStreams reads a file with which describe its columns, and are deprecated, see
+      # RocketJob::Category::Input#schema.
+      DEPRECATED_UPLOAD_OPTIONS = %i[allowed_columns required_columns skip_unknown cleanse_header].freeze
+      private_constant :DEPRECATED_UPLOAD_OPTIONS
+
       # Download the output data into the supplied file, io, IOStreams::Path, or IOStreams::Stream.
       # Returns [Integer] the number of records / lines downloaded.
       #
@@ -411,7 +427,7 @@ module RocketJob
       #   job.download(path)
       #
       # Example: Supply custom options. Set the file name within the zip file.
-      #   path = IOStreams.path('myfile.csv.zip').option(:zip, zip_file_name: 'myfile.csv')
+      #   path = IOStreams.path('myfile.csv.zip').option(:zip, entry_file_name: 'myfile.csv')
       #   job.download(path)
       #
       # Example: Download into a tempfile, or stream, using the original file name to determine the streams to apply:
@@ -459,7 +475,7 @@ module RocketJob
           binary_header_line = output_collection.slice_class.to_binary(header_line) if header_line
 
           # Don't overwrite supplied stream options if any
-          stream = stream&.is_a?(IOStreams::Stream) ? stream.dup : IOStreams.new(category.file_name)
+          stream = stream.is_a?(IOStreams::Stream) ? stream.dup : IOStreams.new(category.file_name)
           stream.remove_from_pipeline(output_collection.slice_class.binary_format)
           stream.writer(**args) do |io|
             # TODO: Binary formats should return the record count, instead of the slice count.
@@ -473,6 +489,21 @@ module RocketJob
           end
         end
       end
+
+      private
+
+      # Reports a deprecation warning, see RocketJob::Deprecation, at the line of the application that uploads a file
+      # with options that describe the columns of the file, which the schema of the input category replaces.
+      def rocketjob_warn_deprecated_upload_options(args)
+        names = args.keys & DEPRECATED_UPLOAD_OPTIONS
+        return if names.empty?
+
+        RocketJob::Deprecation.warn(
+          "#{names.collect { |name| "#{name}:" }.join(', ')} #{names.size == 1 ? 'is' : 'are'} deprecated, describe " \
+          "the columns of the file with the schema: of the input category instead, see " \
+          "#{RocketJob::Deprecation::UPGRADE_GUIDE}"
+        )
+      end
     end
   end
 end

data/lib/rocket_job/batch/logger.rb

@@ -29,7 +29,7 @@ module RocketJob
       #   - log_exception logs entire exception if raised
       #   - on_exception_level changes log level from info to error on exception
       #   - silence noisy jobs by raising log level
-      def rocket_job_batch_slice_logger(&block)
+      def rocket_job_batch_slice_logger(&)
         logger.measure_info(
           "Completed slice",
           metric:             "#{self.class.name}/slice",
@@ -37,7 +37,7 @@ module RocketJob
           on_exception_level: :error,
           silence:            log_level,
           payload:            {records: rocket_job_slice&.size},
-          &block
+          &
         )
       end
 

data/lib/rocket_job/batch/model.rb

@@ -66,14 +66,14 @@ module RocketJob
             percent = percent_complete
             if percent >= 5
               secs                        = seconds.to_f
-              h["est_remaining_duration"] = RocketJob.seconds_as_duration((((secs / percent) * 100) - secs))
+              h["est_remaining_duration"] = RocketJob.seconds_as_duration(((secs / percent) * 100) - secs)
             end
           end
         elsif completed?
           secs                  = seconds.to_f
           h["records_per_hour"] = ((record_count.to_f / secs) * 60 * 60).round if record_count&.positive? && (secs > 0.0)
         end
-        h.merge!(super(time_zone))
+        h.merge!(super)
         h.delete("result")
         # Worker name should be retrieved from the slices when processing
         h.delete("worker_name") if sub_state == :processing

data/lib/rocket_job/batch/performance.rb

@@ -81,7 +81,7 @@ module RocketJob
           o.on("-E", "--encrypt", "Turn on encryption") do
             self.encrypt = true
           end
-          o.on("-s", "--slice_size COUNT", "Slice size") do
+          o.on("-s", "--slice_size COUNT", "Slice size") do |arg|
             self.slice_size = arg.to_i
           end
         end

data/lib/rocket_job/batch/statistics.rb

@@ -50,9 +50,16 @@ module RocketJob
           return unless last
 
           last_target = paths.inject(in_memory) do |target, sub_key|
-            target.key?(sub_key) ? target[sub_key] : target[sub_key] = Hash.new(0)
+            # Assigning a Hash into a BSON::Document (Mongoid 9) stores a
+            # converted copy, so read the value back rather than reusing the
+            # assigned object, otherwise nested increments are lost.
+            target[sub_key] = Hash.new(0) unless target.key?(sub_key)
+            target[sub_key]
           end
-          last_target[last] += increment
+          # Mongoid 9 demongoizes Hash fields to a BSON::Document without a
+          # default, so the leaf value may be nil even when the field default
+          # is `Hash.new(0)`. Default missing leaves to zero before adding.
+          last_target[last] = (last_target[last] || 0) + increment
         end
       end
 

data/lib/rocket_job/batch/throttle.rb

@@ -45,11 +45,16 @@ module RocketJob
         #     Or, a block that will return the filter.
         #     Default: :throttle_filter_class (Throttle all jobs of this class)
         #
+        #   description: [String|Proc]
+        #     Human readable reason why the job is throttled, persisted to `throttled_by`
+        #     and shown in Mission Control. A Proc is called with the job and slice and
+        #     must return a String. Default: a humanized version of the method name.
+        #
         # Note: Throttles are executed in the order they are defined.
-        def define_batch_throttle(method_name, filter: :throttle_filter_class)
+        def define_batch_throttle(method_name, filter: :throttle_filter_class, description: nil)
           # Duplicate to prevent modifying parent class throttles
           definitions = rocket_job_batch_throttles ? rocket_job_batch_throttles.dup : ThrottleDefinitions.new
-          definitions.add(method_name, filter)
+          definitions.add(method_name, filter, description)
           self.rocket_job_batch_throttles = definitions
         end
 

data/lib/rocket_job/batch/throttle_running_workers.rb

@@ -36,7 +36,11 @@ module RocketJob
 
         validates :throttle_running_workers, numericality: {greater_than_or_equal_to: 0}, allow_nil: true
 
-        define_batch_throttle :throttle_running_workers_exceeded?, filter: :throttle_filter_id
+        define_batch_throttle :throttle_running_workers_exceeded?,
+                              filter:      :throttle_filter_id,
+                              description: lambda { |job, *|
+                                "Throttled: maximum of #{job.throttle_running_workers} running workers reached"
+                              }
       end
 
       private

data/lib/rocket_job/batch/throttle_windows.rb

@@ -43,7 +43,9 @@ module RocketJob
         # Duration in seconds of the secondary window.
         field :secondary_duration, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
 
-        define_batch_throttle :throttle_windows_exceeded?, filter: :throttle_filter_id
+        define_batch_throttle :throttle_windows_exceeded?,
+                              filter:      :throttle_filter_id,
+                              description: "Throttled: outside of its processing window"
 
         validates_each :primary_schedule, :secondary_schedule do |record, attr, value|
           record.errors.add(attr, "Invalid #{attr}: #{value.inspect}") if value && !Fugit::Cron.new(value)
@@ -62,7 +64,7 @@ module RocketJob
 
       def throttle_outside_window?(schedule, duration)
         cron = Fugit::Cron.new(schedule)
-        time = Time.now.utc + 1
+        time = Time.now.getutc + 1
         # Add 1 second since right now could be the very beginning of the processing window.
         previous_time = cron.previous_time(time).to_utc_time
         previous_time + duration < time

data/lib/rocket_job/batch/worker.rb

@@ -69,7 +69,7 @@ module RocketJob
       # Note: The slice will be removed from processing when this method completes
       #
       # @deprecated Please open a ticket if you need this behavior.
-      def work_first_slice(&block)
+      def work_first_slice(&)
         raise "#work_first_slice can only be called from within before_batch callbacks" unless sub_state == :before
 
         # TODO: Make these settings configurable
@@ -89,7 +89,7 @@ module RocketJob
 
         # TODO: Persist that the first slice is being processed by this worker
         slice.start
-        rocket_job_process_slice(slice, &block)
+        rocket_job_process_slice(slice, &)
       end
 
       # Returns [Array<ActiveWorker>] All workers actively working on this job
@@ -111,15 +111,52 @@ module RocketJob
       end
 
       def rocket_job_batch_throttled?(slice, worker)
-        filter = self.class.rocket_job_batch_throttles.matching_filter(self, slice)
-        return false unless filter
+        throttle = self.class.rocket_job_batch_throttles.matching_throttle(self, slice)
+        unless throttle
+          rocket_job_clear_throttled
+          return false
+        end
 
         # Restore retrieved slice so that other workers can process it later.
         slice.set(worker_name: nil, state: :queued, started_at: nil)
-        worker.add_to_current_filter(filter)
+        worker.add_to_current_filter(throttle.extract_filter(self, slice))
+        rocket_job_set_throttled(throttle.extract_description(self, slice))
         true
       end
 
+      # Record why this batch job's slices are being throttled, for visibility in
+      # Mission Control.
+      #
+      # Unlike a simple job, a running batch job document is shared by every worker
+      # and is deliberately not written on each poll (see Worker#find_and_assign_job).
+      # The reason is therefore persisted with a guarded conditional update so the
+      # shared document is only written when the reason actually changes, not on
+      # every throttled poll.
+      def rocket_job_set_throttled(reason)
+        return if throttled_by == reason
+
+        self.class.with(write: {w: 1}) do |query|
+          query.where(id: id, state: :running, :throttled_by.ne => reason).
+            update_all("$set" => {throttled_by: reason, throttled_at: Time.now})
+        end
+        self.throttled_by = reason
+      end
+
+      # Clear the throttle reason once slices are flowing again.
+      #
+      # Gated on the in-memory value so a worker that never throttled this job
+      # issues no database round trip per slice; only a worker that previously set
+      # a reason clears it, with a guarded update to avoid a redundant write.
+      def rocket_job_clear_throttled
+        return if throttled_by.nil?
+
+        self.class.with(write: {w: 1}) do |query|
+          query.where(id: id, :throttled_by.ne => nil).
+            update_all("$unset" => {throttled_by: "", throttled_at: ""})
+        end
+        self.throttled_by = nil
+      end
+
       # Process a single slice from Mongo
       # Once the slice has been successfully processed it will be removed from the input collection
       # Returns [Integer] the number of records successfully processed
@@ -151,7 +188,7 @@ module RocketJob
           if slice.processing_record_number.to_i > 1
             append = true
             logger.info("Resuming previously incomplete slice from record number #{slice.processing_record_number}")
-            slice.records[slice.processing_record_number - 1..-1]
+            slice.records[(slice.processing_record_number - 1)..]
           else
             # Reprocess all records in this slice.
             slice.processing_record_number = 0
@@ -162,8 +199,11 @@ module RocketJob
         RocketJob::Sliced::Writer::Output.collect(self, input_slice: slice, append: append) do |writer|
           records.each do |record|
             slice.processing_record_number += 1
-            SemanticLogger.named_tagged(record: slice.current_record_number) do
-              writer << rocket_job_batch_perform(slice, record, &block)
+            # `current_record_number` reads Mongoid fields, so compute it once per
+            # record and reuse it for both the log tag and `#perform`.
+            record_number = slice.current_record_number
+            SemanticLogger.named_tagged(record: record_number) do
+              writer << rocket_job_batch_perform(slice, record, record_number, &block)
               count += 1
             end
           end
@@ -172,8 +212,8 @@ module RocketJob
       end
 
       # Perform a single record within the current slice.
-      def rocket_job_batch_perform(slice, record)
-        @rocket_job_record_number = slice.current_record_number
+      def rocket_job_batch_perform(slice, record, record_number = slice.current_record_number)
+        @rocket_job_record_number = record_number
 
         return block_given? ? yield(record) : perform(record) if _perform_callbacks.empty?
 
@@ -248,7 +288,7 @@ module RocketJob
         unless new_record?
           # Fail job iff no other worker has already finished it
           # Must set write concern to at least 1 since we need the nModified back
-          result   = self.class.with(write: {w: 1}) do |query|
+          result = self.class.with(write: {w: 1}) do |query|
             query.
               where(id: id, state: :running, sub_state: :processing).
               update({"$set" => {state: "failed", worker_name: worker_name}})

data/lib/rocket_job/category/base.rb

@@ -15,6 +15,10 @@ module RocketJob
         # The header columns when the file does not include a header row.
         # Note:
         # - All column names must be strings so that it can be serialized into MongoDB.
+        # - With a schema, see #schema, the columns of an input category hold the header row read from the uploaded
+        #   file, see RocketJob::Category::Input, and those of an output category cannot be set.
+        # - Deprecated without a schema, use `schema`, whose fields are the columns, with `"header": false` for a file
+        #   without a header row.
         field :columns, type: Array
 
         # On an input collection `format` specifies the format of the input data so that it can be
@@ -28,17 +32,76 @@ module RocketJob
         #
         # Default: `nil`
         field :format, type: ::Mongoid::StringifiedSymbol
-        validates_inclusion_of :format, in: [nil, :auto] + IOStreams::Tabular.registered_formats
+        validates_inclusion_of :format, in: [nil, :auto] + IOStreams.tabular_formats
 
         # Any specialized format specific options. For example, `:fixed` format requires a `:layout`.
+        #
+        # Deprecated, use `schema`, see #schema, which gives the width of each field of a fixed width file. It cannot be
+        # set with a schema.
         field :format_options, type: Hash
 
+        # The fields of the records in this category and the type of each one, as a Frictionless Table Schema, see
+        # https://datapackage.org/standard/table-schema/, so that records are read as typed values, such as an Integer,
+        # a BigDecimal or a Date, and written in the text form of each type, for every tabular format. It also gives
+        # the position of each field of a fixed width file, with its `width`, and `"header": false` describes a file
+        # without a header row. See `IOStreams::Schema`, and `IOStreams::Stream#schema`.
+        #
+        # A Hash, or the path or url of a JSON file that holds the schema, which is read when the category is first
+        # used to read or write records.
+        #
+        # Example, for a fixed width file:
+        #   {"fields" => [{"name" => "name", "type" => "string", "width" => 10},
+        #                 {"name" => "zip", "type" => "integer", "width" => 5}]}
+        #
+        # The schema describes the columns of the file, so `format_options` and `columns` cannot be set with it, nor on
+        # an input category `allowed_columns`, `required_columns`, `skip_unknown` and `header_cleanser`. It needs a
+        # `format`, such as `:csv`, or `:auto` to take it from the file name.
+        #
+        # Default: nil, which reads and writes each value as text, as before.
+        field :schema
+        validate :schema_valid
+        validate :schema_replaces_fields
+
+        # Setting the schema builds the tabular again from it when it is next used, see #tabular.
+        define_method(:schema=) do |value|
+          @tabular = nil
+          super(value)
+        end
+
         # When `:format` is not supplied the file name can be used to infer the required format.
         # Optional.
         # Default: nil
         field :file_name, type: IOStreams::Path
       end
 
+      module ClassMethods
+        # Returns [Array<String>] the names of the deprecated options in the supplied properties of a category, with
+        # String keys, that its schema replaces, see #schema: its `columns`. Overridden by the input category.
+        def deprecated_column_options(properties)
+          properties["columns"].present? ? ["columns"] : []
+        end
+
+        # Reports a deprecation warning, see RocketJob::Deprecation, for the deprecated options in the supplied
+        # properties of a category, such as those that a job class defines it with, or that a Dirmon entry holds.
+        def warn_deprecated_options(properties)
+          properties = properties.to_h.stringify_keys
+          names      = deprecated_column_options(properties)
+          unless names.empty?
+            options = names.collect { |name| "#{name}:" }.join(", ")
+            RocketJob::Deprecation.warn(
+              "#{options} #{names.size == 1 ? 'is' : 'are'} deprecated, describe the columns of the file with schema: " \
+              "instead, see #{RocketJob::Deprecation::UPGRADE_GUIDE}"
+            )
+          end
+          return if properties["format_options"].blank?
+
+          RocketJob::Deprecation.warn(
+            "format_options: is deprecated, use schema: instead, which gives the width of each field of a fixed width " \
+            "file, see #{RocketJob::Deprecation::UPGRADE_GUIDE}"
+          )
+        end
+      end
+
       # Return which slice serializer class to use that matches the current options.
       def serializer_class
         case serializer
@@ -58,6 +121,8 @@ module RocketJob
       end
 
       def tabular
+        return @tabular ||= schema_tabular if schema.present?
+
         @tabular ||= IOStreams::Tabular.new(
           columns:        columns,
           format:         format == :auto ? nil : format,
@@ -76,6 +141,59 @@ module RocketJob
         collection_name << ".#{name}" unless name == :main
         collection_name
       end
+
+      private
+
+      # Returns [IOStreams::Tabular] that reads or writes the records of this category by its schema, see #schema,
+      # reading the file that holds it when it is a path or url.
+      def schema_tabular
+        IOStreams::Tabular.new(
+          schema:    IOStreams::Schema::Source.new(schema).schema,
+          format:    format == :auto ? nil : format,
+          file_name: file_name
+        )
+      end
+
+      # Returns [Array<Symbol>] the fields of this category that are set, which its schema replaces, see #schema.
+      def fields_replaced_by_schema
+        format_options.present? ? [:format_options] : []
+      end
+
+      # The fields that the schema replaces cannot be set with it, since IOStreams raises for them with a schema, rather
+      # than apply one and ignore the other.
+      def schema_replaces_fields
+        return if schema.blank?
+
+        errors.add(:schema, "needs a format, such as :csv, or :auto to take it from the file name") if format.blank?
+        replaced = fields_replaced_by_schema
+        errors.add(:schema, "replaces #{replaced.to_sentence}, which cannot be set with it") unless replaced.empty?
+      end
+
+      # Checks the schema without reading a file that holds it, since its store may not be available yet.
+      def schema_valid
+        case schema
+        when nil
+          nil
+        when Hash
+          IOStreams::Schema.new(schema)
+        when String
+          schema_path_valid
+        else
+          errors.add(:schema, "must be a Hash, or the path or url of a JSON file")
+        end
+      rescue ArgumentError => e
+        errors.add(:schema, "is not valid: #{e.message}")
+      end
+
+      def schema_path_valid
+        IOStreams.path(schema)
+      rescue LoadError
+        # A gem that the path needs is not installed in this process, so the path is checked when the job runs.
+        nil
+      rescue StandardError
+        # The path is not included in the message, since a url can include credentials.
+        errors.add(:schema, "is not a valid path or url")
+      end
     end
   end
 end