rocketjob 7.0.0 → 7.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7cf0e18a83d97aadac0d2741a399bcb55fa33884ccf2660836a5a3cfd946f772
-  data.tar.gz: d95563d0c77495dcdb3498dccf7cfc22fefee78bd22df244b823fc3939ef5373
+  metadata.gz: 4a97fb3e6b6ed4130f52e5796bdb0737661c20e83f158c1f00667629438ea316
+  data.tar.gz: 298727e29cd1f8d37cd4cab34c23d7a8b3fee805f0f264c784e72af873309640
 SHA512:
-  metadata.gz: 8f379b7890ddce715719616c311b501a916bc081bfd8eaba5055fa124badd58084985d9f38da0c081a8b5bcff3c67108ba3ea5ca52d8f2a8d7a92aa7297c94e8
-  data.tar.gz: db4e047d9c07faed020db569f7268fc0dea981ede5792ec40b7636718b118514fc6edc8d08b961d3dfe1d4ecadada40f548dde490ed4ede5326bbcb12ef5bedd
+  metadata.gz: df159598ced406d32f34802c65361c6f429075862722afb078324e265f4c71e779f12646292b1fe03d3a9e53bcd94c32f9f9e18aea3853260e21368b0e46bb5a
+  data.tar.gz: f1c0e817b687fa470f66ee3cbe7b519d52835813ad9ec99c96f092d557e1438f09358af1391b6288832df47c45249011d7d151668e782cf0207b25a431a274df

data/README.md

@@ -1,5 +1,5 @@
 # Rocket Job
-[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg)
+[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/reidmorrison/rocketjob/badges/coverage.json)](https://github.com/reidmorrison/rocketjob/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg)
 
 **Process millions of records across thousands of workers.**
 

data/lib/rocket_job/batch/categories.rb

@@ -29,6 +29,7 @@ module RocketJob
         # Define a new input category
         # @see RocketJob::Category::Input
         def input_category(**args)
+          RocketJob::Category::Input.warn_deprecated_options(args)
           category = RocketJob::Category::Input.new(**args)
           if defined_input_categories.nil?
             self.defined_input_categories = [category]
@@ -40,6 +41,7 @@ module RocketJob
         # Define a new output category
         # @see RocketJob::Category::Output
         def output_category(**args)
+          RocketJob::Category::Output.warn_deprecated_options(args)
           category = RocketJob::Category::Output.new(**args)
           if defined_output_categories.nil?
             self.defined_output_categories = [category]
@@ -120,28 +122,39 @@ module RocketJob
         false
       end
 
+      # Sets the supplied properties of each input category on it, such as those that a Dirmon entry or an
+      # UploadFileJob holds, and reports a deprecation warning for a deprecated option among them, see
+      # Category::Base.warn_deprecated_options.
       def merge_input_categories(categories)
         return if categories.blank?
 
         categories.each do |properties|
           category_name = (properties["name"] || properties[:name] || :main).to_sym
           category      = input_category(category_name)
-          properties.each { |key, value| category.public_send(:"#{key}=", value) }
+          rocketjob_categories_merge(category, properties)
         end
       end
 
+      # Sets the supplied properties of each output category on it, see #merge_input_categories.
       def merge_output_categories(categories)
         return if categories.blank?
 
         categories.each do |properties|
           category_name = (properties["name"] || properties[:name] || :main).to_sym
           category      = output_category(category_name)
-          properties.each { |key, value| category.public_send(:"#{key}=", value) }
+          rocketjob_categories_merge(category, properties)
         end
       end
 
       private
 
+      # Sets the supplied properties on the category, after reporting a deprecation warning for a deprecated option
+      # among them, which includes the category's schema, since an input category's columns are not deprecated with one.
+      def rocketjob_categories_merge(category, properties)
+        category.class.warn_deprecated_options(category.attributes.slice("schema").merge(properties.to_h.stringify_keys))
+        properties.each { |key, value| category.public_send(:"#{key}=", value) }
+      end
+
       def rocketjob_categories_assign
         # Input categories defaults to :main if none was set in the class
         if input_categories.empty?
@@ -218,7 +231,7 @@ module RocketJob
 
       # Migrate existing v5 batch jobs to v6
       def rocketjob_categories_migrate
-        return unless attribute_present?(:input_categories) && self[:input_categories]&.first.is_a?(Symbol)
+        return unless attributes["input_categories"]&.first.is_a?(Symbol)
 
         serializer = :none
         if attribute_present?(:compress)

data/lib/rocket_job/batch/io.rb

@@ -95,6 +95,9 @@ module RocketJob
       #       Parses each line from the file as an Array and uploads each array for processing by workers.
       #     :hash
       #       Parses each line from the file into a Hash and uploads each hash for processing by workers.
+      #     Default: :line
+      #     With a schema, the default is the mode of the input category, and the lines of the file are uploaded
+      #     in the :line and :hash modes, see RocketJob::Category::Input#mode.
       #     See IOStreams::Stream#each.
       #
       #   category [Symbol|RocketJob::Category::Input]
@@ -109,7 +112,7 @@ module RocketJob
       # Example:
       #   # Load plain text records from a file, stripping all non-printable characters,
       #   # as well as any characters that cannot be converted to UTF-8
-      #   path = IOStreams.path('hello.csv').option(:encode, cleaner: :printable, replace: '')
+      #   path = IOStreams.path('hello.csv').encoding(cleaner: :printable, replace: '')
       #   job.upload(path)
       #
       # Example: Zip
@@ -120,11 +123,11 @@ module RocketJob
       #   job.upload('myfile.csv.zip.enc')
       #
       # Example: Explicitly set the streams
-      #   path = IOStreams.path('myfile.ze').stream(:encode, encoding: 'UTF-8').stream(:zip).stream(:enc)
+      #   path = IOStreams.path('myfile.ze').stream(:zip).stream(:enc).encoding('UTF-8')
       #   job.upload(path)
       #
       # Example: Supply custom options
-      #   path = IOStreams.path('myfile.csv.enc').option(:enc, compress: false).option(:encode, encoding: 'UTF-8')
+      #   path = IOStreams.path('myfile.csv.enc').option(:enc, compress: false).encoding('UTF-8')
       #   job.upload(path)
       #
       # Example: Read from a tempfile and use the original file name to determine which streams to apply
@@ -324,14 +327,16 @@ module RocketJob
           else
             raise(ArgumentError, "Unknown keyword argument :columns when uploading a file") if columns
 
+            rocketjob_warn_deprecated_upload_options(args)
             category = input_category(category)
 
-            # Extract the header line during the upload when applicable.
+            # The path sets the format of a category whose format is :auto, which decides whether the file has a
+            # header row to extract during the upload.
+            path           = category.upload_path(object, original_file_name: file_name)
             extract_header = category.extract_header_callback(on_first)
-            path = category.upload_path(object, original_file_name: file_name)
 
             input_collection.upload(on_first: extract_header, slice_batch_size: slice_batch_size) do |io|
-              path.each(stream_mode || :line, **args) { |line| io << line }
+              category.each_record(path, mode: stream_mode, **args) { |record| io << record }
             end
 
           end
@@ -392,6 +397,11 @@ module RocketJob
         count
       end
 
+      # The options that IOStreams reads a file with which describe its columns, and are deprecated, see
+      # RocketJob::Category::Input#schema.
+      DEPRECATED_UPLOAD_OPTIONS = %i[allowed_columns required_columns skip_unknown cleanse_header].freeze
+      private_constant :DEPRECATED_UPLOAD_OPTIONS
+
       # Download the output data into the supplied file, io, IOStreams::Path, or IOStreams::Stream.
       # Returns [Integer] the number of records / lines downloaded.
       #
@@ -417,7 +427,7 @@ module RocketJob
       #   job.download(path)
       #
       # Example: Supply custom options. Set the file name within the zip file.
-      #   path = IOStreams.path('myfile.csv.zip').option(:zip, zip_file_name: 'myfile.csv')
+      #   path = IOStreams.path('myfile.csv.zip').option(:zip, entry_file_name: 'myfile.csv')
       #   job.download(path)
       #
       # Example: Download into a tempfile, or stream, using the original file name to determine the streams to apply:
@@ -479,6 +489,21 @@ module RocketJob
           end
         end
       end
+
+      private
+
+      # Reports a deprecation warning, see RocketJob::Deprecation, at the line of the application that uploads a file
+      # with options that describe the columns of the file, which the schema of the input category replaces.
+      def rocketjob_warn_deprecated_upload_options(args)
+        names = args.keys & DEPRECATED_UPLOAD_OPTIONS
+        return if names.empty?
+
+        RocketJob::Deprecation.warn(
+          "#{names.collect { |name| "#{name}:" }.join(', ')} #{names.size == 1 ? 'is' : 'are'} deprecated, describe " \
+          "the columns of the file with the schema: of the input category instead, see " \
+          "#{RocketJob::Deprecation::UPGRADE_GUIDE}"
+        )
+      end
     end
   end
 end

data/lib/rocket_job/category/base.rb

@@ -15,6 +15,10 @@ module RocketJob
         # The header columns when the file does not include a header row.
         # Note:
         # - All column names must be strings so that it can be serialized into MongoDB.
+        # - With a schema, see #schema, the columns of an input category hold the header row read from the uploaded
+        #   file, see RocketJob::Category::Input, and those of an output category cannot be set.
+        # - Deprecated without a schema, use `schema`, whose fields are the columns, with `"header": false` for a file
+        #   without a header row.
         field :columns, type: Array
 
         # On an input collection `format` specifies the format of the input data so that it can be
@@ -28,17 +32,76 @@ module RocketJob
         #
         # Default: `nil`
         field :format, type: ::Mongoid::StringifiedSymbol
-        validates_inclusion_of :format, in: [nil, :auto] + IOStreams::Tabular.registered_formats
+        validates_inclusion_of :format, in: [nil, :auto] + IOStreams.tabular_formats
 
         # Any specialized format specific options. For example, `:fixed` format requires a `:layout`.
+        #
+        # Deprecated, use `schema`, see #schema, which gives the width of each field of a fixed width file. It cannot be
+        # set with a schema.
         field :format_options, type: Hash
 
+        # The fields of the records in this category and the type of each one, as a Frictionless Table Schema, see
+        # https://datapackage.org/standard/table-schema/, so that records are read as typed values, such as an Integer,
+        # a BigDecimal or a Date, and written in the text form of each type, for every tabular format. It also gives
+        # the position of each field of a fixed width file, with its `width`, and `"header": false` describes a file
+        # without a header row. See `IOStreams::Schema`, and `IOStreams::Stream#schema`.
+        #
+        # A Hash, or the path or url of a JSON file that holds the schema, which is read when the category is first
+        # used to read or write records.
+        #
+        # Example, for a fixed width file:
+        #   {"fields" => [{"name" => "name", "type" => "string", "width" => 10},
+        #                 {"name" => "zip", "type" => "integer", "width" => 5}]}
+        #
+        # The schema describes the columns of the file, so `format_options` and `columns` cannot be set with it, nor on
+        # an input category `allowed_columns`, `required_columns`, `skip_unknown` and `header_cleanser`. It needs a
+        # `format`, such as `:csv`, or `:auto` to take it from the file name.
+        #
+        # Default: nil, which reads and writes each value as text, as before.
+        field :schema
+        validate :schema_valid
+        validate :schema_replaces_fields
+
+        # Setting the schema builds the tabular again from it when it is next used, see #tabular.
+        define_method(:schema=) do |value|
+          @tabular = nil
+          super(value)
+        end
+
         # When `:format` is not supplied the file name can be used to infer the required format.
         # Optional.
         # Default: nil
         field :file_name, type: IOStreams::Path
       end
 
+      module ClassMethods
+        # Returns [Array<String>] the names of the deprecated options in the supplied properties of a category, with
+        # String keys, that its schema replaces, see #schema: its `columns`. Overridden by the input category.
+        def deprecated_column_options(properties)
+          properties["columns"].present? ? ["columns"] : []
+        end
+
+        # Reports a deprecation warning, see RocketJob::Deprecation, for the deprecated options in the supplied
+        # properties of a category, such as those that a job class defines it with, or that a Dirmon entry holds.
+        def warn_deprecated_options(properties)
+          properties = properties.to_h.stringify_keys
+          names      = deprecated_column_options(properties)
+          unless names.empty?
+            options = names.collect { |name| "#{name}:" }.join(", ")
+            RocketJob::Deprecation.warn(
+              "#{options} #{names.size == 1 ? 'is' : 'are'} deprecated, describe the columns of the file with schema: " \
+              "instead, see #{RocketJob::Deprecation::UPGRADE_GUIDE}"
+            )
+          end
+          return if properties["format_options"].blank?
+
+          RocketJob::Deprecation.warn(
+            "format_options: is deprecated, use schema: instead, which gives the width of each field of a fixed width " \
+            "file, see #{RocketJob::Deprecation::UPGRADE_GUIDE}"
+          )
+        end
+      end
+
       # Return which slice serializer class to use that matches the current options.
       def serializer_class
         case serializer
@@ -58,6 +121,8 @@ module RocketJob
       end
 
       def tabular
+        return @tabular ||= schema_tabular if schema.present?
+
         @tabular ||= IOStreams::Tabular.new(
           columns:        columns,
           format:         format == :auto ? nil : format,
@@ -76,6 +141,59 @@ module RocketJob
         collection_name << ".#{name}" unless name == :main
         collection_name
       end
+
+      private
+
+      # Returns [IOStreams::Tabular] that reads or writes the records of this category by its schema, see #schema,
+      # reading the file that holds it when it is a path or url.
+      def schema_tabular
+        IOStreams::Tabular.new(
+          schema:    IOStreams::Schema::Source.new(schema).schema,
+          format:    format == :auto ? nil : format,
+          file_name: file_name
+        )
+      end
+
+      # Returns [Array<Symbol>] the fields of this category that are set, which its schema replaces, see #schema.
+      def fields_replaced_by_schema
+        format_options.present? ? [:format_options] : []
+      end
+
+      # The fields that the schema replaces cannot be set with it, since IOStreams raises for them with a schema, rather
+      # than apply one and ignore the other.
+      def schema_replaces_fields
+        return if schema.blank?
+
+        errors.add(:schema, "needs a format, such as :csv, or :auto to take it from the file name") if format.blank?
+        replaced = fields_replaced_by_schema
+        errors.add(:schema, "replaces #{replaced.to_sentence}, which cannot be set with it") unless replaced.empty?
+      end
+
+      # Checks the schema without reading a file that holds it, since its store may not be available yet.
+      def schema_valid
+        case schema
+        when nil
+          nil
+        when Hash
+          IOStreams::Schema.new(schema)
+        when String
+          schema_path_valid
+        else
+          errors.add(:schema, "must be a Hash, or the path or url of a JSON file")
+        end
+      rescue ArgumentError => e
+        errors.add(:schema, "is not valid: #{e.message}")
+      end
+
+      def schema_path_valid
+        IOStreams.path(schema)
+      rescue LoadError
+        # A gem that the path needs is not installed in this process, so the path is checked when the job runs.
+        nil
+      rescue StandardError
+        # The path is not included in the message, since a url can include credentials.
+        errors.add(:schema, "is not a valid path or url")
+      end
     end
   end
 end

data/lib/rocket_job/category/input.rb

@@ -21,9 +21,13 @@ module RocketJob
       # Note:
       #   When supplied any columns that are rejected will be returned in the cleansed columns
       #   as nil so that they can be ignored during processing.
+      #
+      # Deprecated, use `schema`, whose `fieldsMatch` says which columns a file can have.
       field :allowed_columns, type: Array
 
       # List of columns that must be present, otherwise an Exception is raised.
+      #
+      # Deprecated, use `schema`, with `constraints: {required: true}` on each field that a file must have.
       field :required_columns, type: Array
 
       # Whether to skip unknown columns in the uploaded file.
@@ -41,29 +45,26 @@ module RocketJob
       # Notes:
       # - Only applicable when `allowed_columns` has been set.
       # - Recommended to leave as `false` otherwise a misspelled column can result in missed columns.
+      # - Deprecated, use `schema`, whose `fieldsMatch` of "subset" or "partial" skips the other columns.
       field :skip_unknown, type: ::Mongoid::Boolean, default: false
       validates_inclusion_of :skip_unknown, in: [true, false]
 
-      # When `#upload` is called with a file_name, it uploads the file using any of the following approaches:
+      # How `#upload` reads a file into a category with a schema, see #schema, unless the upload is given a
+      # `stream_mode:`:
       # :line
-      #   Uploads the file a line (String) at a time for processing by workers.
-      #   This is the default behavior and is the most performant since it leaves the parsing of each line
-      #   up to the workers themselves.
-      # :array
-      #   Parses each line from the file as an Array and uploads each array for processing by workers.
-      #   Every line in the input file is parsed and converted into an array before uploading.
-      #   This approach ensures that the entire files is valid before starting to process it.
-      #   Ideal for when files may contain invalid lines.
-      #   Not recommended for large files since the CSV or other parsing is performed sequentially during the
-      #   upload process.
+      #   Uploads the lines of the file, which the workers parse by the schema, so that each value reaches `#perform` as
+      #   its type, such as a Date. This is the default, and the most performant, since the workers parse the lines.
       # :hash
-      #   Parses each line from the file into a Hash and uploads each hash for processing by workers.
-      #   Similar to :array above in that the entire file is parsed before processing is started.
-      #   Slightly less efficient than :array since it stores every record as a hash with both the key and value.
+      #   Also parses every line by the schema when it is uploaded, so that a file with a value that is not valid is not
+      #   uploaded at all. The lines are uploaded, as in :line mode. Not recommended for large files, since the lines
+      #   are parsed one after the other during the upload.
+      # :array
+      #   Cannot be used with a schema.
       #
-      # Recommend using :array when the entire file must be parsed/validated before processing is started, and
-      # upload time is not important.
-      # See IOStreams#each for more details.
+      # Without a schema the mode does not change how a file is uploaded: its lines are uploaded, unless the upload is
+      # given `stream_mode: :array` or `stream_mode: :hash`, which parses each line into an Array or a Hash first, see
+      # RocketJob::Batch::IO#upload. A category with a format whose files have a header row, such as CSV, cannot be
+      # uploaded with `stream_mode: :hash` without a schema.
       field :mode, type: ::Mongoid::StringifiedSymbol, default: :line
       validates_inclusion_of :mode, in: %i[line array hash]
 
@@ -80,11 +81,27 @@ module RocketJob
       #     Do not cleanse the columns names supplied in the header row.
       #
       # Note: Submit a ticket if you have other cleansers that you want added.
+      #
+      # Deprecated, use `schema`, whose fields name the keys of each record, and which the columns of the header row
+      # are matched to whatever their case, spaces and punctuation.
       field :header_cleanser, type: ::Mongoid::StringifiedSymbol, default: :default
       validates :header_cleanser, inclusion: %i[default none]
 
       validates_inclusion_of :serializer, in: %i[none compress encrypt]
 
+      # Returns [Array<String>] the names of the deprecated options in the supplied properties of an input category,
+      # with String keys, see Category::Base.deprecated_column_options: `allowed_columns`, `required_columns`,
+      # `skip_unknown` when true, `header_cleanser` when :none, and `columns` without a schema, since with one they
+      # hold the header row read from the file.
+      def self.deprecated_column_options(properties)
+        names = []
+        names << "columns" if properties["columns"].present? && properties["schema"].blank?
+        names.concat(%w[allowed_columns required_columns].select { |name| properties[name].present? })
+        names << "skip_unknown" if [true, "true"].include?(properties["skip_unknown"])
+        names << "header_cleanser" if properties["header_cleanser"].to_s == "none"
+        names
+      end
+
       # Cleanses the header column names when `cleanse_header` is true
       def cleanse_header!
         return unless header_cleanser == :default
@@ -96,6 +113,8 @@ module RocketJob
       end
 
       def tabular
+        return @tabular ||= schema_tabular if schema.present?
+
         @tabular ||= IOStreams::Tabular.new(
           columns:          columns,
           format:           format == :auto ? nil : format,
@@ -133,36 +152,143 @@ module RocketJob
           @tabular = nil
         end
 
-        # Remove non-printable characters from tabular input formats.
-        if tabular?
+        if schema.present?
+          # IOStreams reads the text of a file with a schema as UTF-8, and raises for text that is not valid UTF-8,
+          # unless an encoding is set on the supplied path, which reads it instead. A schema set on the supplied path
+          # is kept.
+          path.schema(schema) if path.schema.nil?
+        elsif tabular?
+          # Remove non-printable characters from tabular input formats.
           # Cannot change the length of fixed width lines.
           replace = format == :fixed ? " " : ""
-          path.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: replace)
+          path.encoding("UTF-8", cleaner: :printable, replace: replace)
         end
         path
       end
 
-      # Return a lambda to extract the header row from the uploaded file.
+      # Yields each record read from the supplied path for an upload, see RocketJob::Batch::IO#upload, in the supplied
+      # mode, otherwise in :line mode, see `IOStreams::Stream#each`.
+      #
+      # With a schema it yields the lines of the file in the supplied mode, otherwise in this category's mode, see
+      # #mode, and in :hash mode parses each line after the header row by the schema first, so that a value that is not
+      # valid raises before any record is processed.
+      def each_record(path, mode: nil, **args, &)
+        return path.each(mode || :line, **args, &) if schema.blank?
+
+        mode ||= self.mode
+        if mode == :array
+          raise(ArgumentError,
+                "Input category #{name} has a schema, which cannot be uploaded in :array mode, use :line or :hash")
+        end
+        return path.each(:line, **args, &) if mode == :line
+
+        path.each(:line, **args) do |line|
+          # The lines before the header row, and the header row itself, are read by #extract_header_callback.
+          tabular.record_parse(line) unless tabular.header?
+          yield(line)
+        end
+      end
+
+      # Returns the lambda that the upload calls with each record before the first that it stores, instead of storing
+      # it, until the lambda returns true, see `Sliced::Writer::Input`: it reads the header row, when the file has one,
+      # then calls the supplied `on_first` with it. Otherwise it calls `on_first` with the first record. Returns nil
+      # when there is neither.
+      #
+      # A blank line before the header row is skipped, as IOStreams skips it when it reads the header row itself, such
+      # as in :hash mode, so that the header row is not stored as a record.
+      #
+      # The header row is a line, or an Array when the upload parses each line with `stream_mode: :array`, which the
+      # format returns as it is.
       def extract_header_callback(on_first)
-        return on_first unless tabular? && tabular.header?
-
-        case mode
-        when :line
-          lambda do |line|
-            tabular.parse_header(line)
-            cleanse_header!
-            self.columns = tabular.header.columns
-            # Call chained on_first if present
-            on_first&.call(line)
-          end
-        when :array
-          lambda do |row|
-            tabular.header.columns = row
-            cleanse_header!
-            self.columns = category.tabular.header.columns
-            # Call chained on_first if present
-            on_first&.call(line)
-          end
+        return schema_header_callback(on_first) if schema.present?
+        return first_record_callback(on_first) unless tabular? && tabular.header?
+
+        lambda do |line|
+          next false if line.blank?
+
+          tabular.parse_header(line)
+          cleanse_header!
+          self.columns = tabular.header.columns
+          warn_repeated_columns
+          # Call chained on_first if present
+          on_first&.call(line)
+          true
+        end
+      end
+
+      private
+
+      # Reports a deprecation warning, see RocketJob::Deprecation, at the line of the application that uploads a file
+      # whose header row has columns that are the same key once cleansed, such as `a` and `A`, since each record holds
+      # the value of only the last of them, and IOStreams 3.0, which Rocket Job 8 requires, raises for them.
+      def warn_repeated_columns
+        repeated = columns.compact.tally.select { |_column, count| count > 1 }.keys
+        return if repeated.empty?
+
+        RocketJob::Deprecation.warn(
+          "a header row whose columns are the same key, #{repeated.collect(&:inspect).join(', ')}, is deprecated, " \
+          "since each record holds the value of only the last of them, and Rocket Job 8 raises: rename the " \
+          "columns of the file, or describe them with the schema: of the input category, see " \
+          "#{RocketJob::Deprecation::UPGRADE_GUIDE}"
+        )
+      end
+
+      # Returns [IOStreams::Tabular] that reads the records of this category by its schema, see Category::Base#schema,
+      # and has read the header row that an upload read from the file, see #columns, so that the workers parse each line
+      # by it.
+      def schema_tabular
+        tabular = super
+        tabular.read_header(columns) if columns.present? && tabular.header?
+        tabular
+      end
+
+      # Returns the lambda that reads the header row of a file with a schema, see #extract_header_callback. Its columns
+      # are matched to the fields of the schema, which raises IOStreams::Errors::InvalidHeader when they do not match,
+      # and are kept as they are in the file, see #columns. A blank row before it, which IOStreams does not read as a
+      # header row, such as a line of only commas, is skipped, as IOStreams skips it when it reads the header row itself.
+      #
+      # With a schema, `columns` hold the header row of the file uploaded, so an upload reads the header row of its own
+      # file, rather than keep the columns of an earlier one, such as those that a job scheduled again copies from its
+      # last run, see Plugins::Cron.
+      def schema_header_callback(on_first)
+        self.columns = nil
+        @tabular     = nil
+        return first_record_callback(on_first) unless tabular.header?
+
+        lambda do |line|
+          tabular.read_header(line)
+          next false if tabular.header?
+
+          self.columns = tabular.header.columns
+          # Call chained on_first if present
+          on_first&.call(line)
+          true
+        end
+      end
+
+      # The schema describes the columns of the file, so the options that describe them without one cannot be set with
+      # it, see Category::Base#schema.
+      def fields_replaced_by_schema
+        replaced = super + %i[allowed_columns required_columns].select { |name| public_send(name).present? }
+        replaced << :skip_unknown if skip_unknown
+        replaced << :header_cleanser if header_cleanser == :none
+        replaced
+      end
+
+      # The :array mode cannot be used with a schema, see #mode.
+      def schema_replaces_fields
+        super
+        errors.add(:mode, "cannot be :array with a schema, use :line or :hash") if schema.present? && mode == :array
+      end
+
+      # Returns the lambda that calls the supplied `on_first` with the first record uploaded, which is not stored, see
+      # #extract_header_callback, or nil without one.
+      def first_record_callback(on_first)
+        return unless on_first
+
+        lambda do |record|
+          on_first.call(record)
+          true
         end
       end
     end

data/lib/rocket_job/category/output.rb

@@ -29,6 +29,13 @@ module RocketJob
           slice_class:     serializer_class
         )
       end
+
+      private
+
+      # The schema gives the columns of the file written, see Category::Base#schema.
+      def fields_replaced_by_schema
+        columns.present? ? super + [:columns] : super
+      end
     end
   end
 end

data/lib/rocket_job/deprecation.rb

@@ -0,0 +1,51 @@
+module RocketJob
+  # Reports the use of a deprecated option or method as a Ruby deprecation warning, see `Kernel#warn`, as IOStreams
+  # does, see `IOStreams::Deprecation`:
+  # - Only while deprecation warnings are on, see `Warning[:deprecated]`. They are off by default, so production stays
+  #   quiet, and on in tests run by Minitest or test-unit, or with `ruby -W:deprecated` or `RUBYOPT=-W:deprecated`.
+  # - Written to `$stderr`, unless the application routes them elsewhere, such as to its log, or raises them in its
+  #   tests, by overriding `Warning.warn`.
+  #
+  # Each warning names the first line of the call stack outside Rocket Job, such as the line of a job class that
+  # defines a category with a deprecated option, rather than a line of Rocket Job.
+  module Deprecation
+    # The directory of the files of this library, whose lines are skipped when finding the line that used it.
+    LIB_DIR = "#{File.expand_path('..', __dir__)}/".freeze
+
+    # Where the upgrade guide says what replaces each deprecated option.
+    UPGRADE_GUIDE = "https://rocketjob.reidmorrison.com/upgrading.html#upgrading-to-v71".freeze
+
+    # Reports that the application used something deprecated, unless deprecation warnings are off, with the message,
+    # which says what is deprecated and what to use instead, after "Rocket Job: ", at the first line of the call stack
+    # outside this library and the libraries that call it, or without a line when there is none, such as in a worker
+    # that builds the job of a Dirmon entry.
+    #
+    # Returns nil.
+    def self.warn(message)
+      return unless Warning[:deprecated]
+
+      dirs    = library_dirs
+      uplevel = caller_locations(1).index { |location| !library?(location, dirs) }
+      # Kernel#warn counts its own caller, this method, as level 0.
+      Kernel.warn("Rocket Job: #{message}", uplevel: uplevel && (uplevel + 1), category: :deprecated)
+      nil
+    end
+
+    # Returns [Array<String>] the directories whose lines are skipped when finding the line of the application: this
+    # library, the other gems, which call it back, such as Active Support's callbacks and Mongoid, and Ruby's own
+    # library.
+    def self.library_dirs
+      gems = Gem.loaded_specs.each_value.filter_map { |spec| "#{spec.full_gem_path}/" unless spec.name == "rocketjob" }
+      [LIB_DIR, "#{RbConfig::CONFIG['rubylibdir']}/", *gems]
+    end
+    private_class_method :library_dirs
+
+    # Returns [true|false] whether the line of the call stack is in a file of a library, see .library_dirs, or of Ruby
+    # itself.
+    def self.library?(location, dirs)
+      path = (location.absolute_path || location.path).to_s
+      path.start_with?("<internal:") || dirs.any? { |dir| path.start_with?(dir) }
+    end
+    private_class_method :library?
+  end
+end

data/lib/rocket_job/jobs/copy_file_job.rb

@@ -82,8 +82,18 @@ module RocketJob
         self.description ||= "Copying to #{target_url}"
       end
 
+      # Applies the supplied streams to the path, in order. The `none` stream, which `source_streams` and
+      # `target_streams` default to, removes the streams before it, so that on its own the file is read or written as
+      # it is stored, see `IOStreams::Stream#raw`, rather than with IOStreams' deprecated `stream(:none)`.
       def apply_streams(path, streams)
-        streams.each_pair do |stream, args|
+        streams = streams.to_a
+        none    = streams.rindex { |stream, _args| stream.to_s == "none" }
+        if none
+          streams = streams.drop(none + 1)
+          return path.raw if streams.empty?
+        end
+
+        streams.each do |stream, args|
           stream_args = args.nil? ? {} : decode_args(args)
           path.stream(stream.to_sym, **stream_args)
         end

data/lib/rocket_job/plugins/retry.rb

@@ -27,41 +27,42 @@ module RocketJob
     # MyCronJob.create!(retry_limit: 0)
     #
     # Below is a table of the delay for each retry attempt, as well as the _total_ duration spent retrying a job for
-    # the specified number of retries, excluding actual processing time.
+    # the specified number of retries, excluding actual processing time. The delay before retry `n` is
+    # `(n - 1)**4 + 5` seconds.
     #
     # |---------|------------|----------------|
     # | Attempt |      Delay | Total Duration |
     # |---------|------------|----------------|
-    # |      01 |     6.000s |         6.000s |
-    # |      02 |    21.000s |        27.000s |
-    # |      03 |     1m 26s |         1m 53s |
-    # |      04 |     4m 21s |         6m 14s |
-    # |      05 |    10m 30s |        16m 44s |
-    # |      06 |    21m 41s |        38m 25s |
-    # |      07 |     40m 6s |        20h 18m |
-    # |      08 |     20h 8m |        21h 26m |
-    # |      09 |    20h 49m |        23h 16m |
-    # |      10 |    21h 46m |          2h 3m |
-    # |      11 |     23h 4m |          6h 7m |
-    # |      12 |     0h 45m |        11h 52m |
-    # |      13 |     2h 56m |     1d 19h 48m |
-    # |      14 |     5h 40m |      1d 6h 29m |
-    # |      15 |      9h 3m |     2d 20h 33m |
-    # |      16 |    13h 12m |     2d 14h 45m |
-    # |      17 |    18h 12m |     3d 13h 57m |
-    # |      18 |   1d 0h 9m |      5d 19h 7m |
-    # |      19 |  1d 7h 12m |      6d 7h 19m |
-    # |      20 | 1d 15h 26m |      8d 3h 46m |
-    # |      21 |   2d 1h 1m |     10d 9h 47m |
-    # |      22 |  2d 12h 4m |     13d 2h 51m |
-    # |      23 |  3d 0h 44m |     16d 8h 35m |
-    # |      24 |  3d 15h 9m |     20d 4h 45m |
-    # |      25 |  4d 7h 30m |     24d 17h 16 |
-    # |      26 |  5d 1h 56m |     30d 0h 12m |
-    # |      27 | 6d 22h 37m |     36d 3h 49m |
-    # |      28 | 7d 21h 44m |     43d 6h 34m |
-    # |      29 | 8d 23h 28m |     51d 11h 2m |
-    # |      30 |   9d 4h 0m |     61d 20h 2m |
+    # |      01 |     5.000s |         5.000s |
+    # |      02 |     6.000s |        11.000s |
+    # |      03 |    21.000s |        32.000s |
+    # |      04 |     1m 26s |         1m 58s |
+    # |      05 |     4m 21s |         6m 19s |
+    # |      06 |    10m 30s |        16m 49s |
+    # |      07 |    21m 41s |        38m 30s |
+    # |      08 |     40m 6s |         1h 18m |
+    # |      09 |      1h 8m |         2h 26m |
+    # |      10 |     1h 49m |         4h 16m |
+    # |      11 |     2h 46m |          7h 3m |
+    # |      12 |      4h 4m |         11h 7m |
+    # |      13 |     5h 45m |        16h 52m |
+    # |      14 |     7h 56m |      1d 0h 49m |
+    # |      15 |    10h 40m |     1d 11h 29m |
+    # |      16 |     14h 3m |      2d 1h 33m |
+    # |      17 |    18h 12m |     2d 19h 45m |
+    # |      18 |    23h 12m |     3d 18h 57m |
+    # |      19 |   1d 5h 9m |       5d 0h 7m |
+    # |      20 | 1d 12h 12m |     6d 12h 19m |
+    # |      21 | 1d 20h 26m |      8d 8h 46m |
+    # |      22 |   2d 6h 1m |    10d 14h 47m |
+    # |      23 |  2d 17h 4m |     13d 7h 51m |
+    # |      24 |  3d 5h 44m |    16d 13h 36m |
+    # |      25 |  3d 20h 9m |     20d 9h 45m |
+    # |      26 | 4d 12h 30m |    24d 22h 16m |
+    # |      27 |  5d 6h 56m |     30d 5h 12m |
+    # |      28 |  6d 3h 37m |     36d 8h 50m |
+    # |      29 |  7d 2h 44m |    43d 11h 34m |
+    # |      30 |  8d 4h 28m |     51d 16h 2m |
     # |---------|------------|----------------|
     module Retry
       extend ActiveSupport::Concern
@@ -70,7 +71,7 @@ module RocketJob
         after_fail :rocket_job_retry
 
         # Maximum number of times to retry this job.
-        # The default of 25 is almost 25 days of retries.
+        # The default of 25 retries spans about 20 days, see the table above.
         field :retry_limit, type: Integer, default: 25, class_attribute: true, user_editable: true, copy_on_restart: true
 
         # List of times when this job failed
@@ -115,7 +116,7 @@ module RocketJob
       #
       # For example, to see the durations for the first 25 retries:
       #   count = 25
-      #   intervals = (1..count).map { |attempt| attempt**4 + 5 }
+      #   intervals = (0...count).map { |failures| (failures**4) + 5 }
       #
       # Display the above intervals as human readable durations:
       #   intervals.map { |seconds| RocketJob.seconds_as_duration(seconds) }
@@ -124,7 +125,7 @@ module RocketJob
       #   RocketJob.seconds_as_duration(intervals.sum)
       #
       # Or, to see the total durations based on the number of retries:
-      #   (0..count).map{|i| "#{i+1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}"}
+      #   (0...count).map { |i| "#{i + 1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}" }
       def rocket_job_retry_seconds_to_delay
         (rocket_job_failure_count**4) + 5
       end

data/lib/rocket_job/sliced/writer/input.rb

@@ -9,8 +9,9 @@ module RocketJob
         #
         # Parameters
         #   on_first: [Proc]
-        #     Block to call on the first line only, instead of storing in the slice.
-        #     Useful for extracting the header row
+        #     Called with each line before the first that is stored, instead of storing it, until it returns true.
+        #     Useful for extracting the header row, after any blank lines before it, see
+        #     Category::Input#extract_header_callback.
         #     Default: nil
         #
         #   slice_size: [Integer]
@@ -42,8 +43,7 @@ module RocketJob
 
         def <<(line)
           if @on_first
-            @on_first.call(line)
-            @on_first = nil
+            @on_first = nil if @on_first.call(line)
             return self
           end
           @slice << line

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "7.0.0".freeze
+  VERSION = "7.1.0".freeze
 end

data/lib/rocketjob.rb

@@ -23,6 +23,7 @@ module RocketJob
   autoload :ActiveWorker,            "rocket_job/active_worker"
   autoload :Batch,                   "rocket_job/batch"
   autoload :CLI,                     "rocket_job/cli"
+  autoload :Deprecation,             "rocket_job/deprecation"
   autoload :DirmonEntry,             "rocket_job/dirmon_entry"
   autoload :Event,                   "rocket_job/event"
   autoload :Heartbeat,               "rocket_job/heartbeat"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 7.0.0
+  version: 7.1.0
 platform: ruby
 authors:
 - Reid Morrison
@@ -57,14 +57,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
@@ -145,6 +145,7 @@ files:
 - lib/rocket_job/category/output.rb
 - lib/rocket_job/cli.rb
 - lib/rocket_job/config.rb
+- lib/rocket_job/deprecation.rb
 - lib/rocket_job/dirmon_entry.rb
 - lib/rocket_job/event.rb
 - lib/rocket_job/extensions/iostreams/path.rb
@@ -226,7 +227,7 @@ metadata:
   changelog_uri: https://github.com/reidmorrison/rocketjob/blob/main/CHANGELOG.md
   documentation_uri: https://rocketjob.reidmorrison.com
   homepage_uri: https://rocketjob.reidmorrison.com
-  source_code_uri: https://github.com/reidmorrison/rocketjob/tree/v7.0.0
+  source_code_uri: https://github.com/reidmorrison/rocketjob/tree/v7.1.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: