iostreams 2.1.0 → 2.2.0

data/lib/io_streams/schema.rb

@@ -0,0 +1,616 @@
+# frozen_string_literal: true
+
+require "json"
+
+module IOStreams
+  # The fields of the rows and records of a tabular file and the type of each one, described with a Frictionless Table
+  # Schema, see https://datapackage.org/standard/table-schema/, so that reading rows and records returns each value
+  # as its type, such as an Integer or a Date, and writing them writes each value in the text form of its type.
+  #
+  # Supply a schema with `IOStreams::Stream#schema`, as a Hash, or as the path or url of a JSON file. It applies to
+  # every tabular format. A fixed width file also needs the `width` of each field, which IOStreams adds to the
+  # properties of Frictionless, see `IOStreams::Schema::Field`.
+  #
+  # The columns of a file are matched to the fields by their names, see #field_for, and the keys of each record that is
+  # read are the names of its fields. The columns that the schema does not list are skipped when `fieldsMatch` allows
+  # them, or kept with `extraColumns`, see `IOStreams::Schema::Columns`.
+  #
+  # Properties at the top level of the schema:
+  #   fields: [Array<Hash>]
+  #     The fields, in the order of the columns of the file, see `IOStreams::Schema::Field`.
+  #
+  #   fieldsMatch: [String]
+  #     How the columns of the file must match the fields, as Frictionless defines it:
+  #       "exact":    the columns are the fields, in the same order.
+  #       "equal":    the columns are the fields, in any order.
+  #       "subset":   the columns include every field, and the columns that the schema does not list are skipped.
+  #       "superset": every column is a field, but a field can be missing.
+  #       "partial":  at least one column is a field, and the columns that the schema does not list are skipped.
+  #     Whatever its value, a field whose `constraints` are `required` must be one of the columns, see
+  #     `IOStreams::Schema::Field`. A format whose records supply their own keys, such as JSON, applies it to the keys
+  #     of each record, in any order. The columns of a file without a header row, see `header`, are the fields in
+  #     order, so for a format with a header row, such as CSV, it only accepts "exact". A fixed width file always has
+  #     every field, in order.
+  #     Default: "exact"
+  #
+  #   header: [true|false]
+  #     Whether the file has a header row, as the `header` of a Frictionless Table Dialect says. When false, the
+  #     columns of a file of a format with a header row, such as CSV, are the fields, in order: reading takes every line
+  #     as a row, and writing writes no header row, and the fields in order, so the schema needs fields, and fieldsMatch
+  #     "exact", see #check_fields_in_order. A format without a header row, such as JSON or fixed width, ignores it.
+  #     Default: true
+  #
+  #   width: [Integer|"remainder"]
+  #     The width of every line of a fixed width file, when it ends after the last field, such as after a filler.
+  #     "remainder" accepts lines of any length, ignoring what follows the last field.
+  #     Default: the end of the last field.
+  #
+  #   keyType: [String]
+  #     How the key of each field in the records read is formed from its name, which IOStreams adds to the properties
+  #     of Frictionless, see #key_for:
+  #       "string":         the name as a String, such as "zip" for the name :zip.
+  #       "cleansed":       the name once cleansed as `cleanse_header` cleanses a header row, see .cleansed, such as
+  #                         "first_name" for the name "First Name".
+  #       "symbol":         the name as a Symbol, such as :zip for the name "zip", so that a schema held as JSON, whose
+  #                         names are Strings, reads the Symbol keys of a layout, see .from_layout.
+  #       "cleansedSymbol": the cleansed name as a Symbol, such as :first_name for the name "First Name".
+  #     The names of the fields are still the columns of the header row written, and the keys of the records written
+  #     can be the keys that it forms, as well as the names, see #field_for.
+  #     Default: the names as they are given, a String or a Symbol.
+  #
+  #   extraColumns: [String]
+  #     What reading does with the columns that the schema does not list, which `fieldsMatch` "subset" and "partial"
+  #     accept, which IOStreams adds to the properties of Frictionless:
+  #       "skip": leaves them out of the rows and records read.
+  #       "keep": keeps them, after the fields, in the order of the columns, each by the key that `keyType` forms from
+  #               its column, see #key_for, and with its value as it is, as for a field of type "any". So a schema can
+  #               then have no fields, with "subset", to read every column. The keys of each JSON record are kept the
+  #               same way. A fixed width file has only its fields, so it ignores it.
+  #     Writing writes only the fields, whichever it is, since a file that is written has the columns of its fields.
+  #     Raises ArgumentError for "keep" with any other `fieldsMatch`, which does not accept those columns.
+  #     Default: "skip"
+  #
+  # Properties that only describe the schema to people, `$schema`, `title` and `description`, are accepted and have
+  # no effect, as are the properties of other tools, whose names include a `:`, such as `"mytool:owner"`. Any other
+  # property raises `ArgumentError`, including those of Frictionless that IOStreams does not support yet, such as
+  # `primaryKey`, so that a schema never appears to apply a rule that it does not.
+  class Schema
+    autoload :Columns,     "io_streams/schema/columns"
+    autoload :Constraints, "io_streams/schema/constraints"
+    autoload :Field,       "io_streams/schema/field"
+    autoload :JsonNumber,  "io_streams/schema/json_number"
+    autoload :Keys,        "io_streams/schema/keys"
+    autoload :Pattern,     "io_streams/schema/pattern"
+    autoload :Source,      "io_streams/schema/source"
+
+    # The properties of a schema that IOStreams supports.
+    PROPERTIES = %w[fields fieldsMatch header width keyType extraColumns].freeze
+
+    # The properties of a schema that only describe it to people.
+    DESCRIPTIVE_PROPERTIES = %w[$schema title description].freeze
+
+    # The properties of a Frictionless Table Schema that IOStreams does not support yet.
+    UNSUPPORTED_PROPERTIES = %w[missingValues primaryKey foreignKeys uniqueKeys].freeze
+
+    # The values of `fieldsMatch`, see above, those that skip the columns that the schema does not list, and those that
+    # need every field.
+    FIELDS_MATCH      = %w[exact equal subset superset partial].freeze
+    SKIPS_UNKNOWN     = %w[subset partial].freeze
+    NEEDS_EVERY_FIELD = %w[exact equal subset].freeze
+
+    # The values of `extraColumns`, see above.
+    EXTRA_COLUMNS = %w[skip keep].freeze
+
+    # The characters that the loose comparison of a column and a field ignores, see .match_key: all but letters, the
+    # marks that combine with them, and digits, in every language.
+    UNMATCHED = /[^\p{L}\p{M}\p{N}]/
+
+    # The characters that the loose comparison ignores in ASCII text, and in binary text, whose other bytes are not
+    # letters that it can compare: all but the ASCII letters and digits.
+    UNMATCHED_ASCII  = /[^a-z0-9]/
+    UNMATCHED_BINARY = /[^a-z0-9]/n
+
+    # The bytes of the byte order mark that some editors, such as Notepad, write at the start of UTF-8 text.
+    BYTE_ORDER_MARK = "\xEF\xBB\xBF".b
+
+    # Returns [Array<IOStreams::Schema::Field>] the fields, in the order of the columns of the file.
+    attr_reader :fields
+
+    # Returns [String] how the columns of the file must match the fields, see `fieldsMatch` above.
+    attr_reader :fields_match
+
+    # Returns [Integer|:remainder|nil] the width of every line of a fixed width file, or nil when it ends after the last
+    # field.
+    attr_reader :width
+
+    # Returns [String] the text of the label once it is cleansed as `cleanse_header` cleanses a header row without a
+    # schema, see `IOStreams::Tabular::Header#cleanse!`, so that a field named as the cleansing named its column, such
+    # as `area_m` for the column `Area (m²)`, matches the column, see #field_for.
+    #
+    # Unlike the cleansing of a header row without a schema, which keeps only ASCII letters and digits, it keeps the
+    # letters and digits of every language, as IOStreams 3.0 cleanses a header row, so that a field named as 3.0 names
+    # the column matches it in both. The header row of a file without a schema is still cleansed as before.
+    def self.cleansed(label)
+      # Unlike `strip` and `\s`, which are only ASCII, `[[:space:]]` is whitespace of every language, such as a
+      # no-break space or an ideographic space, so that it separates words like a space. Like `strip`, also
+      # removes a leading or trailing null. The end is found by the last other character, rather than by matching
+      # what follows it, which takes time that grows with the square of the length of whitespace within the name.
+      cleansed = label.to_s.downcase.sub(/\A[[:space:]\0]+/, "")
+      last     = cleansed.rindex(/[^[:space:]\0]/)
+      cleansed = last ? cleansed[0..last] : +""
+      cleansed.gsub!(/[[:space:]]+/, "_")
+      cleansed.gsub!(/-+/, "_")
+      # Unlike `\W`, which is only ASCII, keeps the letters and digits of every language.
+      cleansed.gsub!(/[^[:word:]]+/, "")
+      cleansed
+    end
+
+    # Returns [IOStreams::Schema] the schema of the descriptor read from the supplied path or url, a String or a
+    # path, such as `"s3://bucket/schemas/people.json"`, whose text is JSON.
+    #
+    # A byte order mark at the start of the text is ignored, since it is not part of the JSON.
+    #
+    # Raises ArgumentError when the file is not JSON, or is not a valid schema.
+    def self.read(location)
+      path = location.is_a?(IOStreams::Stream) ? location : IOStreams.path(location)
+      text = path.read
+      # Compared as bytes, since the text that a path reads is binary.
+      text = text.byteslice(BYTE_ORDER_MARK.bytesize..) if text.byteslice(0, BYTE_ORDER_MARK.bytesize).b == BYTE_ORDER_MARK
+      begin
+        new(JSON.parse(text))
+      rescue JSON::ParserError, ArgumentError => e
+        # The path within its store, without the credentials that the url of a store can hold.
+        name = path.is_a?(IOStreams::Path) ? path.path : path.inspect
+        raise(ArgumentError, "Invalid schema in #{name}: #{e.message}")
+      end
+    end
+
+    # Returns [Hash] the descriptor of the schema of the layout of a fixed width file, set with the deprecated
+    # `IOStreams::Stream#format_options`, so that the layout held by an application, such as the format options of a
+    # Rocket Job category, can be replaced with its schema. The names of its fields are the keys of the layout, so that
+    # the records read with it have the same keys. See `IOStreams::Tabular::Parser::Fixed.schema` for the layout,
+    # whose columns can have String keys, such as a layout read from JSON.
+    #
+    # The schema reads and writes the file as IOStreams 3.0 does, which is not always as the layout does: for
+    # example, the width of each field counts the characters of the UTF-8 text of the line, where the layout counts its
+    # bytes, and a number field that holds no number raises, where the layout reads it as zero. So check that the
+    # schema reads the files of the application before it replaces the layout. See "Moving a layout to a schema" in
+    # docs/upgrading.md for each difference.
+    #
+    # Unlike `IOStreams::Stream#format_options`, it reports no deprecation warning, since it replaces the layout, and
+    # Rocket Job 7.1 calls it. IOStreams 3.0 removes it, with the layout, so convert each layout with it once, and save
+    # the schema that it returns, rather than call it each time.
+    #
+    # Parameters:
+    #   format_options:
+    #     The format options of the layout, as `IOStreams::Stream#format_options` takes them: `layout:`, and
+    #     optionally `truncate:`, see `IOStreams::Tabular::Parser::Fixed.schema`.
+    #
+    # Raises `IOStreams::Errors::InvalidLayout` when the layout is not valid, and ArgumentError for a property that a
+    # column of a layout does not have, for any other format option, or when its schema is not valid.
+    def self.from_layout(**format_options)
+      descriptor = IOStreams::Tabular::Parser::Fixed.schema(**format_options)
+      new(descriptor)
+      descriptor
+    end
+
+    # Returns [String|nil] the text by which a column is compared loosely to the names and aliases of the fields, see
+    # #field_for: its letters and digits, in every language, without their case, and with the forms of a character that
+    # Unicode treats as the same made the same, such as the full width `ＩＤ` and `ID`. So `First Name`, `first_name`,
+    # `first.name` and `FIRSTNAME` are the same, as are `E-mail` and `email`.
+    #
+    # Returns nil when the label has no letters or digits, such as `#`, so that it is not compared loosely, or is not
+    # valid text in its encoding.
+    def self.match_key(label)
+      text = label.is_a?(String) ? label : label.to_s
+      return unless text.valid_encoding?
+
+      key =
+        if text.ascii_only?
+          text.downcase(:ascii).gsub(UNMATCHED_ASCII, "")
+        elsif text.encoding == Encoding::BINARY
+          text.downcase(:ascii).gsub(UNMATCHED_BINARY, "")
+        else
+          text.encode(Encoding::UTF_8).unicode_normalize(:nfkc).downcase(:fold).gsub(UNMATCHED, "")
+        end
+      key unless key.empty?
+    rescue EncodingError
+      nil
+    end
+
+    # Parameters:
+    #   descriptor: [Hash]
+    #     The schema, see above, whose keys are Strings or Symbols.
+    #
+    # Raises ArgumentError when the descriptor is not a valid schema.
+    def initialize(descriptor)
+      raise(ArgumentError, "A schema must be a Hash: #{descriptor.inspect}") unless descriptor.is_a?(Hash)
+
+      descriptor = descriptor.transform_keys(&:to_s)
+      check_properties(descriptor)
+      @fields       = build_fields(descriptor["fields"])
+      @header       = check_header(descriptor["header"])
+      @fields_match = check_fields_match(descriptor["fieldsMatch"])
+      @width        = check_width(descriptor["width"])
+      @keys         = Keys.new(descriptor["keyType"], @fields)
+      @keep_extra   = check_extra_columns(descriptor["extraColumns"])
+      @named        = build_named
+      @loose        = build_loose { |name| self.class.match_key(name) }
+      @cleansed     = build_loose { |name| self.class.match_key(self.class.cleansed(name)) }
+      @decimals     = @fields.any?(&:decimal?)
+      freeze
+    end
+
+    # Returns [true|false] whether a field is a number that keeps every digit, see `IOStreams::Schema::Field#decimal?`,
+    # so that a format that reads numbers itself, such as JSON, reads them without rounding them.
+    def decimals?
+      @decimals
+    end
+
+    # Returns [true|false] whether the file has a header row, see `header` above.
+    def header?
+      @header
+    end
+
+    # Returns [String|nil] how the keys of the records read are formed from the names of the fields, see `keyType`
+    # above, or nil when they are the names as they are given.
+    def key_type
+      @keys.type
+    end
+
+    # Returns [String|Symbol] the key in the records read of the field or column with the supplied name, formed as
+    # `keyType` says, see `IOStreams::Schema::Keys#key_for`.
+    def key_for(name)
+      @keys.key_for(name)
+    end
+
+    # Returns [String|Symbol] the key of the field in the records read, see #key_for.
+    def key(field)
+      @keys[field]
+    end
+
+    # Returns [IOStreams::Schema::Columns] a new matcher of the columns of one file to these fields, which a reader or
+    # writer holds while it reads or writes the file.
+    def columns
+      Columns.new(self)
+    end
+
+    # Returns [IOStreams::Schema::Field] the field that the supplied column of a header row, or key of a record, names:
+    # the field whose name, alias or key, see #key, it is, whether it is a String or a Symbol, so that a record written
+    # with the keys of the records read is written, otherwise the field whose name or alias
+    # is the same once both are compared loosely, see .match_key, so that the field `first_name` is the column
+    # `First Name`, and the field `E-mail` is the key `email`, otherwise the field whose name or alias is the same once
+    # both are cleansed, see .cleansed, and then compared loosely. Cleansing drops the characters that Unicode makes
+    # digits when it compares them loosely, such as `²`, `½` and `①`, so that the field `area_m`, named as
+    # `cleanse_header` named the column, is still the column `Area (m²)`, and the field `area_m2` is too.
+    #
+    # A column that is the same as the names or aliases of several fields once compared loosely, or as none, is the
+    # field whose name or alias is the column once cleansed, if there is one, so that a schema whose fields are named as
+    # `cleanse_header` named the columns finds every column that it found, such as `E-mail` for the field `e_mail`
+    # beside the field `email`.
+    #
+    # Returns nil when no field is the column, including a column whose bytes are not valid in its encoding, which is
+    # only the field whose name it is.
+    #
+    # Raises `IOStreams::Errors::InvalidHeader` when the column is the same as the names or aliases of several fields
+    # once compared loosely, and is not the name or alias of one of them once cleansed, since either could be meant.
+    # The noun, such as "column" or "key", names it.
+    def field_for(label, noun = "column")
+      return if label.nil?
+
+      @named[label.is_a?(String) ? label : label.to_s] || loose_field(label, noun)
+    end
+
+    # Raises `IOStreams::Errors::InvalidHeader` unless the columns of a header row, or the keys of a record, match the
+    # fields as `fieldsMatch` requires, and every required field is among them, see `constraints` in
+    # `IOStreams::Schema::Field`.
+    #
+    # Parameters:
+    #   found: [Array<IOStreams::Schema::Field>]
+    #     The fields of the columns, in the order of the columns, see #field_for.
+    #
+    #   unknown: [Array]
+    #     The columns that are not fields.
+    #
+    #   ordered: [true|false]
+    #     Whether the order of the columns matters, as it does for a header row, but not for the keys of a record.
+    #
+    #   noun: [String]
+    #     What the columns are, "column" or "key", for the message.
+    def check_columns(found, unknown, ordered:, noun:)
+      problems = []
+      problems << unknown_problem(unknown, noun) unless unknown.empty? || skip_unknown?
+      missing = fields - found
+      if every_field? && !missing.empty?
+        problems << "missing the #{plural('field', missing)} #{names_of(missing)}, which fieldsMatch #{fields_match.inspect} " \
+                    "requires; \"superset\" and \"partial\" accept missing fields"
+      else
+        required = missing.select(&:required?)
+        problems << "missing the required #{plural('field', required)} #{names_of(required)}" unless required.empty?
+      end
+      if fields_match == "partial" && found.empty?
+        problems << "no #{noun} is a field of the schema, but fieldsMatch \"partial\" needs at least one"
+      end
+      if ordered && fields_match == "exact" && problems.empty? && found != fields
+        problems << "the #{noun}s are not in the order of the fields, #{names_of(fields)}, which fieldsMatch \"exact\" " \
+                    "requires; \"equal\" accepts any order"
+      end
+      raise(IOStreams::Errors::InvalidHeader, "#{subject(noun)} #{problems.join('; ')}") unless problems.empty?
+    end
+
+    # Raises `IOStreams::Errors::InvalidHeader` when a record or row that is written has columns, keys or values that
+    # are not fields, unless `fieldsMatch` skips them. A file that is written always has every field, so this is the
+    # only part of `fieldsMatch` that applies to writing. Reading also applies it to the values of a row after its
+    # last column, see `IOStreams::Schema::Columns#row`.
+    #
+    # Parameters:
+    #   unknown: [Array]
+    #     The keys or columns that are not fields, or the values after the last field or column of a row.
+    #
+    #   noun: [String]
+    #     What they are, "key", "column" or "value", for the message.
+    #
+    #   last: [String]
+    #     What the values of a row come after, "field" or "column", for the message.
+    #
+    #   fields_in_order: [true|false]
+    #     Whether the columns of the file are the fields in order, see #check_fields_in_order, so that the message says
+    #     why `header: false` does not accept them, rather than suggest a `fieldsMatch` that it does not accept.
+    def check_unknown(unknown, noun:, last: "field", fields_in_order: false)
+      return if unknown.empty? || skip_unknown?
+
+      problem = unknown_problem(unknown, noun, last: last, fields_in_order: fields_in_order)
+      raise(IOStreams::Errors::InvalidHeader, "#{subject(noun)} #{problem}")
+    end
+
+    # Raises `IOStreams::Errors::InvalidHeader` unless the columns of a header row that is written before its rows, see
+    # `IOStreams::Schema::Columns#match_written`, match the fields: a column that is not a field raises, unless
+    # `fieldsMatch` skips it, see #check_unknown, and so does a header row without a column that is a field, whatever
+    # `fieldsMatch` is, as reading with "partial" needs at least one, since the rows after it would have no value for
+    # any field, such as a row of values written as a header row.
+    #
+    # Parameters:
+    #   found: [Array<IOStreams::Schema::Field>]
+    #     The fields of the columns, see #check_columns.
+    #
+    #   unknown: [Array]
+    #     The columns that are not fields.
+    #
+    #   fields_in_order: [true|false]
+    #     See #check_unknown.
+    def check_written_columns(found, unknown, fields_in_order: false)
+      check_unknown(unknown, noun: "column", fields_in_order: fields_in_order)
+      return unless found.empty?
+
+      columns =
+        if unknown.empty?
+          "it has no columns"
+        else
+          "#{describe(unknown, 'column')} #{unknown.size == 1 ? 'is not a field' : 'are not fields'} of the schema"
+        end
+      raise(IOStreams::Errors::InvalidHeader, "#{subject('column')} #{columns}, so the rows after it would have no value " \
+                                              "for any field. A header row names the column of each value of the rows " \
+                                              "after it, so give it the names of the fields, or of their columns, or " \
+                                              "write rows of the values of the fields, in order, without one")
+    end
+
+    # Raises ArgumentError unless the columns of a file can be the fields in order, as they are for a format with a
+    # header row, such as CSV, whose file has none, see `header` above: the schema needs fields, and `fieldsMatch`
+    # "exact", since a file without a header row has no names to match its columns by. A format without a header row,
+    # such as JSON, ignores `header`, so these rules only apply to a format with one, once it is known, see
+    # `IOStreams::Schema::Columns#match_fields`.
+    def check_fields_in_order
+      raise(ArgumentError, "A schema with header: false needs fields, which are the columns of the file") if fields.empty?
+      return if fields_match == "exact"
+
+      raise(ArgumentError, "A schema with header: false has the fields in order, so fieldsMatch must be \"exact\": " \
+                           "#{fields_match.inspect}")
+    end
+
+    # Returns [true|false] whether the columns that the schema does not list are skipped, rather than raise, see
+    # `fieldsMatch` above.
+    def skip_unknown?
+      SKIPS_UNKNOWN.include?(fields_match)
+    end
+
+    # Returns [true|false] whether reading keeps the columns that the schema does not list, see `extraColumns` above.
+    def keep_extra?
+      @keep_extra
+    end
+
+    # Returns [true|false] whether reading converts the value of every column that it reads to the type of its field,
+    # rather than read it as it is: no field of type "any", apart from one that it skips, which reading leaves out, and no
+    # columns kept that the schema does not list, see `extraColumns` above.
+    def types_every_column?
+      !keep_extra? && fields.none? { |field| field.type == "any" && !field.skip? }
+    end
+
+    private
+
+    # Returns [true|false] whether every field must be one of the columns.
+    def every_field?
+      NEEDS_EVERY_FIELD.include?(fields_match)
+    end
+
+    def subject(noun)
+      case noun
+      when "key" then "The keys of a record do not match the schema:"
+      when "value" then "The values of a row do not match the schema:"
+      else "The header row does not match the schema:"
+      end
+    end
+
+    # Returns [String] why the columns, keys or values are not accepted, and the values of `fieldsMatch` that skip them,
+    # unless the columns of the file are the fields in order, see #check_fields_in_order, so that it only accepts
+    # "exact".
+    def unknown_problem(unknown, noun, last: "field", fields_in_order: false)
+      text =
+        if noun == "value"
+          "#{unknown.size} #{plural('value', unknown)} after the last #{last}"
+        else
+          "#{describe(unknown, noun)} #{unknown.size == 1 ? 'is not a field' : 'are not fields'} of the schema"
+        end
+      if fields_in_order
+        return "#{text}, which a schema with header: false does not accept, since its columns are the fields in order"
+      end
+
+      "#{text}, which fieldsMatch #{fields_match.inspect} does not accept; \"subset\" and \"partial\" skip them"
+    end
+
+    # Returns [String] the columns, with those without a name counted, since they cannot be named, which are blank, see
+    # `IOStreams::Utils.blank_value?`.
+    def describe(labels, noun)
+      blank, named = labels.partition { |label| IOStreams::Utils.blank_value?(label) }
+      parts = []
+      parts << "the #{plural(noun, named)} #{sentence(named.map(&:inspect))}" unless named.empty?
+      parts << (blank.size == 1 ? "a #{noun} without a name" : "#{blank.size} #{noun}s without a name") unless blank.empty?
+      parts.join(" and ")
+    end
+
+    def names_of(fields)
+      sentence(fields.map { |field| field.name.to_s.inspect })
+    end
+
+    def plural(noun, items)
+      items.size == 1 ? noun : "#{noun}s"
+    end
+
+    def sentence(parts)
+      parts.size <= 2 ? parts.join(" and ") : "#{parts[0..-2].join(', ')} and #{parts.last}"
+    end
+
+    # Returns [IOStreams::Schema::Field] the field whose name or alias is the label once both are compared loosely, or
+    # else once both are cleansed and compared loosely, see #field_for. When that finds no field, or several, the field
+    # whose name or alias is the label once cleansed, if there is one, see .cleansed, as the deprecated `cleanse_header`
+    # named the column, so that `E-mail` is the field `e_mail` rather than either of `e_mail` and `email`, and ` _ ` is
+    # the field `_`, which has no letters or digits to compare loosely. A label whose bytes are not valid in its
+    # encoding is no field, since its text cannot be compared, nor cleansed.
+    def loose_field(label, noun)
+      text = label.is_a?(String) ? label : label.to_s
+      return unless text.valid_encoding?
+
+      key    = self.class.match_key(text)
+      fields = key && @loose[key]
+      return fields.first if fields&.size == 1
+
+      cleansed = self.class.cleansed(text)
+      fields ||= key && @cleansed[self.class.match_key(cleansed)]
+      return fields.first if fields&.size == 1
+
+      named = @named[cleansed]
+      return named if named
+      return if fields.nil?
+
+      raise(
+        IOStreams::Errors::InvalidHeader,
+        "The #{noun} #{label.inspect} could be any of the fields #{names_of(fields)}, which are the same once compared " \
+        "without their case, spaces and punctuation. Rename the #{noun}, or give the field that it is an alias that is " \
+        "the #{noun} exactly"
+      )
+    end
+
+    def check_properties(descriptor)
+      descriptor.each_key do |name|
+        next if PROPERTIES.include?(name) || DESCRIPTIVE_PROPERTIES.include?(name) || name.include?(":")
+
+        raise(ArgumentError, "The schema property #{name.inspect} is not supported") if UNSUPPORTED_PROPERTIES.include?(name)
+
+        raise(ArgumentError, "Unknown schema property: #{name.inspect}")
+      end
+    end
+
+    def build_fields(fields)
+      raise(ArgumentError, "A schema must have an Array of fields: #{fields.inspect}") unless fields.is_a?(Array)
+
+      fields = fields.collect { |field| Field.new(field) }
+      names  = fields.collect { |field| field.name.to_s }
+      duplicates = names.select { |name| names.count(name) > 1 }.uniq
+      raise(ArgumentError, "Field names must be unique: #{duplicates.join(', ')}") unless duplicates.empty?
+
+      fields
+    end
+
+    # Returns [Hash<String, IOStreams::Schema::Field>] the field of each name, alias and key, see #key, as text.
+    #
+    # Raises ArgumentError for an alias or key that is the name or alias of another field, since a column of that name
+    # could be either field.
+    def build_named
+      named = @fields.to_h { |field| [field.name.to_s, field] }
+      @fields.each do |field|
+        field.aliases.each { |name| add_named(named, name, field, "the alias #{name.inspect}") }
+        next unless key_type
+
+        name = key(field).to_s
+        add_named(named, name, field, "its key #{name.inspect}, which keyType #{key_type.inspect} forms,")
+      end
+      named
+    end
+
+    def add_named(named, name, field, what)
+      other = named[name]
+      if other && !other.equal?(field)
+        raise(ArgumentError, "Field #{field.name.inspect}: #{what} is already the name or an alias of the field " \
+                             "#{other.name.inspect}")
+      end
+
+      named[name] = field
+    end
+
+    # Returns [Hash<String, Array<IOStreams::Schema::Field>>] the fields whose names or aliases are the same when compared
+    # loosely, by the text that the block returns for each name or alias, see .match_key. Several fields can share one,
+    # such as `Amount ($)` and `Amount (€)`, whose names are still matched exactly.
+    def build_loose
+      loose = {}
+      @fields.each do |field|
+        [field.name, *field.aliases].each do |name|
+          key = yield(name)
+          next unless key
+
+          list = (loose[key] ||= [])
+          list << field unless list.include?(field)
+        end
+      end
+      loose
+    end
+
+    def check_header(header)
+      return true if header.nil?
+      raise(ArgumentError, "The schema header must be true or false: #{header.inspect}") unless [true, false].include?(header)
+
+      header
+    end
+
+    def check_fields_match(fields_match)
+      return "exact" if fields_match.nil?
+
+      unless FIELDS_MATCH.include?(fields_match)
+        raise(ArgumentError, "The schema fieldsMatch must be one of #{sentence(FIELDS_MATCH.map(&:inspect))}: " \
+                             "#{fields_match.inspect}")
+      end
+
+      fields_match
+    end
+
+    def check_extra_columns(extra_columns)
+      return false if extra_columns.nil?
+
+      unless EXTRA_COLUMNS.include?(extra_columns)
+        raise(ArgumentError, "The schema extraColumns must be one of #{sentence(EXTRA_COLUMNS.map(&:inspect))}: " \
+                             "#{extra_columns.inspect}")
+      end
+      return false if extra_columns == "skip"
+      return true if skip_unknown?
+
+      raise(ArgumentError, "The schema extraColumns \"keep\" keeps the columns that the schema does not list, which " \
+                           "fieldsMatch #{fields_match.inspect} does not accept; \"subset\" and \"partial\" accept them")
+    end
+
+    def check_width(width)
+      return if width.nil?
+      return :remainder if [:remainder, "remainder"].include?(width)
+      return width if width.is_a?(Integer) && width.positive?
+
+      raise(ArgumentError, "The schema width must be a positive Integer or \"remainder\": #{width.inspect}")
+    end
+  end
+end