iostreams 2.1.0 → 2.2.0

data/lib/io_streams/symmetric_encryption/reader.rb

@@ -5,6 +5,14 @@ module IOStreams
         %i[buffer_size version]
       end
 
+      # Returns [Array<Symbol>] the methods that the stream that v3.0 supplies to a reader block, in place of the one that
+      # this reader yields, a `Zlib::GzipReader` for a compressed file, otherwise a `SymmetricEncryption::Reader`,
+      # answers: those of an IO that read, see `IOStreams::Reader.v3_stream_methods`. The stream of a compressed file
+      # also answers #close in v3.0, which the block need not call, since the stream is closed once the block returns.
+      def self.v3_stream_methods(_options)
+        V3_READ_METHODS
+      end
+
       # read from a file/stream using Symmetric Encryption
       def self.stream(input_stream, **args, &)
         Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)

data/lib/io_streams/tabular/header.rb

@@ -5,6 +5,18 @@ module IOStreams
       # Column names that begin with this prefix have been rejected and should be ignored.
       IGNORE_PREFIX = "__rejected__".freeze
 
+      # The options of a header, see #initialize: the columns, and the restrictions of the columns read, which only
+      # apply when reading.
+      OPTION_NAMES      = %i[columns allowed_columns required_columns skip_unknown].freeze
+      READ_OPTION_NAMES = %i[allowed_columns required_columns skip_unknown].freeze
+
+      # Returns [Array<Symbol>] the names of the supplied options, see #initialize, that describe the columns of a file:
+      # those that are not nil, other than `skip_unknown`, which only applies to `allowed_columns`, so that on its own it
+      # never changes the columns. Any other option is left to #initialize, which raises for it.
+      def self.column_option_names(options)
+        (options.compact.keys & OPTION_NAMES) - %i[skip_unknown]
+      end
+
       attr_accessor :allowed_columns, :required_columns, :skip_unknown
       attr_reader :columns
 
@@ -118,6 +130,23 @@ module IOStreams
         end
       end
 
+      # Returns [String|nil] the first key that two of the columns are, such as "a" for the header row `a,a,b`, or for the
+      # columns `A` and `a` once cleansed, see #cleanse!, of which a Hash made by #to_hash holds the value of only the
+      # last column, or nil when each column is its own key. A blank or rejected column, which #to_hash leaves out, is
+      # not a key.
+      def repeated_key
+        return if columns.nil?
+
+        keys = {}
+        columns.each do |column|
+          next if IOStreams::Utils.blank?(column) || column.start_with?(IGNORE_PREFIX)
+          return column if keys.key?(column)
+
+          keys[column] = true
+        end
+        nil
+      end
+
       # Returns [true|false] whether `allowed_columns` or `required_columns` restrict the columns.
       def restricted?
         !allowed_columns.nil? || !required_columns.nil?

data/lib/io_streams/tabular/header_mapping.rb

@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+
+module IOStreams
+  class Tabular
+    # Maps the columns of a file without a schema, see `IOStreams::Tabular::Mapping`, by its header: the columns of its
+    # header row, or the columns supplied in place of one, and the column options, `allowed_columns`,
+    # `required_columns` and `skip_unknown`, see `IOStreams::Tabular::Header`.
+    #
+    # It reads and writes the rows and records as IOStreams 2.1 does, whose readers and writers of rows and records
+    # held the code of its reading and writing methods, see `IOStreams::Row::Reader` and the others, so that a file
+    # without a schema is read and written as before.
+    class HeaderMapping < Mapping
+      class << self
+        # Sets whether the column restrictions, the `allowed_columns`, `required_columns` and `skip_unknown` options of
+        # the header, apply to every input: also to the columns supplied, to a header row read with
+        # `cleanse_header: false`, and to the keys of each record of a format without a header row, such as JSON, see
+        # #restrict_header and #restrict, rather than only to a header row that is read and cleansed. Each mapping reads
+        # it once, when it is created, see #initialize.
+        #
+        # The application sets it with `IOStreams.enforce_column_restrictions=`, which is deprecated, and reports a
+        # deprecation warning. The mapping, which applies the setting, holds it, so that IOStreams reads it without one.
+        attr_writer :enforce_column_restrictions
+
+        # Returns [true|false] whether the column restrictions apply to every input, see .enforce_column_restrictions=.
+        def enforce_column_restrictions?
+          @enforce_column_restrictions
+        end
+      end
+
+      @enforce_column_restrictions = false
+
+      # Parameters:
+      #   cleanse: [true|false]
+      #     Whether to cleanse the header row read, see `IOStreams::Tabular::Header#cleanse!`, as the `cleanse_header`
+      #     option of `IOStreams::Tabular.new` says, which #read_header, #read_row, #read_record_line and
+      #     #restrict_columns apply, unless #read_header is told otherwise.
+      #     Default: true
+      #
+      #   See `IOStreams::Tabular::Mapping#initialize` for the others.
+      def initialize(header:, parser:, format:, cleanse: true)
+        super(header: header, parser: parser, format: format)
+        @cleanse  = cleanse
+        # Whether the columns were supplied, which a writer writes as the header row before the first row or record,
+        # see #start_rows.
+        @supplied = !header.columns.nil?
+        # Whether a reader of records has logged that the column restrictions would change what it reads, which it
+        # logs once, see #warn_restriction.
+        @warned   = false
+        # Whether the column restrictions apply to every input, read once, rather than for each record, see
+        # .enforce_column_restrictions?.
+        @enforced = HeaderMapping.enforce_column_restrictions?
+      end
+
+      # Returns [true|false] whether a header is still required in order to parse or render the current format.
+      def header?
+        @parser.requires_header? && IOStreams::Utils.blank?(header.columns)
+      end
+
+      # Returns [true|false] whether a header row show be rendered on output.
+      def requires_header?
+        @parser.requires_header?
+      end
+
+      # Returns [true|false] whether the rows read start with a header row: the header row of a file with one, until it
+      # is read, see #header?.
+      def rows_header?
+        header?
+      end
+
+      # Returns [true|false] whether the rows read are the records of the file, see `IOStreams::Tabular#record_rows?`:
+      # those of a format whose parser reads each line into a record, keyed by the names of its values, such as JSON, as
+      # its class says, see `IOStreams::Tabular::Parser::Base.keyed?`. A format that an application registers need not
+      # have a parser of that class, see `IOStreams::Tabular.register_format`, so one whose class does not say reads
+      # rows, as the class does by default.
+      def record_rows?
+        klass = @parser.class
+        klass.respond_to?(:keyed?) && klass.keyed?
+      end
+
+      # Returns [nil] only the fields of a schema are the columns of a file without a header row, see
+      # `IOStreams::Tabular#headerless_columns`.
+      def headerless_columns
+        nil
+      end
+
+      # Returns [String|nil] the key of a record that two columns of the header are, see
+      # `IOStreams::Tabular::Header#repeated_key`, of which a record holds the value of only the last column.
+      def repeated_key
+        header.repeated_key
+      end
+
+      # Returns [Array] the header row/line after parsing.
+      # Returns `nil` if the row/line is blank, or a header is not required for the supplied format (:json, :hash).
+      #
+      # Notes:
+      # * Call `header?` first to determine if the header should be parsed first.
+      # * The header columns are set after parsing the row, but the header is not cleansed.
+      def parse_header(line)
+        return if IOStreams::Utils.blank?(line) || !@parser.requires_header?
+
+        header.columns = @parser.parse(line)
+      end
+
+      # Parses the header row, see #parse_header, then cleanses its columns when `cleanse`, otherwise applies the column
+      # restrictions to them, as a reader of records reads its header row, see #read_record_line. `cleanse` defaults to
+      # whether to cleanse the header row read, see #initialize.
+      #
+      # Returns [Array] the header row as read, before it is cleansed, or nil when the line is blank, or the format does
+      # not have a header row.
+      def read_header(line, cleanse: @cleanse)
+        columns = parse_header(line)
+        if cleanse
+          header.cleanse!(rename: cleanse)
+        elsif restricted?
+          restrict_header(cleanse)
+        end
+        columns
+      end
+
+      # Returns [Array] the row/line as a parsed Array of values.
+      # Returns nil if the row/line is blank.
+      def row_parse(line)
+        return if IOStreams::Utils.blank?(line)
+
+        @parser.parse(line)
+      end
+
+      # Returns [Hash<String,Object>] the line as a hash.
+      # Returns nil if the line is blank.
+      def record_parse(line)
+        line = row_parse(line)
+        header.to_hash(line) if line
+      end
+
+      # Renders the output row
+      def render(row)
+        return if IOStreams::Utils.blank?(row)
+
+        @parser.render(row, header)
+      end
+
+      # Returns [String] the header rendered for the output format
+      # Return nil if no header is required.
+      def render_header
+        return unless requires_header?
+
+        if IOStreams::Utils.blank?(header.columns)
+          raise(
+            Errors::MissingHeader,
+            "Header columns must be set before attempting to render a header for format: #{@format.inspect}"
+          )
+        end
+
+        @parser.render(header.columns, header)
+      end
+
+      # Yields [Array] each row of the line of a file of rows: the header row as it was parsed, see #parse_header, while
+      # it is expected, see #header?, otherwise the row, see #row_parse, or nil for a blank line. The header columns are
+      # then cleansed, when the header row read is cleansed, see #initialize, but the header row yielded is left as it is.
+      def read_row(line)
+        if header?
+          columns = parse_header(line)
+          header.cleanse! if @cleanse
+          yield columns
+        else
+          yield row_parse(line)
+        end
+      end
+
+      # Yields [Hash] the record of the line of a file of records, see #record_parse, or nil for a blank line, unless the
+      # line is the header row, while it is expected, see #header?, which it reads instead, see #read_header. The column
+      # restrictions are applied to the keys of each record of a format without a header row, such as JSON, when they
+      # are enforced, see .enforce_column_restrictions? and #restrict.
+      def read_record_line(line)
+        if header?
+          read_header(line)
+          nil
+        else
+          yield restrict(record_parse(line))
+        end
+      end
+
+      # Applies the column restrictions to the columns supplied, which take the place of a header row, as a reader of
+      # records does before it reads the file, see `IOStreams::Record::Reader#initialize`, renaming them when the header
+      # row read is cleansed, see #initialize.
+      def restrict_columns
+        restrict_header(@cleanse) if restricted? && !header?
+      end
+
+      # Yields [String] the header line of the columns supplied, which a writer of rows or records writes before the
+      # first row or record, see `IOStreams::Row::Writer#initialize`, when the format has a header row.
+      def start_rows
+        yield render_header if @supplied && requires_header?
+      end
+
+      alias start_records start_rows
+
+      # Yields [String|nil] the line that a writer of rows writes for the row: the header line, when the columns are not
+      # known yet, whose columns it gives, otherwise the line of the row, see #render, or nil for a blank row.
+      #
+      # Returns the result of the block.
+      def render_row(row)
+        if header?
+          # If header (columns) was not supplied as an argument, assume first line is the header.
+          header.columns = row
+          yield render_header
+        else
+          yield render(row)
+        end
+      end
+
+      # Yields [String|nil] each line that a writer of records writes for the record: the header line first, once, of
+      # the keys of the first record, when the columns are not known yet, then the line of the record, see #render.
+      #
+      # Returns the result of the block.
+      def render_record(record)
+        if header?
+          # Extract header from the keys from the first row when not supplied above.
+          header.columns = record.keys
+          yield render_header
+        end
+        yield render(record)
+      end
+
+      # Yields [String|nil] the line that a writer of rows writes for the header row of the rows after it, such as an
+      # `IOStreams::Row::HeaderRow` that `IOStreams::Stream#each(:array)` yields with a schema, or that a copy hands it,
+      # see `IOStreams::Tabular#copy_row`: as the first row, see #render_row, unless the format has no header row, such
+      # as JSON, and the columns are not known, when it takes them as the columns of the rows after it, without a line.
+      # Yields nothing when the columns were supplied, which name the values of the rows instead.
+      #
+      # Raises `IOStreams::Errors::MissingHeader` when the columns that it takes are empty.
+      # Raises `IOStreams::Errors::InvalidHeader` for a format without a header row whose rows are not records, see
+      # `IOStreams::Tabular::Parser::Base.keyed?`, such as fixed width, whose layout gives its columns by their keys,
+      # which the columns of a header row are not matched to, rather than write blank values. IOStreams 2.1 wrote it
+      # as a row, which raised it too.
+      def render_header_row(columns, &)
+        return if @supplied
+        return render_row(columns, &) if requires_header? || !header.columns.nil?
+
+        unless @parser.class.keyed?
+          raise(Errors::InvalidHeader, "The header row of the rows written to the #{@format.inspect} format cannot give " \
+                                       "its columns, which its layout gives by their keys. Describe the columns of the " \
+                                       "file with #schema instead, whose fields the columns of the header row are " \
+                                       "matched to")
+        end
+        if IOStreams::Utils.blank?(columns)
+          raise(Errors::MissingHeader, "The header row of the rows written to the #{@format.inspect} format has no columns")
+        end
+
+        header.columns = columns
+        nil
+      end
+
+      # Returns [Array<String>] the header columns once they are cleansed, see `IOStreams::Tabular::Header#cleanse!`.
+      def cleanse_header!
+        header.cleanse!
+        header.columns
+      end
+
+      # Yields nothing: only the fields of a schema are the columns of a file without a header row, see
+      # `IOStreams::Tabular#start_copy`.
+      def start_copy
+        nil
+      end
+
+      # Yields [Array|nil] each row of the line that a copy writes, see `IOStreams::Tabular#copy_row`: as #read_row yields
+      # it, but the header row as an `IOStreams::Row::HeaderRow`, which #read_row yields as an Array, as IOStreams 2.1
+      # does.
+      #
+      # Returns the result of the block.
+      def copy_row(line, &)
+        return read_row(line, &) unless header?
+
+        read_row(line) { |columns| yield(columns && IOStreams::Row::HeaderRow.new(columns)) }
+      end
+
+      private
+
+      def restricted?
+        header.restricted?
+      end
+
+      # Apply the allowed and required columns to supplied columns, or to a header row read with
+      # `cleanse_header: false`, renaming them when `cleanse`. Unless they are enforced, see
+      # .enforce_column_restrictions?, only warn when they would change the columns.
+      def restrict_header(cleanse)
+        return header.cleanse!(rename: cleanse) if @enforced
+        return if @warned
+
+        columns = header.columns
+        changed = changed_by_restriction? do
+          copy = IOStreams::Tabular::Header.new(
+            columns:          columns,
+            allowed_columns:  header.allowed_columns,
+            required_columns: header.required_columns,
+            skip_unknown:     header.skip_unknown
+          )
+          copy.cleanse!(rename: cleanse)
+          copy.columns != columns
+        end
+        warn_restriction if changed
+      end
+
+      # Formats such as JSON have no header row, so apply the allowed and required columns to each record's keys,
+      # renaming them when the header row read is cleansed, see #initialize. Unless they are enforced, see
+      # .enforce_column_restrictions?, only warn when they would change the record.
+      def restrict(record)
+        return record unless record.is_a?(Hash) && restricted? && header.columns.nil?
+        return header.restrict_hash(record, rename: @cleanse) if @enforced
+        return record if @warned
+
+        warn_restriction if changed_by_restriction? { header.restrict_hash(record, rename: @cleanse) != record }
+        record
+      end
+
+      def changed_by_restriction?
+        yield
+      rescue IOStreams::Errors::InvalidHeader
+        true
+      end
+
+      # Warn once per reader, since the same columns usually apply to every record.
+      def warn_restriction
+        @warned = true
+        IOStreams.logger&.warn(
+          "allowed_columns and required_columns are not applied to this input, but would change the records read. " \
+          "They are deprecated: describe the columns with #schema instead, which applies to every input."
+        )
+      end
+    end
+  end
+end

data/lib/io_streams/tabular/mapping.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module IOStreams
+  class Tabular
+    # Maps the columns of one file to the values of its rows and records, for `IOStreams::Tabular`, which asks it to
+    # read and render them rather than depend on what describes the columns. The Tabular chooses one when it is created:
+    #
+    # * `IOStreams::Tabular::HeaderMapping`, without a schema: by the columns of the header row, or the columns
+    #   supplied in place of one, and the column options, held by its header, see `IOStreams::Tabular::Header`.
+    # * `IOStreams::Tabular::SchemaMapping`, with a schema: by its fields, see `IOStreams::Schema::Columns`.
+    #
+    # Each one does the same things, in its own way, by the methods below, which each one implements:
+    # * Reads the lines of a file: its header row, `#parse_header` and `#read_header`, and its rows and records,
+    #   `#row_parse` and `#record_parse`, which its parser parses.
+    # * Renders the rows and records written into lines, `#render`, and the header row, `#render_header`.
+    # * Reads and writes a line at a time for the readers and writers of rows and records, see `IOStreams::Row::Reader`
+    #   and the others, which only iterate: `#read_row`, `#read_record_line` and `#restrict_columns` when reading,
+    #   `#start_copy` and `#copy_row` when the rows read are copied, `#start_rows`, `#render_row`, `#start_records` and
+    #   `#render_record` when writing, and `#render_header_row` for the header row of the rows written, such as that of
+    #   the file that is copied.
+    # * Cleanses the header row, `#cleanse_header!`, when the keys of the records are the columns of the file.
+    # * Answers the questions of the protocol: whether the file has a header row, `#requires_header?`, whether it is
+    #   still to be read, `#header?`, whether the rows read start with a header row, `#rows_header?`, whether they are
+    #   records rather than Arrays, `#record_rows?`, the columns of the rows of a file without one,
+    #   `#headerless_columns`, and the key that two columns are, `#repeated_key`.
+    #
+    # Each reader and writer has its own, through its Tabular, since it holds the columns of one file.
+    class Mapping
+      # Returns [IOStreams::Tabular::Header] the columns of the file: the header row read, or the columns supplied in
+      # place of one, and the column options, see `IOStreams::Tabular#header`.
+      attr_reader :header
+
+      # Parameters:
+      #   header: [IOStreams::Tabular::Header]
+      #     The columns of the file, and the column options of the call that reads or writes it.
+      #
+      #   parser: [IOStreams::Tabular::Parser::Base]
+      #     Parses and renders each line of the file in its format.
+      #
+      #   format: [Symbol]
+      #     The name of the format, for the messages of errors.
+      def initialize(header:, parser:, format:)
+        @header = header
+        @parser = parser
+        @format = format
+      end
+
+      # Returns [true|false] whether the header row is still to be read, see `IOStreams::Tabular#header?`.
+      def header?
+        raise NotImplementedError
+      end
+
+      # Returns [true|false] whether the file has a header row, see `IOStreams::Tabular#requires_header?`.
+      def requires_header?
+        raise NotImplementedError
+      end
+
+      # Returns [true|false] whether the rows read start with a header row, see `IOStreams::Tabular#rows_header?`.
+      def rows_header?
+        raise NotImplementedError
+      end
+
+      # Returns [true|false] whether the rows read are the records of the file, see `IOStreams::Tabular#record_rows?`.
+      def record_rows?
+        raise NotImplementedError
+      end
+
+      # Returns [Array<String|Symbol>|nil] the columns of the rows of a file without a header row, see
+      # `IOStreams::Tabular#headerless_columns`.
+      def headerless_columns
+        raise NotImplementedError
+      end
+
+      # Returns [String|nil] the key of a record that two columns are, see `IOStreams::Tabular#repeated_key`.
+      def repeated_key
+        raise NotImplementedError
+      end
+
+      # Returns [Array|nil] the header row parsed from the line, see `IOStreams::Tabular#parse_header`.
+      def parse_header(line)
+        raise NotImplementedError
+      end
+
+      # Returns [Array|nil] the header row read from the line, see `IOStreams::Tabular#read_header`.
+      def read_header(line, cleanse: nil)
+        raise NotImplementedError
+      end
+
+      # Returns [Array|nil] the row of the line, see `IOStreams::Tabular#row_parse`.
+      def row_parse(line)
+        raise NotImplementedError
+      end
+
+      # Returns [Hash|nil] the record of the line, see `IOStreams::Tabular#record_parse`.
+      def record_parse(line)
+        raise NotImplementedError
+      end
+
+      # Returns [String|Array|Hash|nil] the line of the row or record written, see `IOStreams::Tabular#render`.
+      def render(row)
+        raise NotImplementedError
+      end
+
+      # Returns [String|nil] the header row written, see `IOStreams::Tabular#render_header`.
+      def render_header
+        raise NotImplementedError
+      end
+
+      # Yields each row of the line of a file of rows, see `IOStreams::Tabular#read_row`.
+      def read_row(line)
+        raise NotImplementedError
+      end
+
+      # Yields the record of the line of a file of records, see `IOStreams::Tabular#read_record_line`.
+      def read_record_line(line)
+        raise NotImplementedError
+      end
+
+      # Applies the column restrictions to the columns supplied, see `IOStreams::Tabular#restrict_columns`.
+      def restrict_columns
+        raise NotImplementedError
+      end
+
+      # Returns [Array<String>] the header columns once they are cleansed, see `IOStreams::Tabular#cleanse_header!`.
+      def cleanse_header!
+        raise NotImplementedError
+      end
+
+      # Yields the header row that a copy writes before the first line, see `IOStreams::Tabular#start_copy`.
+      def start_copy
+        raise NotImplementedError
+      end
+
+      # Yields each row of the line that a copy writes, see `IOStreams::Tabular#copy_row`.
+      def copy_row(line)
+        raise NotImplementedError
+      end
+
+      # Yields the header line written before the first row, see `IOStreams::Tabular#start_rows`.
+      def start_rows
+        raise NotImplementedError
+      end
+
+      # Yields the line written for the row, see `IOStreams::Tabular#render_row`.
+      def render_row(row)
+        raise NotImplementedError
+      end
+
+      # Yields the line written for the header row of the rows after it, see `IOStreams::Tabular#render_header_row`.
+      def render_header_row(columns)
+        raise NotImplementedError
+      end
+
+      # Yields the header line written before the first record, see `IOStreams::Tabular#start_records`.
+      def start_records
+        raise NotImplementedError
+      end
+
+      # Yields each line written for the record, see `IOStreams::Tabular#render_record`.
+      def render_record(record)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/io_streams/tabular/parser/array.rb

@@ -3,6 +3,11 @@ module IOStreams
   class Tabular
     module Parser
       class Array < Base
+        # Returns [:typed] each row is a Ruby object, so it holds the values of a schema themselves.
+        def self.value_form
+          :typed
+        end
+
         # Returns Array
         def parse(row)
           raise(IOStreams::Errors::TypeMismatch, "Format is :array. Invalid input: #{row.class.name}") unless row.is_a?(::Array)

data/lib/io_streams/tabular/parser/base.rb

@@ -12,6 +12,30 @@ module IOStreams
           nil
         end
 
+        # Returns [true|false] whether each row of this format is a record, a Hash whose keys name its values, such as a
+        # line of JSON, rather than its values in the order of its columns, an Array, such as a line of CSV, so that a
+        # schema arranges the values of each row that is written in the form that the parser renders, see
+        # `IOStreams::Tabular::SchemaMapping`.
+        #
+        # Defined at the class level since it is a static property of the format, see `.quote_character`.
+        def self.keyed?
+          false
+        end
+
+        # Returns [:text|:json|:typed] the form in which this format writes the values of a schema, see
+        # `IOStreams::Schema::Field#render`: as text, the default.
+        def self.value_form
+          :text
+        end
+
+        # Returns [IOStreams::Tabular::Parser::Base] the parser of the lines of this format for the supplied schema, see
+        # `IOStreams::Tabular.new`: a parser of this format, whose lines are the same with a schema, which arranges the
+        # values of each row itself, see `IOStreams::Tabular::SchemaMapping`. A format whose lines depend on the schema,
+        # such as fixed width, whose fields give the positions in the line, returns a parser of its own.
+        def self.schema_parser(_schema)
+          new
+        end
+
         # Returns [true|false] whether a header row is required for this format.
         def requires_header?
           true