iostreams 2.1.0 → 2.2.0

data/lib/io_streams/tabular/parser/fixed.rb

@@ -5,6 +5,47 @@ module IOStreams
       class Fixed < Base
         attr_reader :layout, :truncate
 
+        # Returns [IOStreams::Tabular::Parser::FixedFields] the parser of the lines of a fixed width file whose schema
+        # gives the position of each field, in place of the layout of this parser, see
+        # `IOStreams::Tabular::Parser::Base.schema_parser`.
+        def self.schema_parser(schema)
+          FixedFields.new(schema)
+        end
+
+        # Returns [Hash] the schema of the layout of a fixed width file, supplied with `IOStreams::Stream#format_options`,
+        # so that an application that holds a layout can replace it with its schema, see `IOStreams::Schema.from_layout`.
+        # The layout itself is still read and written by this parser, as before.
+        #
+        # Its `fieldsMatch` is "subset", so that writing skips the keys of a record that the layout does not have, as a
+        # layout always did. A line that is read always has every field. When every key of the layout is a Symbol, its
+        # `keyType` is "symbol", so that the records read have the same keys once the schema is held as JSON, or in a
+        # database such as MongoDB, whose names are Strings, see `IOStreams::Schema`.
+        #
+        # Parameters:
+        #   layout: [Array<Hash>]
+        #     Each column of the file, in order, a Hash whose keys are Symbols or Strings, such as those of a layout read
+        #     from JSON or YAML:
+        #       size:     [Integer|:remainder] The number of characters of the column, or `:remainder` to take the
+        #                 rest of the line. The layout reads nothing after a column of the size `:remainder`, so the
+        #                 columns after it are not in the schema. Required.
+        #       key:      [String|Symbol] The key of the column in each record, which must be its own. A column without
+        #                 a key is a filler, which is skipped when reading and is spaces when writing.
+        #       type:     [:string|:integer|:float] Default: :string
+        #       decimals: [Integer] The number of decimals of a :float column, 0 or more. A float that takes the
+        #                 remainder of the line is written with the digits that it has, as the layout writes it, so
+        #                 it ignores them. Default: 2
+        #
+        #   truncate: [true|false]
+        #     Whether to truncate a string that is longer than its column, rather than raise
+        #     `IOStreams::Errors::ValueTooLong`.
+        #     Default: true
+        #
+        # Raises `IOStreams::Errors::InvalidLayout` when the layout is not valid, including when its schema could not
+        # hold it, such as for two columns with the same key, which the records read could not hold either.
+        def self.schema(layout:, **)
+          new(layout: Layout.properties(layout), **).schema
+        end
+
         # Returns [IOStreams::Tabular::Parser]
         #
         # Parameters:
@@ -58,6 +99,11 @@ module IOStreams
           @truncate = truncate
         end
 
+        # Returns [Hash] the schema of this layout, see .schema.
+        def schema
+          layout.schema(truncate)
+        end
+
         # The required line length for every fixed length line
         def line_length
           layout.length
@@ -112,6 +158,18 @@ module IOStreams
         class Layout
           attr_reader :columns, :length
 
+          # Returns [Array<Hash>] the columns of a layout held as data, such as a layout read from JSON or YAML, with
+          # the properties of each one as .new takes them, see `Column.properties`.
+          #
+          # Raises `IOStreams::Errors::InvalidLayout` when the layout is not an Array.
+          def self.properties(layout)
+            unless layout.is_a?(::Array)
+              raise(Errors::InvalidLayout, "The layout must be an Array of columns: #{layout.inspect}")
+            end
+
+            layout.map { |column| Column.properties(column) }
+          end
+
           # Returns [Array<FixedLayout>] the layout for this fixed width file.
           # Also validates values
           def initialize(layout)
@@ -119,8 +177,63 @@ module IOStreams
             @columns = parse_layout(layout)
           end
 
+          # Returns [Hash] the descriptor of the schema of this layout, whose fields are at the positions of its
+          # columns, see `IOStreams::Tabular::Parser::Fixed.schema`, with its strings truncated when `truncate` is true.
+          #
+          # A column after a filler starts after the filler, and a filler at the end of the line gives the width of the
+          # line. Since nothing is read after a column of the size `:remainder`, which takes the rest of the line, the
+          # columns after it are left out.
+          #
+          # Raises `IOStreams::Errors::InvalidLayout` when a field could not hold a column, see `Column#field`, or two
+          # columns have the same key.
+          def schema(truncate)
+            fields    = []
+            keys      = {}
+            position  = 0
+            filler    = false
+            remainder = false
+            columns.each do |column|
+              field = column.field(filler ? position + 1 : nil, truncate)
+              if field
+                check_key(field[:name], keys)
+                fields << field
+              end
+              filler    = field.nil?
+              remainder = column.size == -1
+              break if remainder
+
+              position += column.size
+            end
+
+            schema = {fields: fields, fieldsMatch: "subset"}
+            # A filler at the end of the line gives the width of the line.
+            schema[:width] = remainder ? "remainder" : position if filler
+            # Symbol keys stay Symbols once the schema is held as JSON, whose names are Strings.
+            schema[:keyType] = "symbol" if !fields.empty? && fields.all? { |field| field[:name].is_a?(Symbol) }
+            schema
+          end
+
           private
 
+          # Raises `IOStreams::Errors::InvalidLayout` when another column of the layout has the same key as text, see
+          # #schema, since the key of each column is its key in each record, and the name of its field. Adds the key to
+          # the keys of the columns before it, by their text.
+          def check_key(key, keys)
+            other = keys[key.to_s]
+            keys[key.to_s] = key
+            return if other.nil?
+
+            subject =
+              if other.eql?(key)
+                "More than one column of the layout has the key #{key.inspect}"
+              else
+                "The columns of the layout with the keys #{other.inspect} and #{key.inspect} have the same key"
+              end
+            raise(Errors::InvalidLayout, "#{subject}, but each column needs a key of its own, its key in each record. A " \
+                                         "column without a key is a filler, which is skipped when reading and written as " \
+                                         "spaces, so remove the key of a filler.")
+          end
+
           def parse_layout(layout)
             @length = 0
             layout.collect do |hash|
@@ -144,9 +257,46 @@ module IOStreams
         class Column
           TYPES = %i[string integer float].freeze
           LINE_BREAK = /\r\n|\r|\n/
+          # The type of the field of a schema for each type of column, see #field.
+          FIELD_TYPES = {string: "string", integer: "integer", float: "number"}.freeze
+          # The decimals of a :float column as text that #render writes: digits, or none for no decimals.
+          DECIMALS = /\A\d*\z/
 
           attr_reader :key, :size, :type, :decimals
 
+          # Returns [Hash] the properties of a column of a layout held as data, such as one read from JSON or YAML, or
+          # taken from the params of a Rails request, with Symbol keys, as .new takes them.
+          #
+          # Raises `IOStreams::Errors::InvalidLayout` when the column is not a Hash, has a property both as a String and
+          # as a Symbol, since either could be meant, or has a size or a type that .new cannot convert, such as true or
+          # nil, for which .new raises NoMethodError, as it did before.
+          def self.properties(column)
+            unless column.is_a?(::Hash)
+              raise(Errors::InvalidLayout, "Each column of the layout must be a Hash, such as {size: 10, key: :name}: " \
+                                           "#{column.inspect}")
+            end
+
+            # A plain Hash, since a Hash of another class, such as ActiveSupport::HashWithIndifferentAccess, can keep its
+            # keys as Strings.
+            properties = column.to_h.transform_keys { |name| name.is_a?(String) ? name.to_sym : name }
+            unless properties.size == column.size
+              raise(Errors::InvalidLayout, "A column of the layout has a property both as a String and as a Symbol: " \
+                                           "#{column.inspect}")
+            end
+
+            size = properties[:size]
+            unless size == :remainder || size.respond_to?(:to_i)
+              raise(Errors::InvalidLayout, "The size of a column of the layout must be a number or :remainder: " \
+                                           "#{column.inspect}")
+            end
+            if properties.key?(:type) && !properties[:type].respond_to?(:to_sym)
+              raise(Errors::InvalidLayout, "The type of a column of the layout must be :string, :integer or :float: " \
+                                           "#{column.inspect}")
+            end
+
+            properties
+          end
+
           def initialize(size:, key: nil, type: :string, decimals: 2)
             @key      = key
             @size     = [:remainder, "remainder"].include?(size) ? -1 : size.to_i
@@ -205,6 +355,53 @@ module IOStreams
 
             formatted
           end
+
+          # Returns [Hash|nil] the field of a schema for this column, named by its key, with its size, type and decimals,
+          # see `IOStreams::Tabular::Parser::Fixed.schema`, or nil for a filler, which has no key.
+          #
+          # Parameters:
+          #   start: [Integer|nil]
+          #     The position of the first character of the column, counting from 1, or nil when it is right after the
+          #     field before it.
+          #
+          #   truncate: [true|false]
+          #     Whether #render truncates a string that is longer than its column.
+          #
+          # Raises `IOStreams::Errors::InvalidLayout` when a field cannot hold the column: for a key that is neither a
+          # String nor a Symbol, which cannot name a field, or for decimals that #render cannot write.
+          def field(start, truncate)
+            return unless key
+
+            unless key.is_a?(String) || key.is_a?(Symbol)
+              raise(Errors::InvalidLayout, "The key of a column of the layout must be a String or a Symbol, such as " \
+                                           "#{key.to_s.inspect}, or none for a filler: #{key.inspect}")
+            end
+
+            field = {name: key, type: FIELD_TYPES[type], width: size == -1 ? "remainder" : size}
+            field[:start] = start if start
+            case type
+            when :string then field[:truncate] = true if truncate
+            when :float
+              field[:floatNumber] = true
+              # A float that takes the remainder of the line is written with the digits that it has.
+              field[:decimals] = field_decimals unless size == -1
+            end
+            field
+          end
+
+          private
+
+          # Returns [Integer] the decimals of a :float column, see #field, as #render writes them: nil or empty text
+          # for none.
+          #
+          # Raises `IOStreams::Errors::InvalidLayout` for decimals that #render cannot write, such as -1, 2.5 or "abc".
+          def field_decimals
+            return decimals if decimals.is_a?(Integer) && !decimals.negative?
+            return decimals.to_i if decimals.nil? || (decimals.is_a?(String) && DECIMALS.match?(decimals))
+
+            raise(Errors::InvalidLayout, "The decimals of the column #{key.inspect} of the layout must be a whole " \
+                                         "number of 0 or more: #{decimals.inspect}")
+          end
         end
       end
     end

data/lib/io_streams/tabular/parser/fixed_fields.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module IOStreams
+  class Tabular
+    module Parser
+      # Parses and renders the lines of a fixed width file whose schema gives the position of each field in every line,
+      # with the `width`, and optionally the `start`, of each field, see `IOStreams::Schema`. Each row is the values of
+      # the fields, in their order, as for CSV, since a line has no names, so the schema names them, see
+      # `IOStreams::Tabular::SchemaMapping`. A fixed width file without a schema is still parsed by the layout of
+      # `IOStreams::Tabular::Parser::Fixed`, as before.
+      #
+      # When reading, each value is the text at the position of its field without its leading and trailing spaces,
+      # which the schema then converts to the type of its field.
+      #
+      # When writing, a number is right justified and zero filled, and a missing number is written as zero, as most
+      # specifications of fixed width files require, except a number that is not finite, such as NaN, which is space
+      # filled, so that it can be read back. Any other value is left justified and space filled, with each
+      # line break replaced with a space, so that a value cannot add records. The characters that no field covers,
+      # such as fillers, are spaces.
+      #
+      # The positions count the characters of the line, which a schema reads as UTF-8, unless an encoding is set,
+      # while the layout counts the bytes of the binary line, see `IOStreams::Schema.from_layout`.
+      class FixedFields < Base
+        LINE_BREAK = /\r\n|\r|\n/
+        DIGIT      = /\d/
+
+        # Where a field is in each line: the offset of its first character, and its width, or -1 for the remainder of
+        # the line.
+        Column = Struct.new(:field, :index, :width)
+
+        # Returns [Integer] the length of every line, or -1 when a line can be any length, after a field or a schema
+        # width of "remainder", or 0 when there are no fields.
+        attr_reader :line_length
+
+        # Parameters:
+        #   schema: [IOStreams::Schema]
+        #     The schema, which must give the width of each field.
+        #
+        # Raises ArgumentError when the schema does not give the position of each field.
+        def initialize(schema)
+          super()
+          @schema      = schema
+          @columns     = []
+          @line_length = build_columns
+        end
+
+        # Returns [Array] the text of each field of the line, in the order of the fields, without its leading and
+        # trailing spaces, or the row supplied, an Array that is already parsed, as it is, as CSV does, see
+        # `IOStreams::Tabular#record_parse`.
+        def parse(line)
+          return line if line.is_a?(::Array)
+          unless line.is_a?(String)
+            raise(Errors::TypeMismatch, "Line must be a String when format is :fixed. Actual: #{line.class.name}")
+          end
+
+          if line_length.positive? && line.length != line_length
+            raise(Errors::InvalidLineLength, "Expected line length: #{line_length}, actual line length: #{line.length}")
+          end
+
+          @columns.map do |column|
+            value = column.width == -1 ? line[column.index..] : line[column.index, column.width]
+            value&.strip
+          end
+        end
+
+        # Returns [String] the line of the values of the fields, in the order of the fields, which the schema has already
+        # rendered as text, see `IOStreams::Tabular::SchemaMapping#render`, so that the header supplied is not needed.
+        def render(row, _header)
+          line = +""
+          @columns.each_with_index do |column, index|
+            line << (" " * (column.index - line.length)) if column.index > line.length
+            line << render_value(column, row[index])
+          end
+          line << (" " * (line_length - line.length)) if line_length > line.length
+          line
+        end
+
+        # The header is required as an argument and cannot be supplied in the file itself.
+        def requires_header?
+          false
+        end
+
+        private
+
+        # Returns [Integer] the length of every line, see #line_length, once it works out where each field is.
+        def build_columns
+          position = 0
+          @schema.fields.each do |field|
+            if position == -1
+              raise(ArgumentError, "Only the last field can have a width of \"remainder\", " \
+                                   "but field #{field.name.inspect} follows it")
+            end
+            raise(ArgumentError, "Field #{field.name.inspect} needs a width for the :fixed format") if field.width.nil?
+
+            index = field.start ? field.start - 1 : position
+            if index < position
+              raise(ArgumentError, "Field #{field.name.inspect} starts at #{field.start}, within the field before it, " \
+                                   "which ends at #{position}")
+            end
+
+            size = field.width == :remainder ? -1 : field.width
+            @columns << Column.new(field, index, size)
+            position = size == -1 ? -1 : index + size
+          end
+          schema_line_length(position)
+        end
+
+        def schema_line_length(position)
+          width = @schema.width
+          return position if width.nil?
+          return -1 if width == :remainder
+
+          if position == -1
+            raise(ArgumentError, "The schema width cannot be set when the last field takes the remainder of the line")
+          end
+          if width < position
+            raise(ArgumentError, "The schema width #{width} is less than the end of the last field, at #{position}")
+          end
+
+          width
+        end
+
+        def render_value(column, text)
+          field = column.field
+          return render_number(column, text.nil? ? field.render(0, :text) : text.to_s) if field.numeric?
+
+          text = text.to_s
+          text = text.gsub(LINE_BREAK, " ") if LINE_BREAK.match?(text)
+          return text if column.width == -1
+          return text.ljust(column.width) if text.length <= column.width
+          raise(value_too_long(column, text)) unless field.truncate?
+
+          text[0, column.width]
+        end
+
+        def render_number(column, text)
+          return text if column.width == -1
+
+          padded =
+            # A number that is not finite, such as NaN or -INF, has no digits to zero fill, so that it can be read back.
+            if !DIGIT.match?(text)
+              text.rjust(column.width)
+            elsif text.start_with?("-")
+              "-#{text.delete_prefix('-').rjust(column.width - 1, '0')}"
+            else
+              text.rjust(column.width, "0")
+            end
+          raise(value_too_long(column, text)) if padded.length > column.width
+
+          padded
+        end
+
+        def value_too_long(column, text)
+          message = "Value: #{text} is too large to fit into column:#{column.field.name} of size:#{column.width}"
+          Errors::ValueTooLong.new(message)
+        end
+      end
+    end
+  end
+end

data/lib/io_streams/tabular/parser/hash.rb

@@ -3,6 +3,16 @@ module IOStreams
   class Tabular
     module Parser
       class Hash < Base
+        # Returns [true] each row is a Hash, whose keys name its values.
+        def self.keyed?
+          true
+        end
+
+        # Returns [:typed] each row is a Ruby object, so it holds the values of a schema themselves.
+        def self.value_form
+          :typed
+        end
+
         def parse(row)
           raise(IOStreams::Errors::TypeMismatch, "Format is :hash. Invalid input: #{row.class.name}") unless row.is_a?(::Hash)
 

data/lib/io_streams/tabular/parser/json.rb

@@ -4,6 +4,23 @@ module IOStreams
     module Parser
       # For parsing a single line of JSON at a time
       class Json < Base
+        # Returns [true] each line is a JSON object, whose keys name its values.
+        def self.keyed?
+          true
+        end
+
+        # Returns [:json] JSON writes the numbers and booleans of a schema as themselves, and every other value as text.
+        def self.value_form
+          :json
+        end
+
+        # Returns [IOStreams::Tabular::Parser::JsonFields] the parser of the lines of JSON with the supplied schema,
+        # which reads every digit of the numbers of its number fields, and writes them, see
+        # `IOStreams::Tabular::Parser::Base.schema_parser`.
+        def self.schema_parser(schema)
+          JsonFields.new(schema)
+        end
+
         def parse(row)
           return row if row.is_a?(::Hash)
 

data/lib/io_streams/tabular/parser/json_fields.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "json"
+module IOStreams
+  class Tabular
+    module Parser
+      # Parses and renders each line of JSON of a file with a schema, see `IOStreams::Stream#schema`, a record whose keys
+      # the schema matches to its fields, see `IOStreams::Tabular::SchemaMapping`, which converts its values:
+      # * Reads the numbers of each number field with every digit of their text, as BigDecimals, rather than through a
+      #   Float, which keeps about 16, while every other number is still a Float, as without a schema.
+      # * Raises for a line that is not an object, since only a record has the keys that name its values.
+      # * Writes the numbers that the schema renders with every digit, see `IOStreams::Schema::JsonNumber`.
+      #
+      # A file without a schema is still parsed by `IOStreams::Tabular::Parser::Json`, as before.
+      class JsonFields < Json
+        # Returns [JSON::Coder|nil] a coder that writes a record as `IOStreams::Schema::Field.any_json(record).to_json`
+        # does, in one pass over its values in the json gem, rather than one pass in Ruby, then another to write them.
+        #
+        # Returns nil before json 2.10, which added `JSON::Coder`, for a version whose coder does not write a key that is
+        # not a String the same way, and when another library, such as ActiveSupport, has replaced `Hash#to_json`, since
+        # the coder would not write the JSON that it does.
+        def self.coder
+          return unless ::Hash.instance_method(:to_json).owner.name&.start_with?("JSON::")
+          return @coder if defined?(@coder)
+
+          @coder = build_coder
+        end
+
+        def self.build_coder
+          return unless defined?(::JSON::Coder) && defined?(::JSON::Fragment)
+
+          coder = ::JSON::Coder.new do |value, key|
+            # Like `Hash#to_json`, which writes a key that is not a String with `to_s`.
+            next value.to_s if key
+
+            json = IOStreams::Schema::Field.any_json(value)
+            json.equal?(value) ? ::JSON::Fragment.new(value.to_json) : json
+          end
+          coder if coder.dump({1 => nil}) == '{"1":null}'
+        rescue StandardError
+          nil
+        end
+        private_class_method :build_coder
+
+        # The most keys whose fields it keeps, see #decimal_key?.
+        DECIMAL_KEYS = 1000
+
+        # Parameters:
+        #   schema: [IOStreams::Schema]
+        #     The fields of the records, which say which numbers keep every digit, see `IOStreams::Schema#decimals?`.
+        def initialize(schema)
+          super()
+          @schema = schema
+          @coder  = self.class.coder
+          # Whether each key of the records is the column of a number field that keeps every digit, see #parse.
+          @decimal_keys = {} if schema.decimals?
+        end
+
+        # Returns [Hash] the record of the line, or nil for a line of `null`, which is a blank line. Returns the record or
+        # row supplied, a Hash, or an Array that is already parsed, as it is, see `IOStreams::Tabular#record_parse`.
+        #
+        # With a schema that has a number field that keeps every digit, see `IOStreams::Schema#decimals?`, the numbers
+        # of its column are read as BigDecimals, with every digit of their text, rather than through a Float, which keeps
+        # about 16. Every other number is still a Float, the same as without a schema.
+        #
+        # Raises `IOStreams::Errors::TypeMismatch` for a line that is not an object, such as `[1,2]` or `5`, since only an
+        # object has the keys that name its values.
+        def parse(row)
+          # Each line read is a String, so that is checked first.
+          unless row.is_a?(String)
+            return row if row.is_a?(::Hash) || row.is_a?(::Array)
+
+            raise(IOStreams::Errors::TypeMismatch, "Format is :json. Invalid input: #{row.class.name}")
+          end
+
+          values = @decimal_keys ? parse_decimals(row) : JSON.parse(row)
+          return values if values.is_a?(::Hash) || values.nil?
+
+          raise(IOStreams::Errors::TypeMismatch, "Each line of the :json format must be an object, whose keys name its " \
+                                                 "values, rather than #{values.class.name}")
+        end
+
+        # Returns [String] the record, by the names of its fields, as a single line JSON string, once the schema has
+        # rendered its values, see `IOStreams::Schema::Field#render`, which the header supplied leaves as they are, see
+        # `IOStreams::Tabular::SchemaMapping#render`.
+        #
+        # A BigDecimal, Date or Time is written as its text, see `IOStreams::Schema::Field.any_json`, such as "12.5"
+        # rather than "0.125e2", so that a BigDecimal keeps all of its digits. A Float that JSON cannot hold, such as
+        # `Float::NAN`, never reaches it, since the field of its value raises `IOStreams::Errors::InvalidValue` for it,
+        # see `IOStreams::Schema::Field#render`.
+        def render(row, _header)
+          @coder ? @coder.dump(row) : IOStreams::Schema::Field.any_json(row).to_json
+        end
+
+        private
+
+        # Returns the value of the line, with the numbers of each key that is the column of a number field that keeps
+        # every digit as BigDecimals, and every other number as a Float, see #parse.
+        def parse_decimals(line)
+          values = JSON.parse(line, decimal_class: ::BigDecimal)
+          return values unless values.is_a?(::Hash)
+
+          values.each_pair do |key, value|
+            next unless value.is_a?(::BigDecimal) || value.is_a?(::Array) || value.is_a?(::Hash)
+
+            values[key] = floats(value) unless decimal_key?(key)
+          end
+          values
+        end
+
+        # Returns [true|false] whether the key is the column of a number field that keeps every digit, worked out once
+        # for each key, see `IOStreams::Schema#field_for`.
+        def decimal_key?(key)
+          @decimal_keys.fetch(key) do
+            @decimal_keys.clear if @decimal_keys.size >= DECIMAL_KEYS
+            @decimal_keys[key] = @schema.field_for(key, "key")&.decimal? || false
+          end
+        end
+
+        # Returns the value, with each BigDecimal within it as a Float, as JSON reads a number without a schema.
+        def floats(value)
+          case value
+          when ::BigDecimal then value.to_f
+          when ::Array then value.map { |item| floats(item) }
+          when ::Hash then value.transform_values { |item| floats(item) }
+          else value
+          end
+        end
+      end
+    end
+  end
+end