iostreams 1.2.1 → 1.6.2

data/lib/io_streams/tabular.rb

@@ -52,7 +52,7 @@ module IOStreams
     #   format: [Symbol]
     #     :csv, :hash, :array, :json, :psv, :fixed
     #
-    #   file_name: [String]
+    #   file_name: [IOStreams::Path | String]
     #     When `:format` is not supplied the file name can be used to infer the required format.
     #     Optional. Default: nil
     #
@@ -81,15 +81,20 @@ module IOStreams
     #       #as_hash will skip these additional columns entirely as if they were not in the file at all.
     #     false:
     #       Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
-    def initialize(format: nil, file_name: nil, format_options: nil, **args)
+    #
+    #   default_format: [Symbol]
+    #     When the format is not supplied, and the format cannot be inferred from the supplied file name
+    #     then this default format will be used.
+    #     Default: :csv
+    #     Set to nil to force it to raise an exception when the format is undefined.
+    def initialize(format: nil, file_name: nil, format_options: nil, default_format: :csv, **args)
       @header = Header.new(**args)
-      klass   =
-        if file_name && format.nil?
-          self.class.parser_class_for_file_name(file_name)
-        else
-          self.class.parser_class(format)
-        end
-      @parser = format_options ? klass.new(format_options) : klass.new
+      @format = file_name && format.nil? ? self.class.format_from_file_name(file_name) : format
+      @format ||= default_format
+      raise(UnknownFormat, "The format cannot be inferred from the file name: #{file_name}") unless @format
+
+      klass   = self.class.parser_class(@format)
+      @parser = format_options ? klass.new(**format_options) : klass.new
     end
 
     # Returns [true|false] whether a header is still required in order to parse or render the current format.
@@ -142,7 +147,10 @@ module IOStreams
       return unless requires_header?
 
       if IOStreams::Utils.blank?(header.columns)
-        raise(Errors::MissingHeader, "Header columns must be set before attempting to render a header for format: #{format.inspect}")
+        raise(
+          Errors::MissingHeader,
+          "Header columns must be set before attempting to render a header for format: #{format.inspect}"
+        )
       end
 
       parser.render(header.columns, header)
@@ -159,9 +167,9 @@ module IOStreams
     # Example:
     #   register_format(:csv, IOStreams::Tabular::Parser::Csv)
     def self.register_format(format, parser)
-      raise(ArgumentError, "Invalid format #{format.inspect}") unless format.nil? || format.to_s =~ /\A\w+\Z/
+      raise(ArgumentError, "Invalid format #{format.inspect}") unless format.to_s =~ /\A\w+\Z/
 
-      @formats[format.nil? ? nil : format.to_sym] = parser
+      @formats[format.to_sym] = parser
     end
 
     # De-Register a file format
@@ -181,28 +189,21 @@ module IOStreams
       @formats.keys
     end
 
-    private
-
     # A registry to hold formats for processing files during upload or download
     @formats = {}
 
-    def self.parser_class(format)
-      @formats[format.nil? ? nil : format.to_sym] || raise(ArgumentError, "Unknown Tabular Format: #{format.inspect}")
+    # Returns the registered format that will be used for the supplied file name.
+    def self.format_from_file_name(file_name)
+      file_name.to_s.split(".").reverse_each { |ext| return ext.to_sym if @formats.include?(ext.to_sym) }
+      nil
     end
 
-    # Returns the parser to use with tabular for the supplied file_name
-    def self.parser_class_for_file_name(file_name)
-      format = nil
-      file_name.to_s.split(".").reverse_each do |ext|
-        if @formats.include?(ext.to_sym)
-          format = ext.to_sym
-          break
-        end
-      end
-      parser_class(format)
+    # Returns the parser class for the registered format.
+    def self.parser_class(format)
+      @formats[format.nil? ? nil : format.to_sym] ||
+        raise(ArgumentError, "Unknown Tabular Format: #{format.inspect}")
     end
 
-    register_format(nil, IOStreams::Tabular::Parser::Csv)
     register_format(:array, IOStreams::Tabular::Parser::Array)
     register_format(:csv, IOStreams::Tabular::Parser::Csv)
     register_format(:fixed, IOStreams::Tabular::Parser::Fixed)

data/lib/io_streams/tabular/header.rb

@@ -109,7 +109,10 @@ module IOStreams
         end
 
         unless row.is_a?(Array)
-          raise(IOStreams::Errors::TypeMismatch, "Don't know how to convert #{row.class.name} to an Array without the header columns being set.")
+          raise(
+            IOStreams::Errors::TypeMismatch,
+            "Don't know how to convert #{row.class.name} to an Array without the header columns being set."
+          )
         end
 
         row
@@ -126,18 +129,17 @@ module IOStreams
       # Perform cleansing on returned Hash keys during the narrowing process.
       # For example, avoids issues with case etc.
       def cleanse_hash(hash)
-        h = {}
-        hash.each_pair do |key, value|
-          cleansed_key    =
-            if columns.include?(key)
-              key
-            else
-              key = cleanse_column(key)
-              key if columns.include?(key)
-            end
-          h[cleansed_key] = value if cleansed_key
+        unmatched = columns - hash.keys
+        unless unmatched.empty?
+          hash = hash.dup
+          unmatched.each { |name| hash[cleanse_column(name)] = hash.delete(name) }
+        end
+        # Hash#slice as of Ruby 2.5
+        if hash.respond_to?(:slice)
+          hash.slice(*columns)
+        else
+          columns.each_with_object({}) { |column, new_hash| new_hash[column] = hash[column] }
         end
-        h
       end
 
       def cleanse_column(name)

data/lib/io_streams/tabular/parser/csv.rb

@@ -5,8 +5,10 @@ module IOStreams
       class Csv < Base
         attr_reader :csv_parser
 
-        def initialize
-          @csv_parser = Utility::CSVRow.new unless RUBY_VERSION.to_f >= 2.6
+        unless RUBY_VERSION.to_f >= 2.6
+          def initialize
+            @csv_parser = Utility::CSVRow.new
+          end
         end
 
         # Returns [Array<String>] the header row.

data/lib/io_streams/tabular/parser/fixed.rb

@@ -3,31 +3,77 @@ module IOStreams
     module Parser
       # Parsing and rendering fixed length data
       class Fixed < Base
-        attr_reader :fixed_layout
+        attr_reader :layout, :truncate
 
         # Returns [IOStreams::Tabular::Parser]
         #
         # Parameters:
         #   layout: [Array<Hash>]
         #     [
-        #       {key: 'name',    size: 23 },
-        #       {key: 'address', size: 40 },
-        #       {key: 'zip',     size: 5 }
+        #       {size: 23, key: "name"},
+        #       {size: 40, key: "address"},
+        #       {size:  2},
+        #       {size:  5, key: "zip"},
+        #       {size:  8, key: "age", type: :integer},
+        #       {size: 10, key: "weight", type: :float, decimals: 2}
         #     ]
-        def initialize(layout:)
-          @fixed_layout = parse_layout(layout)
+        #
+        # Notes:
+        # * Leave out the name of the key to ignore that column during parsing,
+        #   and to space fill when rendering. For example as a filler.
+        #
+        # Types:
+        #    :string
+        #      This is the default type.
+        #      Applies space padding and the value is left justified.
+        #      Returns value as a String
+        #    :integer
+        #      Applies zero padding to the left.
+        #      Returns value as an Integer
+        #      Raises Errors::ValueTooLong when the supplied value cannot be rendered in `size` characters.
+        #    :float
+        #      Applies zero padding to the left.
+        #      Returns value as a float.
+        #      The :size is the total size of this field including the `.` and the decimals.
+        #      Number of :decimals
+        #      Raises Errors::ValueTooLong when the supplied value cannot be rendered in `size` characters.
+        #
+        # In some circumstances the length of the last column is variable.
+        #   layout: [Array<Hash>]
+        #     [
+        #       {size: 23, key: "name"},
+        #       {size: :remainder, key: "rest"}
+        #     ]
+        # By setting a size of `:remainder` it will take the rest of the line as the value for that column.
+        #
+        # A size of `:remainder` and no `:key` will discard the remainder of the line without validating the length.
+        #   layout: [Array<Hash>]
+        #     [
+        #       {size: 23, key: "name"},
+        #       {size: :remainder}
+        #     ]
+        #
+        def initialize(layout:, truncate: true)
+          @layout   = Layout.new(layout)
+          @truncate = truncate
+        end
+
+        # The required line length for every fixed length line
+        def line_length
+          layout.length
         end
 
         # Returns [String] fixed layout values extracted from the supplied hash.
-        # String will be encoded to `encoding`
+        #
+        # Notes:
+        # * A nil value is considered an empty string
+        # * When a supplied value exceeds the column size it is truncated.
         def render(row, header)
           hash = header.to_hash(row)
 
           result = ""
-          fixed_layout.each do |map|
-            # A nil value is considered an empty string
-            value = hash[map.key].to_s
-            result << format("%-#{map.size}.#{map.size}s", value)
+          layout.columns.each do |column|
+            result << column.render(hash[column.key], truncate)
           end
           result
         end
@@ -36,32 +82,126 @@ module IOStreams
         # String will be encoded to `encoding`
         def parse(line)
           unless line.is_a?(String)
-            raise(IOStreams::Errors::TypeMismatch, "Format is :fixed. Invalid parse input: #{line.class.name}")
+            raise(Errors::TypeMismatch, "Line must be a String when format is :fixed. Actual: #{line.class.name}")
+          end
+
+          if layout.length.positive? && (line.length != layout.length)
+            raise(Errors::InvalidLineLength, "Expected line length: #{layout.length}, actual line length: #{line.length}")
           end
 
           hash  = {}
           index = 0
-          fixed_layout.each do |map|
-            value = line[index..(index + map.size - 1)]
-            index += map.size
-            hash[map.key] = value.to_s.strip
+          layout.columns.each do |column|
+            if column.size == -1
+              hash[column.key] = column.parse(line[index..-1]) if column.key
+              break
+            end
+
+            # Ignore "columns" that have no keys. E.g. Fillers
+            hash[column.key] = column.parse(line[index, column.size]) if column.key
+            index += column.size
           end
           hash
         end
 
-        private
+        # The header is required as an argument and cannot be supplied in the file itself.
+        def requires_header?
+          false
+        end
+
+        class Layout
+          attr_reader :columns, :length
+
+          # Returns [Array<FixedLayout>] the layout for this fixed width file.
+          # Also validates values
+          def initialize(layout)
+            @length  = 0
+            @columns = parse_layout(layout)
+          end
+
+          private
+
+          def parse_layout(layout)
+            @length = 0
+            layout.collect do |hash|
+              raise(Errors::InvalidLayout, "Missing required :size in: #{hash.inspect}") unless hash.key?(:size)
+
+              column = Column.new(**hash)
+              if column.size == -1
+                if @length == -1
+                  raise(Errors::InvalidLayout, "Only the last :size can be '-1' or :remainder in: #{hash.inspect}")
+                end
+
+                @length = -1
+              else
+                @length += column.size
+              end
+              column
+            end
+          end
+        end
+
+        class Column
+          TYPES = %i[string integer float].freeze
+
+          attr_reader :key, :size, :type, :decimals
+
+          def initialize(key: nil, size:, type: :string, decimals: 2)
+            @key      = key
+            @size     = size == :remainder ? -1 : size.to_i
+            @type     = type.to_sym
+            @decimals = decimals
+
+            unless @size.positive? || (@size == -1)
+              raise(Errors::InvalidLayout, "Size #{size.inspect} must be positive or :remainder")
+            end
+            raise(Errors::InvalidLayout, "Unknown type: #{type.inspect}") unless TYPES.include?(type)
+          end
+
+          def parse(value)
+            return if value.nil?
+
+            stripped_value = value.to_s.strip
+
+            case type
+            when :string
+              stripped_value
+            when :integer
+              stripped_value.length.zero? ? nil : value.to_i
+            when :float
+              stripped_value.length.zero? ? nil : value.to_f
+            else
+              raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
+            end
+          end
+
+          def render(value, truncate)
+            formatted =
+              case type
+              when :string
+                value = value.to_s
+                return value if size == -1
+
+                format(truncate ? "%-#{size}.#{size}s" : "%-#{size}s", value)
+              when :integer
+                return value.to_i.to_s if size == -1
+
+                truncate = false
+                format("%0#{size}d", value.to_i)
+              when :float
+                return value.to_f.to_s if size == -1
 
-        FixedLayout = Struct.new(:key, :size)
+                truncate = false
+                format("%0#{size}.#{decimals}f", value.to_f)
+              else
+                raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
+              end
 
-        # Returns [Array<FixedLayout>] the layout for this fixed width file.
-        # Also validates values
-        def parse_layout(layout)
-          layout.collect do |map|
-            size = map[:size]
-            key  = map[:key]
-            raise(ArgumentError, "Missing required :key and :size in: #{map.inspect}") unless size && key
+            if !truncate && formatted.length > size
+              raise(Errors::ValueTooLong, "Value: #{value} is too large to fit into column:#{key} of size:#{size}")
+            end
 
-            FixedLayout.new(key, size)
+            formatted
           end
         end
       end

data/lib/io_streams/tabular/utility/csv_row.rb

@@ -6,10 +6,7 @@ module IOStreams
       # 2 to 3 times better performance than CSV.parse_line and considerably less
       # garbage collection required.
       #
-      # Note:
-      #   This parser does not support line feeds embedded in quoted fields since
-      #   the file is broken apart based on line feeds during the upload process and
-      #   is then processed by each worker on a line by line basis.
+      # Note: Only used prior to Ruby 2.6
       class CSVRow < ::CSV
         UTF8_ENCODING = Encoding.find("UTF-8").freeze
 

data/lib/io_streams/utils.rb

@@ -49,10 +49,10 @@ module IOStreams
         @user     = uri.user
         @password = uri.password
         @port     = uri.port
-        if uri.query
-          @query = {}
-          ::URI.decode_www_form(uri.query).each { |key, value| @query[key] = value }
-        end
+        return unless uri.query
+
+        @query = {}
+        ::URI.decode_www_form(uri.query).each { |key, value| @query[key] = value }
       end
     end
   end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.2.1".freeze
+  VERSION = "1.6.2".freeze
 end

data/lib/io_streams/zip/reader.rb

@@ -38,7 +38,7 @@ module IOStreams
             return true
           end
 
-          while entry = zin.get_next_entry
+          while (entry = zin.get_next_entry)
             return true if entry.name == entry_file_name
           end
           false

data/test/builder_test.rb

@@ -41,6 +41,35 @@ class BuilderTest < Minitest::Test
       end
     end
 
+    describe "#format" do
+      it "detects the format from the file name" do
+        streams = IOStreams::Builder.new("abc.json")
+        assert_equal :json, streams.format
+      end
+
+      it "is nil if the file name has no meaningful format" do
+        assert_nil streams.format
+      end
+
+      it "returns set format with no file_name" do
+        streams        = IOStreams::Builder.new
+        streams.format = :csv
+        assert_equal :csv, streams.format
+      end
+
+      it "returns set format with file_name" do
+        streams        = IOStreams::Builder.new("abc.json")
+        streams.format = :csv
+        assert_equal :csv, streams.format
+      end
+
+      it "validates bad format" do
+        assert_raises ArgumentError do
+          streams.format = :blah
+        end
+      end
+    end
+
     describe "#stream" do
       it "adds one stream" do
         streams.stream(:pgp, passphrase: "unlock-me")