iostreams 1.2.1 → 1.4.0

data/lib/io_streams/stream.rb

@@ -17,7 +17,7 @@ module IOStreams
     #
     # Example:
     #
-    # IOStreams.path('tempfile2527').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
+    # IOStreams.path("tempfile2527").stream(:zip).stream(:pgp, passphrase: "receiver_passphrase").read
     def stream(stream, **options)
       builder.stream(stream, **options)
       self
@@ -27,12 +27,12 @@ module IOStreams
     # If the relevant stream is not found for this file it is ignored.
     # For example, if the file does not have a pgp extension then the pgp option is not relevant.
     #
-    # IOStreams.path('keep_safe.pgp').option(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
+    # IOStreams.path("keep_safe.pgp").option(:pgp, passphrase: "receiver_passphrase").read
     #
     # # In this case the file is not pgp so the `passphrase` option is ignored.
-    # IOStreams.path('keep_safe.enc').option(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
+    # IOStreams.path("keep_safe.enc").option(:pgp, passphrase: "receiver_passphrase").read
     #
-    # IOStreams.path(output_file_name).option(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
+    # IOStreams.path(output_file_name).option(:pgp, passphrase: "receiver_passphrase").read
     def option(stream, **options)
       builder.option(stream, **options)
       self
@@ -177,7 +177,7 @@ module IOStreams
     end
 
     def copy_to(target, convert: true)
-      target = IOStreams.path(target) unless target.is_a?(Stream)
+      target = IOStreams.new(target)
       target.copy_from(self, convert: convert)
     end
 
@@ -282,7 +282,12 @@ module IOStreams
     def line_reader(embedded_within: nil, **args)
       embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?(".csv")
 
-      stream_reader { |io| yield IOStreams::Line::Reader.new(io, original_file_name: builder.file_name, embedded_within: embedded_within, **args) }
+      stream_reader do |io|
+        yield IOStreams::Line::Reader.new(io,
+                                          original_file_name: builder.file_name,
+                                          embedded_within:    embedded_within,
+                                          **args)
+      end
     end
 
     # Iterate over a file / stream returning each line as an array, one at a time.
@@ -306,19 +311,25 @@ module IOStreams
     def line_writer(**args, &block)
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Line::Writer)
 
-      writer { |io| IOStreams::Line::Writer.stream(io, original_file_name: builder.file_name, **args, &block) }
+      writer do |io|
+        IOStreams::Line::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+      end
     end
 
     def row_writer(delimiter: $/, **args, &block)
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Row::Writer)
 
-      line_writer(delimiter: delimiter) { |io| IOStreams::Row::Writer.stream(io, original_file_name: builder.file_name, **args, &block) }
+      line_writer(delimiter: delimiter) do |io|
+        IOStreams::Row::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+      end
     end
 
     def record_writer(delimiter: $/, **args, &block)
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Record::Writer)
 
-      line_writer(delimiter: delimiter) { |io| IOStreams::Record::Writer.stream(io, original_file_name: builder.file_name, **args, &block) }
+      line_writer(delimiter: delimiter) do |io|
+        IOStreams::Record::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+      end
     end
   end
 end

data/lib/io_streams/tabular.rb

@@ -89,7 +89,7 @@ module IOStreams
         else
           self.class.parser_class(format)
         end
-      @parser = format_options ? klass.new(format_options) : klass.new
+      @parser = format_options ? klass.new(**format_options) : klass.new
     end
 
     # Returns [true|false] whether a header is still required in order to parse or render the current format.
@@ -142,7 +142,10 @@ module IOStreams
       return unless requires_header?
 
       if IOStreams::Utils.blank?(header.columns)
-        raise(Errors::MissingHeader, "Header columns must be set before attempting to render a header for format: #{format.inspect}")
+        raise(
+          Errors::MissingHeader,
+          "Header columns must be set before attempting to render a header for format: #{format.inspect}"
+        )
       end
 
       parser.render(header.columns, header)

data/lib/io_streams/tabular/header.rb

@@ -109,7 +109,10 @@ module IOStreams
         end
 
         unless row.is_a?(Array)
-          raise(IOStreams::Errors::TypeMismatch, "Don't know how to convert #{row.class.name} to an Array without the header columns being set.")
+          raise(
+            IOStreams::Errors::TypeMismatch,
+            "Don't know how to convert #{row.class.name} to an Array without the header columns being set."
+          )
         end
 
         row
@@ -126,18 +129,17 @@ module IOStreams
       # Perform cleansing on returned Hash keys during the narrowing process.
       # For example, avoids issues with case etc.
       def cleanse_hash(hash)
-        h = {}
-        hash.each_pair do |key, value|
-          cleansed_key    =
-            if columns.include?(key)
-              key
-            else
-              key = cleanse_column(key)
-              key if columns.include?(key)
-            end
-          h[cleansed_key] = value if cleansed_key
+        unmatched = columns - hash.keys
+        unless unmatched.empty?
+          hash = hash.dup
+          unmatched.each { |name| hash[cleanse_column(name)] = hash.delete(name) }
+        end
+        # Hash#slice as of Ruby 2.5
+        if hash.respond_to?(:slice)
+          hash.slice(*columns)
+        else
+          columns.each_with_object({}) { |column, new_hash| new_hash[column] = hash[column] }
         end
-        h
       end
 
       def cleanse_column(name)

data/lib/io_streams/tabular/parser/fixed.rb

@@ -3,31 +3,66 @@ module IOStreams
     module Parser
       # Parsing and rendering fixed length data
       class Fixed < Base
-        attr_reader :fixed_layout
+        attr_reader :layout, :truncate
 
         # Returns [IOStreams::Tabular::Parser]
         #
         # Parameters:
         #   layout: [Array<Hash>]
         #     [
-        #       {key: 'name',    size: 23 },
-        #       {key: 'address', size: 40 },
-        #       {key: 'zip',     size: 5 }
+        #       {size: 23, key: "name"},
+        #       {size: 40, key: "address"},
+        #       {size:  2},
+        #       {size:  5, key: "zip"},
+        #       {size:  8, key: "age", type: :integer},
+        #       {size: 10, key: "weight", type: :float, decimals: 2}
         #     ]
-        def initialize(layout:)
-          @fixed_layout = parse_layout(layout)
+        #
+        # Notes:
+        # * Leave out the name of the key to ignore that column during parsing,
+        #   and to space fill when rendering. For example as a filler.
+        #
+        # Types:
+        #    :string
+        #      This is the default type.
+        #      Applies space padding and the value is left justified.
+        #      Returns value as a String
+        #    :integer
+        #      Applies zero padding to the left.
+        #      Returns value as an Integer
+        #      Raises Errors::ValueTooLong when the supplied value cannot be rendered in `size` characters.
+        #    :float
+        #      Applies zero padding to the left.
+        #      Returns value as a float.
+        #      The :size is the total size of this field including the `.` and the decimals.
+        #      Number of :decimals
+        #      Raises Errors::ValueTooLong when the supplied value cannot be rendered in `size` characters.
+        def initialize(layout:, truncate: true)
+          @layout   = Layout.new(layout)
+          @truncate = truncate
+        end
+
+        # The required line length for every fixed length line
+        def line_length
+          layout.length
         end
 
         # Returns [String] fixed layout values extracted from the supplied hash.
-        # String will be encoded to `encoding`
+        #
+        # Notes:
+        # * A nil value is considered an empty string
+        # * When a supplied value exceeds the column size it is truncated.
         def render(row, header)
           hash = header.to_hash(row)
 
           result = ""
-          fixed_layout.each do |map|
-            # A nil value is considered an empty string
-            value = hash[map.key].to_s
-            result << format("%-#{map.size}.#{map.size}s", value)
+          layout.columns.each do |column|
+            value = hash[column.key].to_s
+            if !truncate && (value.length > column.size)
+              raise(Errors::ValueTooLong, "Value: #{value.inspect} is too long to fit into column #{column.key} of size #{column.size}")
+            end
+
+            result << column.render(value)
           end
           result
         end
@@ -36,32 +71,107 @@ module IOStreams
         # String will be encoded to `encoding`
         def parse(line)
           unless line.is_a?(String)
-            raise(IOStreams::Errors::TypeMismatch, "Format is :fixed. Invalid parse input: #{line.class.name}")
+            raise(Errors::TypeMismatch, "Line must be a String when format is :fixed. Actual: #{line.class.name}")
+          end
+
+          if line.length != layout.length
+            raise(Errors::InvalidLineLength, "Expected line length: #{layout.length}, actual line length: #{line.length}")
           end
 
           hash  = {}
           index = 0
-          fixed_layout.each do |map|
-            value = line[index..(index + map.size - 1)]
-            index += map.size
-            hash[map.key] = value.to_s.strip
+          layout.columns.each do |column|
+            # Ignore "columns" that have no keys. E.g. Fillers
+            hash[column.key] = column.parse(line[index, column.size]) if column.key
+            index += column.size
           end
           hash
         end
 
+        # The header is required as an argument and cannot be supplied in the file itself.
+        def requires_header?
+          false
+        end
+
         private
 
-        FixedLayout = Struct.new(:key, :size)
+        class Layout
+          attr_reader :columns, :length
+
+          # Returns [Array<FixedLayout>] the layout for this fixed width file.
+          # Also validates values
+          def initialize(layout)
+            @length  = 0
+            @columns = parse_layout(layout)
+          end
+
+          private
+
+          def parse_layout(layout)
+            @length = 0
+            layout.collect do |hash|
+              raise(Errors::InvalidLayout, "Missing required :size in: #{hash.inspect}") unless hash.key?(:size)
+
+              column = Column.new(**hash)
+              @length += column.size
+              column
+            end
+          end
+        end
+
+        class Column
+          TYPES = %i[string integer float].freeze
+
+          attr_reader :key, :size, :type, :decimals
+
+          def initialize(key: nil, size:, type: :string, decimals: 2)
+            @key      = key
+            @size     = size.to_i
+            @type     = type.to_sym
+            @decimals = decimals
+
+            raise(Errors::InvalidLayout, "Size #{size.inspect} must be positive") unless @size.positive?
+            raise(Errors::InvalidLayout, "Unknown type: #{type.inspect}") unless TYPES.include?(type)
+          end
+
+          def parse(value)
+            return if value.nil?
+
+            stripped_value = value.to_s.strip
+
+            case type
+            when :string
+              stripped_value
+            when :integer
+              stripped_value.length.zero? ? nil : value.to_i
+            when :float
+              stripped_value.length.zero? ? nil : value.to_f
+            else
+              raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
+            end
+          end
+
+          def render(value)
+            case type
+            when :string
+              format("%-#{size}.#{size}s", value.to_s)
+            when :integer
+              formatted = format("%0#{size}d", value.to_i)
+              if formatted.length > size
+                raise(Errors::ValueTooLong, "Value: #{value} is too large to fit into column:#{key} of size:#{size}")
+              end
 
-        # Returns [Array<FixedLayout>] the layout for this fixed width file.
-        # Also validates values
-        def parse_layout(layout)
-          layout.collect do |map|
-            size = map[:size]
-            key  = map[:key]
-            raise(ArgumentError, "Missing required :key and :size in: #{map.inspect}") unless size && key
+              formatted
+            when :float
+              formatted = format("%0#{size}.#{decimals}f", value.to_f)
+              if formatted.length > size
+                raise(Errors::ValueTooLong, "Value: #{value} is too large to fit into column:#{key} of size:#{size}")
+              end
 
-            FixedLayout.new(key, size)
+              formatted
+            else
+              raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
+            end
           end
         end
       end

data/lib/io_streams/utils.rb

@@ -49,10 +49,10 @@ module IOStreams
         @user     = uri.user
         @password = uri.password
         @port     = uri.port
-        if uri.query
-          @query = {}
-          ::URI.decode_www_form(uri.query).each { |key, value| @query[key] = value }
-        end
+        return unless uri.query
+
+        @query = {}
+        ::URI.decode_www_form(uri.query).each { |key, value| @query[key] = value }
       end
     end
   end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.2.1".freeze
+  VERSION = "1.4.0".freeze
 end

data/lib/io_streams/zip/reader.rb

@@ -38,7 +38,7 @@ module IOStreams
             return true
           end
 
-          while entry = zin.get_next_entry
+          while (entry = zin.get_next_entry)
             return true if entry.name == entry_file_name
           end
           false

data/test/bzip2_writer_test.rb

@@ -22,13 +22,14 @@ class Bzip2WriterTest < Minitest::Test
       it "file" do
         IOStreams::Bzip2::Writer.file(file_name) do |io|
           io.write(decompressed)
+          io.write(decompressed)
         end
 
         File.open(file_name, "rb") do |file|
-          io     = RBzip2.default_adapter::Decompressor.new(file)
+          io     = ::Bzip2::FFI::Reader.new(file)
           result = io.read
           temp_file.delete
-          assert_equal decompressed, result
+          assert_equal decompressed + decompressed, result
         end
       end
 
@@ -36,12 +37,13 @@ class Bzip2WriterTest < Minitest::Test
         io_string = StringIO.new("".b)
         IOStreams::Bzip2::Writer.stream(io_string) do |io|
           io.write(decompressed)
+          io.write(decompressed)
         end
 
         io     = StringIO.new(io_string.string)
-        rbzip2 = RBzip2.default_adapter::Decompressor.new(io)
+        rbzip2 = ::Bzip2::FFI::Reader.new(io)
         data   = rbzip2.read
-        assert_equal decompressed, data
+        assert_equal decompressed + decompressed, data
       end
     end
   end

data/test/io_streams_test.rb

@@ -16,7 +16,7 @@ module IOStreams
       end
 
       let :json_file_name do
-        "/tmp/io_streams/abc.json"
+        "/tmp/iostreams_abc.json"
       end
 
       describe ".root" do
@@ -90,7 +90,7 @@ module IOStreams
         it "hash reader detects json format from file name" do
           ::File.open(json_file_name, "wb") { |file| file.write(expected_json) }
           rows = []
-          path = IOStreams.path("/tmp/io_streams/abc.json")
+          path = IOStreams.path(json_file_name)
           path.each(:hash) do |row|
             rows << row
           end

data/test/paths/file_test.rb

@@ -171,7 +171,7 @@ module Paths
 
       describe "reader" do
         it "reads file" do
-          assert_equal data, file_path.reader(&:read)
+          assert_equal data, file_path.read
         end
       end
 

data/test/paths/s3_test.rb

@@ -73,7 +73,7 @@ module Paths
 
       describe "#reader" do
         it "reads" do
-          assert_equal raw, existing_path.reader(&:read)
+          assert_equal raw, existing_path.read
         end
       end
 
@@ -89,7 +89,7 @@ module Paths
 
       describe "#writer" do
         it "writes" do
-          assert_equal raw.size, write_path.writer { |io| io.write(raw) }
+          assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
           assert write_path.exist?
           assert_equal raw, write_path.read
         end
@@ -138,7 +138,7 @@ module Paths
 
         it "returns all the children under a sub-dir" do
           write_raw_data
-          expected = abd_file_names.collect { |file_name| each_root.join(file_name) }
+          expected = %w[abd/test1.txt abd/test5.file].collect { |file_name| each_root.join(file_name) }
           assert_equal expected.collect(&:to_s).sort, each_root.children("abd/*").collect(&:to_s).sort
         end