iostreams 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36a51f41f33a1c58f1d5d03ed857784e86de4588ca38aa5a596e479f8661faf3
-  data.tar.gz: 23dd967c21581520675799692296bb901e2dc539f008fa30623f4bd72a2c90f3
+  metadata.gz: bef30882ae2eaebcfbc240858a6854f44cc54208465868e55fd6296addfc8015
+  data.tar.gz: e9984f367727934b96495a646aa271009681f9972568a0b51e9046edf451e321
 SHA512:
-  metadata.gz: 81d5f7ea50b7a5b06a5d26b758eb0639b0c820735cd0cf1545c6298f244b96b0070d3f8a11deed0ef8398b5b6c461561b3c43661612e3eeda19bdf61abb549ab
-  data.tar.gz: 19f6f2051351533029cd7fca3cd6e7be3a4e5d3ea4ff2ae512b0b98a12b84639ae63e6a4decc0ff3edf1d50d5c14ddc1b869767f2f6c8793fc392f6ad02d939f
+  metadata.gz: 4c30cb2085ce36904551bcfce2d0ec1d4544d79af891bf27660c35daa4081e1f2edaf193332ca7f15ed8324f4aaddef991a906b896d502b1574c53531f27b4d5
+  data.tar.gz: 6066bb59b519568f99121d97ec0ba2637fab650889cc898a788f71d1dedcbce7167fa435206656d3ce607bb141eee5507501825afebbbd7e37e9c2c27c915f0f

data/lib/io_streams/bzip2/reader.rb

@@ -3,9 +3,7 @@ module IOStreams
     class Reader < IOStreams::Reader
       # Read from a Bzip2 stream, decompressing the contents as it is read
       def self.stream(input_stream, **args)
-        unless defined?(::Bzip2::FFI)
-          Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi")
-        end
+        Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
 
         begin
           io = ::Bzip2::FFI::Reader.new(input_stream, args)

data/lib/io_streams/bzip2/writer.rb

@@ -3,9 +3,7 @@ module IOStreams
     class Writer < IOStreams::Writer
       # Write to a stream, compressing with Bzip2
       def self.stream(input_stream, original_file_name: nil, **args)
-        unless defined?(::Bzip2::FFI)
-          Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi")
-        end
+        Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
 
         begin
           io = ::Bzip2::FFI::Writer.new(input_stream, args)

data/lib/io_streams/encode/reader.rb

@@ -78,8 +78,6 @@ module IOStreams
         block
       end
 
-      private
-
       def self.extract_cleaner(cleaner)
         return if cleaner.nil?
 

data/lib/io_streams/io_streams.rb

@@ -298,8 +298,6 @@ module IOStreams
     @schemes[scheme_name.nil? ? nil : scheme_name.to_sym] || raise(ArgumentError, "Unknown Scheme type: #{scheme_name.inspect}")
   end
 
-  private
-
   Extension = Struct.new(:reader_class, :writer_class)
 
   # Hold root paths

data/lib/io_streams/line/reader.rb

@@ -63,11 +63,11 @@ module IOStreams
         # Auto-detect windows/linux line endings if not supplied. \n or \r\n
         @delimiter ||= auto_detect_line_endings
 
-        if @buffer
-          # Change the delimiters encoding to match that of the input stream
-          @delimiter      = @delimiter.encode(@buffer.encoding)
-          @delimiter_size = @delimiter.size
-        end
+        return unless @buffer
+
+        # Change the delimiters encoding to match that of the input stream
+        @delimiter      = @delimiter.encode(@buffer.encoding)
+        @delimiter_size = @delimiter.size
       end
 
       # Iterate over every line in the file/stream passing each line to supplied block in turn.

data/lib/io_streams/path.rb

@@ -82,6 +82,7 @@ module IOStreams
     end
 
     # Cleanup an incomplete write to the target "file" if the copy fails.
+    # rubocop:disable Lint/SuppressedException
     def copy_from(source, **args)
       super(source, **args)
     rescue StandardError => e
@@ -91,6 +92,7 @@ module IOStreams
       end
       raise(e)
     end
+    # rubocop:enable Lint/SuppressedException
 
     # Moves the file by copying it to the new path and then deleting the current path.
     # Returns [IOStreams::Path] the target path.

data/lib/io_streams/paths/sftp.rb

@@ -71,7 +71,9 @@ module IOStreams
       #   end
       #
       #   # When using the sftp executable use an identity file instead of a password to authenticate:
-      #   IOStreams.path("sftp://test.com/path/file_name.csv", username: "jack", ssh_options: {IdentityFile: "~/.ssh/private_key"}).reader do |io|
+      #   IOStreams.path("sftp://test.com/path/file_name.csv",
+      #                  username:    "jack",
+      #                  ssh_options: {IdentityFile: "~/.ssh/private_key"}).reader do |io|
       #     puts io.read
       #   end
       def initialize(url, username: nil, password: nil, ssh_options: {})
@@ -122,7 +124,8 @@ module IOStreams
       #   end
       #
       # Example Output:
-      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group", :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
+      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
+      #   :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
       def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
         Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
 

data/lib/io_streams/pgp.rb

@@ -74,9 +74,10 @@ module IOStreams
 
       raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}") unless status.success?
 
-      if (match = err.match(/gpg: key ([0-9A-F]+)\s+/))
-        match[1]
-      end
+      match = err.match(/gpg: key ([0-9A-F]+)\s+/)
+      return unless match
+
+      match[1]
     end
 
     # Delete all private and public keys for a particular email.
@@ -110,11 +111,6 @@ module IOStreams
       !list_keys(email: email, key_id: key_id, private: private).empty?
     end
 
-    # Deprecated
-    def self.has_key?(**args)
-      key?(**args)
-    end
-
     # Returns [Array<Hash>] the list of keys.
     #   Each Hash consists of:
     #     key_length: [Integer]
@@ -232,7 +228,7 @@ module IOStreams
         err.each_line do |line|
           if line =~ /secret key imported/
             secret = true
-          elsif match = line.match(/key\s+(\w+):\s+(\w+).+\"(.*)<(.*)>\"/)
+          elsif (match = line.match(/key\s+(\w+):\s+(\w+).+\"(.*)<(.*)>\"/))
             results << {
               key_id:  match[1].to_s.strip,
               private: secret,
@@ -347,8 +343,6 @@ module IOStreams
       end
     end
 
-    private
-
     @logger = nil
 
     def self.logger

data/lib/io_streams/tabular.rb

@@ -184,8 +184,6 @@ module IOStreams
       @formats.keys
     end
 
-    private
-
     # A registry to hold formats for processing files during upload or download
     @formats = {}
 

data/lib/io_streams/tabular/parser/fixed.rb

@@ -37,6 +37,22 @@ module IOStreams
         #      The :size is the total size of this field including the `.` and the decimals.
         #      Number of :decimals
         #      Raises Errors::ValueTooLong when the supplied value cannot be rendered in `size` characters.
+        #
+        # In some circumstances the length of the last column is variable.
+        #   layout: [Array<Hash>]
+        #     [
+        #       {size: 23, key: "name"},
+        #       {size: :remainder, key: "rest"}
+        #     ]
+        # By setting a size of `:remainder` it will take the rest of the line as the value for that column.
+        #
+        # A size of `:remainder` and no `:key` will discard the remainder of the line without validating the length.
+        #   layout: [Array<Hash>]
+        #     [
+        #       {size: 23, key: "name"},
+        #       {size: :remainder}
+        #     ]
+        #
         def initialize(layout:, truncate: true)
           @layout   = Layout.new(layout)
           @truncate = truncate
@@ -57,12 +73,7 @@ module IOStreams
 
           result = ""
           layout.columns.each do |column|
-            value = hash[column.key].to_s
-            if !truncate && (value.length > column.size)
-              raise(Errors::ValueTooLong, "Value: #{value.inspect} is too long to fit into column #{column.key} of size #{column.size}")
-            end
-
-            result << column.render(value)
+            result << column.render(hash[column.key], truncate)
           end
           result
         end
@@ -74,13 +85,18 @@ module IOStreams
             raise(Errors::TypeMismatch, "Line must be a String when format is :fixed. Actual: #{line.class.name}")
           end
 
-          if line.length != layout.length
+          if layout.length.positive? && (line.length != layout.length)
             raise(Errors::InvalidLineLength, "Expected line length: #{layout.length}, actual line length: #{line.length}")
           end
 
           hash  = {}
           index = 0
           layout.columns.each do |column|
+            if column.size == -1
+              hash[column.key] = column.parse(line[index..-1]) if column.key
+              break
+            end
+
             # Ignore "columns" that have no keys. E.g. Fillers
             hash[column.key] = column.parse(line[index, column.size]) if column.key
             index += column.size
@@ -93,8 +109,6 @@ module IOStreams
           false
         end
 
-        private
-
         class Layout
           attr_reader :columns, :length
 
@@ -113,7 +127,15 @@ module IOStreams
               raise(Errors::InvalidLayout, "Missing required :size in: #{hash.inspect}") unless hash.key?(:size)
 
               column = Column.new(**hash)
-              @length += column.size
+              if column.size == -1
+                if @length == -1
+                  raise(Errors::InvalidLayout, "Only the last :size can be '-1' or :remainder in: #{hash.inspect}")
+                end
+
+                @length = -1
+              else
+                @length += column.size
+              end
               column
             end
           end
@@ -126,11 +148,13 @@ module IOStreams
 
           def initialize(key: nil, size:, type: :string, decimals: 2)
             @key      = key
-            @size     = size.to_i
+            @size     = size == :remainder ? -1 : size.to_i
             @type     = type.to_sym
             @decimals = decimals
 
-            raise(Errors::InvalidLayout, "Size #{size.inspect} must be positive") unless @size.positive?
+            unless @size.positive? || (@size == -1)
+              raise(Errors::InvalidLayout, "Size #{size.inspect} must be positive or :remainder")
+            end
             raise(Errors::InvalidLayout, "Unknown type: #{type.inspect}") unless TYPES.include?(type)
           end
 
@@ -151,27 +175,33 @@ module IOStreams
             end
           end
 
-          def render(value)
-            case type
-            when :string
-              format("%-#{size}.#{size}s", value.to_s)
-            when :integer
-              formatted = format("%0#{size}d", value.to_i)
-              if formatted.length > size
-                raise(Errors::ValueTooLong, "Value: #{value} is too large to fit into column:#{key} of size:#{size}")
+          def render(value, truncate)
+            formatted =
+              case type
+              when :string
+                value = value.to_s
+                return value if size == -1
+
+                format(truncate ? "%-#{size}.#{size}s" : "%-#{size}s", value)
+              when :integer
+                return value.to_i.to_s if size == -1
+
+                truncate = false
+                format("%0#{size}d", value.to_i)
+              when :float
+                return value.to_f.to_s if size == -1
+
+                truncate = false
+                format("%0#{size}.#{decimals}f", value.to_f)
+              else
+                raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
               end
 
-              formatted
-            when :float
-              formatted = format("%0#{size}.#{decimals}f", value.to_f)
-              if formatted.length > size
-                raise(Errors::ValueTooLong, "Value: #{value} is too large to fit into column:#{key} of size:#{size}")
-              end
-
-              formatted
-            else
-              raise(Errors::InvalidLayout, "Unsupported type: #{type.inspect}")
+            if !truncate && formatted.length > size
+              raise(Errors::ValueTooLong, "Value: #{value} is too large to fit into column:#{key} of size:#{size}")
             end
+
+            formatted
           end
         end
       end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.4.0".freeze
+  VERSION = "1.5.0".freeze
 end

data/test/tabular_test.rb

@@ -10,6 +10,36 @@ class TabularTest < Minitest::Test
       IOStreams::Tabular.new(columns: %w[first_field second third], format: format)
     end
 
+    let :fixed do
+      layout = [
+        {size: 23, key: :name},
+        {size: 40, key: :address},
+        {size: 2},
+        {size: 5, key: :zip, type: :integer},
+        {size: 8, key: :age, type: :integer},
+        {size: 10, key: :weight, type: :float, decimals: 2}
+      ]
+      IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
+    end
+
+    let :fixed_with_remainder do
+      layout = [
+        {size: 23, key: :name},
+        {size: 40, key: :address},
+        {size: :remainder, key: :remainder}
+      ]
+      IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
+    end
+
+    let :fixed_discard_remainder do
+      layout = [
+        {size: 23, key: :name},
+        {size: 40, key: :address},
+        {size: :remainder}
+      ]
+      IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
+    end
+
     describe "#parse_header" do
       it "parses and sets the csv header" do
         tabular = IOStreams::Tabular.new(format: :csv)
@@ -136,57 +166,55 @@ class TabularTest < Minitest::Test
       end
 
       describe ":fixed format" do
-        let :tabular do
-          layout = [
-            {size: 23, key: :name},
-            {size: 40, key: :address},
-            {size: 2},
-            {size: 5, key: :zip, type: :integer},
-            {size: 8, key: :age, type: :integer},
-            {size: 10, key: :weight, type: :float, decimals: 2}
-          ]
-          IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
-        end
-
         it "parses to hash" do
-          assert hash = tabular.record_parse("Jack                   over there                              XX34618012345670012345.01")
+          assert hash = fixed.record_parse("Jack                   over there                              XX34618012345670012345.01")
           assert_equal({name: "Jack", address: "over there", zip: 34_618, age: 1_234_567, weight: 12_345.01}, hash)
         end
 
         it "parses short string" do
           assert_raises IOStreams::Errors::InvalidLineLength do
-            tabular.record_parse("Jack                   over th")
+            fixed.record_parse("Jack                   over th")
           end
         end
 
         it "parses longer string" do
           assert_raises IOStreams::Errors::InvalidLineLength do
-            tabular.record_parse("Jack                   over there                              XX34618012345670012345.01............")
+            fixed.record_parse("Jack                   over there                              XX34618012345670012345.01............")
           end
         end
 
         it "parses zero values" do
-          assert hash = tabular.record_parse("                                                                 00000000000000000000000")
+          assert hash = fixed.record_parse("                                                                 00000000000000000000000")
           assert_equal({name: "", address: "", zip: 0, age: 0, weight: 0.0}, hash)
         end
 
         it "parses empty values" do
-          assert hash = tabular.record_parse("                                                               XX                       ")
+          assert hash = fixed.record_parse("                                                               XX                       ")
           assert_equal({name: "", address: "", zip: nil, age: nil, weight: nil}, hash)
         end
 
         it "parses blank strings" do
-          skip "TODO: Part of tabular refactor to get this working"
-          assert hash = tabular.record_parse("                                                                                        ")
+          skip "TODO: Part of fixed refactor to get this working"
+          assert hash = fixed.record_parse("                                                                                        ")
           assert_equal({name: "", address: "", zip: nil, age: nil, weight: nil}, hash)
         end
 
         it "parses nil data as nil" do
-          refute tabular.record_parse(nil)
+          refute fixed.record_parse(nil)
         end
 
         it "parses empty string as nil" do
-          refute tabular.record_parse("")
+          refute fixed.record_parse("")
+        end
+
+        it "parses remainder" do
+          hash = fixed_with_remainder.record_parse("Jack                   over there                              XX34618012345670012345.01............")
+          assert_equal({name: "Jack", address: "over there", remainder: "XX34618012345670012345.01............"}, hash)
+        end
+
+        it "discards remainder" do
+          hash = fixed_discard_remainder.record_parse("Jack                   over there                              XX34618012345670012345.01............")
+          assert_equal({name: "Jack", address: "over there"}, hash)
         end
       end
 
@@ -236,52 +264,55 @@ class TabularTest < Minitest::Test
       end
 
       describe ":fixed format" do
-        let :tabular do
-          layout = [
-            {size: 23, key: :name},
-            {size: 40, key: :address},
-            {size: 2},
-            {size: 5, key: :zip, type: :integer},
-            {size: 8, key: :age, type: :integer},
-            {size: 10, key: :weight, type: :float, decimals: 2}
-          ]
-          IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
-        end
-
         it "renders fixed data" do
-          assert string = tabular.render(name: "Jack", address: "over there", zip: 34_618, weight: 123_456.789123, age: 21)
+          assert string = fixed.render(name: "Jack", address: "over there", zip: 34_618, weight: 123_456.789123, age: 21)
           assert_equal "Jack                   over there                                34618000000210123456.79", string
         end
 
         it "truncates long strings" do
-          assert string = tabular.render(name: "Jack ran up the beanstalk and when jack reached the top it was truncated", address: "over there", zip: 34_618)
+          assert string = fixed.render(name: "Jack ran up the beanstalk and when jack reached the top it was truncated", address: "over there", zip: 34_618)
           assert_equal "Jack ran up the beanstaover there                                34618000000000000000.00", string
         end
 
         it "when integer is too large" do
           assert_raises IOStreams::Errors::ValueTooLong do
-            tabular.render(zip: 3_461_832_653_653_265)
+            fixed.render(zip: 3_461_832_653_653_265)
           end
         end
 
         it "when float is too large" do
           assert_raises IOStreams::Errors::ValueTooLong do
-            tabular.render(weight: 3_461_832_653_653_265.234)
+            fixed.render(weight: 3_461_832_653_653_265.234)
           end
         end
 
         it "renders nil as empty string" do
-          assert string = tabular.render(zip: 34_618)
+          assert string = fixed.render(zip: 34_618)
           assert_equal "                                                                 34618000000000000000.00", string
         end
 
         it "renders boolean" do
-          assert string = tabular.render(name: true, address: false)
+          assert string = fixed.render(name: true, address: false)
           assert_equal "true                   false                                     00000000000000000000.00", string
         end
 
         it "renders no data as nil" do
-          refute tabular.render({})
+          refute fixed.render({})
+        end
+
+        it "any size last string" do
+          assert string = fixed_with_remainder.render(name: "Jack", address: "over there", remainder: "XX34618012345670012345.01............")
+          assert_equal "Jack                   over there                              XX34618012345670012345.01............", string
+        end
+
+        it "nil last string" do
+          assert string = fixed_with_remainder.render(name: "Jack", address: "over there", remainder: nil)
+          assert_equal "Jack                   over there                              ", string
+        end
+
+        it "skips last filler" do
+          assert string = fixed_discard_remainder.render(name: "Jack", address: "over there")
+          assert_equal "Jack                   over there                              ", string
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-04 00:00:00.000000000 Z
+date: 2020-09-10 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: