iostreams 1.6.2 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a6dd68508835099ef4c9de7f81a9527927d3225138f0bf5ecd00c44194e11858
-  data.tar.gz: 28535e95ca83a4cf0c522de4cd48889a489b4fcefdc8feb9a7bfe0b70124fd7a
+  metadata.gz: 06e71b70e09ed1f57a4d776795d48fc973d32888a078c9e42319fc3eaee56db0
+  data.tar.gz: 7d956af5453840aee0a8c8cca5e7e4e44c465e1dcb13030daa37f2abc8f5cc78
 SHA512:
-  metadata.gz: 8e4d38ef41234f62fdcfde1ae3a96fe59203dcfc38562843106b081c7292efffdda36b88707674059a24b0fa2996d4c1dbe0627694e80a9b12d23d47195a53d3
-  data.tar.gz: 23e16854c305542ee0976f570444d7b99cadcb6ec460d83bebc73a65cba9c28e3beba485c9943797b174b80bdd783c216d89d539e8aecdbda62e1c31f017efce
+  metadata.gz: 9ed848fcc886ed53d36504097036605d13735887a880ed65620460f55d78fc26771a18b1ab32e1685f765cf25097fa42f7e59ba233099aeb15a93b0005936bb5
+  data.tar.gz: 6301601fa85f7f9198e0c61fabdd0a9206c4dfb2b239c0e8155a01a47973f6cb545272a870a593d67437ef25476901aed9c1101c24cacef5ab22e4f794936398

data/lib/io_streams/builder.rb

@@ -79,15 +79,16 @@ module IOStreams
     # with their options that will be applied when the reader or writer is invoked.
     def pipeline
       return streams.dup.freeze if streams
-      return {}.freeze unless file_name
 
-      built_streams          = {}
-      # Encode stream is always first
-      built_streams[:encode] = options[:encode] if options&.key?(:encode)
+      build_pipeline.freeze
+    end
 
-      opts = options || {}
-      parse_extensions.each { |stream| built_streams[stream] = opts[stream] || {} }
-      built_streams.freeze
+    # Removes the named stream from the current pipeline.
+    # If the stream pipeline has not yet been built it will be built from the file_name if present.
+    # Note: Any options must be set _before_ calling this method.
+    def remove_from_pipeline(stream_name)
+      @streams ||= build_pipeline
+      @streams.delete(stream_name.to_sym)
     end
 
     # Returns the tabular format if set, otherwise tries to autodetect the format if the file_name has been set
@@ -97,16 +98,30 @@ module IOStreams
     end
 
     def format=(format)
-      raise(ArgumentError, "Invalid format: #{format.inspect}") unless format.nil? || IOStreams::Tabular.registered_formats.include?(format)
+      unless format.nil? || IOStreams::Tabular.registered_formats.include?(format)
+        raise(ArgumentError, "Invalid format: #{format.inspect}")
+      end
 
       @format = format
     end
 
     private
 
+    def build_pipeline
+      return {} unless file_name
+
+      built_streams          = {}
+      # Encode stream is always first
+      built_streams[:encode] = options[:encode] if options&.key?(:encode)
+
+      opts = options || {}
+      parse_extensions.each { |stream| built_streams[stream] = opts[stream] || {} }
+      built_streams
+    end
+
     def class_for_stream(type, stream)
       ext = IOStreams.extensions[stream.nil? ? nil : stream.to_sym] ||
-        raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
+            raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
       ext.send("#{type}_class") || raise(ArgumentError, "No #{type} registered for Stream type: #{stream.inspect}")
     end
 

data/lib/io_streams/line/reader.rb

@@ -75,6 +75,8 @@ module IOStreams
       # Note:
       # * The line delimiter is _not_ returned.
       def each
+        return to_enum(__method__) unless block_given?
+
         line_count = 0
         until eof?
           line = readline
@@ -96,17 +98,17 @@ module IOStreams
           while line.count(@embedded_within).odd?
             if eof? || line.length > @buffer_size * 10
               raise(Errors::MalformedDataError.new(
-                "Unbalanced delimited field, delimiter: #{@embedded_within}",
-                initial_line_number
-              ))
+                      "Unbalanced delimited field, delimiter: #{@embedded_within}",
+                      initial_line_number
+                    ))
             end
             line << @delimiter
             next_line = _readline
             if next_line.nil?
               raise(Errors::MalformedDataError.new(
-                "Unbalanced delimited field, delimiter: #{@embedded_within}",
-                initial_line_number
-              ))
+                      "Unbalanced delimited field, delimiter: #{@embedded_within}",
+                      initial_line_number
+                    ))
             end
             line << next_line
           end
@@ -146,8 +148,8 @@ module IOStreams
         data
       end
 
-      # Returns [Integer] the number of characters read into the internal buffer
-      # Returns 0 on EOF
+      # Returns whether more data is available to read
+      # Returns false on EOF
       def read_block
         return false if @eof
 
@@ -157,7 +159,8 @@ module IOStreams
               @input_stream.read(@buffer_size, @read_cache_buffer)
             rescue ArgumentError
               # Handle arity of -1 when just 0..1
-              @read_cache_buffer = nil
+              @read_cache_buffer     = nil
+              @use_read_cache_buffer = false
               @input_stream.read(@buffer_size)
             end
           else
@@ -170,6 +173,9 @@ module IOStreams
           return false
         end
 
+        # When less data is returned than was requested, it means the end of the file with partial data.
+        @eof = true if block.size < @buffer_size
+
         if @buffer
           @buffer << block
         else

data/lib/io_streams/paths/file.rb

@@ -89,6 +89,11 @@ module IOStreams
       # "**.rb"     "lib/song.rb"    true
       # "*"         "dave/.profile"  true
       def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
+        unless block_given?
+          return to_enum(__method__, pattern,
+                         case_sensitive: case_sensitive, directories: directories, hidden: hidden)
+        end
+
         flags = 0
         flags |= ::File::FNM_CASEFOLD unless case_sensitive
         flags |= ::File::FNM_DOTMATCH if hidden

data/lib/io_streams/paths/s3.rb

@@ -3,7 +3,7 @@ require "uri"
 module IOStreams
   module Paths
     class S3 < IOStreams::Path
-      attr_reader :bucket_name, :client, :options
+      attr_reader :bucket_name, :options
 
       # Largest file size supported by the S3 copy object api.
       S3_COPY_OBJECT_SIZE_LIMIT = 5 * 1024 * 1024 * 1024
@@ -141,16 +141,17 @@ module IOStreams
 
         @bucket_name = uri.hostname
         key          = uri.path.sub(%r{\A/}, "")
-        if client.is_a?(Hash)
-          client[:access_key_id]     = access_key_id if access_key_id
-          client[:secret_access_key] = secret_access_key if secret_access_key
-          @client                    = ::Aws::S3::Client.new(client)
+
+        if client && !client.is_a?(Hash)
+          @client = client
         else
-          @client = client || ::Aws::S3::Client.new(access_key_id: access_key_id, secret_access_key: secret_access_key)
+          @client_options                     = client.is_a?(Hash) ? client.dup : {}
+          @client_options[:access_key_id]     = access_key_id if access_key_id
+          @client_options[:secret_access_key] = secret_access_key if secret_access_key
         end
-        @options = args
 
-        @options.merge(uri.query) if uri.query
+        @options = args
+        @options.merge!(uri.query.transform_keys(&:to_sym)) if uri.query
 
         super(key)
       end
@@ -190,11 +191,11 @@ module IOStreams
       end
 
       # Make S3 perform direct copies within S3 itself.
-      def copy_to(target_path, convert: true)
-        return super(target_path) if convert || (size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
+      def copy_to(target_path, convert: true, **args)
+        return super(target_path, convert: convert, **args) if convert || (size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
 
         target = IOStreams.new(target_path)
-        return super(target) unless target.is_a?(self.class)
+        return super(target, convert: convert, **args) unless target.is_a?(self.class)
 
         source_name = ::File.join(bucket_name, path)
         client.copy_object(options.merge(bucket: target.bucket_name, key: target.path, copy_source: source_name))
@@ -202,11 +203,13 @@ module IOStreams
       end
 
       # Make S3 perform direct copies within S3 itself.
-      def copy_from(source_path, convert: true)
-        return super(source_path) if convert
+      def copy_from(source_path, convert: true, **args)
+        return super(source_path, convert: true, **args) if convert
 
         source = IOStreams.new(source_path)
-        return super(source) if !source.is_a?(self.class) || (source.size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
+        if !source.is_a?(self.class) || (source.size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
+          return super(source, convert: convert, **args)
+        end
 
         source_name = ::File.join(source.bucket_name, source.path)
         client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))
@@ -281,6 +284,11 @@ module IOStreams
       # Notes:
       # - Currently all S3 lookups are recursive as of the pattern regardless of whether the pattern includes `**`.
       def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
+        unless block_given?
+          return to_enum(__method__, pattern,
+                         case_sensitive: case_sensitive, directories: directories, hidden: hidden)
+        end
+
         matcher = Matcher.new(self, pattern, case_sensitive: case_sensitive, hidden: hidden)
 
         # When the pattern includes an exact file name without any pattern characters
@@ -312,6 +320,11 @@ module IOStreams
       def partial_files_visible?
         false
       end
+
+      # Lazy load S3 client since it takes two seconds to create itself!
+      def client
+        @client ||= ::Aws::S3::Client.new(@client_options)
+      end
     end
   end
 end

data/lib/io_streams/paths/sftp.rb

@@ -47,9 +47,23 @@ module IOStreams
       # password: [String]
       #   Password for the user.
       #
-      # **ssh_options
-      #   Any other options supported by ssh_config.
-      #   `man ssh_config` to see all available options.
+      # ssh_options: [Hash]
+      #   - IdentityKey [String]
+      #     The identity key that this client should use to talk to this host.
+      #     Under the covers this value is written to a file and then the file name is passed as `IdentityFile`
+      #   - HostKey [String]
+      #     The expected SSH Host key that is presented by the remote host.
+      #     Instead of storing the host key in the `known_hosts` file, it can be supplied explicity
+      #     using this option.
+      #     Under the covers this value is written to a file and then the file name is passed as `UserKnownHostsFile`
+      #     Notes:
+      #     - It must contain the entire line that would be stored in `known_hosts`,
+      #       including the hostname, ip address, key type and key value. This value is written as-is into a
+      #       "known_hosts" like file and then passed into sftp using the `UserKnownHostsFile` option.
+      #     - The easiest way to generate the required is to use `ssh-keyscan` and then supply that value in this field.
+      #       For example: `ssh-keyscan hostname`
+      #   - Any other options supported by ssh_config.
+      #     `man ssh_config` to see all available options.
       #
       # Examples:
       #
@@ -128,6 +142,11 @@ module IOStreams
       # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
       #   :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
       def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
+        unless block_given?
+          return to_enum(__method__, pattern,
+                         case_sensitive: case_sensitive, directories: directories, hidden: hidden)
+        end
+
         Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
 
         flags = ::File::FNM_EXTGLOB
@@ -168,38 +187,36 @@ module IOStreams
       def sftp_download(remote_file_name, local_file_name)
         with_sftp_args do |args|
           Open3.popen2e(*args) do |writer, reader, waith_thr|
-            begin
-              # Give time for remote sftp server to get ready to accept the password.
-              sleep self.class.before_password_wait_seconds
-
-              writer.puts password
-
-              # Give time for password to be processed and stdin to be passed to sftp process.
-              sleep self.class.sshpass_wait_seconds
-
-              writer.puts "get #{remote_file_name} #{local_file_name}"
-              writer.puts "bye"
-              writer.close
-              out = reader.read.chomp
-              unless waith_thr.value.success?
-                raise(
-                  Errors::CommunicationsFailure,
-                  "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
-                )
-              end
-
-              out
-            rescue Errno::EPIPE
-              out = begin
-                      reader.read.chomp
-                    rescue StandardError
-                      nil
-                    end
+            # Give time for remote sftp server to get ready to accept the password.
+            sleep self.class.before_password_wait_seconds
+
+            writer.puts password
+
+            # Give time for password to be processed and stdin to be passed to sftp process.
+            sleep self.class.sshpass_wait_seconds
+
+            writer.puts "get #{remote_file_name} #{local_file_name}"
+            writer.puts "bye"
+            writer.close
+            out = reader.read.chomp
+            unless waith_thr.value.success?
               raise(
                 Errors::CommunicationsFailure,
                 "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
               )
             end
+
+            out
+          rescue Errno::EPIPE
+            out = begin
+              reader.read.chomp
+            rescue StandardError
+              nil
+            end
+            raise(
+              Errors::CommunicationsFailure,
+              "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
+            )
           end
         end
       end
@@ -207,48 +224,64 @@ module IOStreams
       def sftp_upload(local_file_name, remote_file_name)
         with_sftp_args do |args|
           Open3.popen2e(*args) do |writer, reader, waith_thr|
-            begin
-              writer.puts(password) if password
-              # Give time for password to be processed and stdin to be passed to sftp process.
-              sleep self.class.sshpass_wait_seconds
-              writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
-              writer.puts "bye"
-              writer.close
-              out = reader.read.chomp
-              unless waith_thr.value.success?
-                raise(
-                  Errors::CommunicationsFailure,
-                  "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
-                )
-              end
-
-              out
-            rescue Errno::EPIPE
-              out = begin
-                      reader.read.chomp
-                    rescue StandardError
-                      nil
-                    end
+            writer.puts(password) if password
+            # Give time for password to be processed and stdin to be passed to sftp process.
+            sleep self.class.sshpass_wait_seconds
+            writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
+            writer.puts "bye"
+            writer.close
+            out = reader.read.chomp
+            unless waith_thr.value.success?
               raise(
                 Errors::CommunicationsFailure,
                 "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
               )
             end
+
+            out
+          rescue Errno::EPIPE
+            out = begin
+              reader.read.chomp
+            rescue StandardError
+              nil
+            end
+            raise(
+              Errors::CommunicationsFailure,
+              "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
+            )
           end
         end
       end
 
       def with_sftp_args
-        return yield sftp_args(ssh_options) unless ssh_options.key?("IdentityKey")
+        return yield sftp_args(ssh_options) if !ssh_options.key?("IdentityKey") && !ssh_options.key?("HostKey")
+
+        with_identity_key(ssh_options.dup) do |options|
+          with_host_key(options) do |options2|
+            yield sftp_args(options2)
+          end
+        end
+      end
+
+      def with_identity_key(options)
+        return yield options unless ssh_options.key?("IdentityKey")
+
+        with_temp_file(options, "IdentityFile", options.delete("IdentityKey")) { yield options }
+      end
+
+      def with_host_key(options)
+        return yield options unless ssh_options.key?("HostKey")
+
+        with_temp_file(options, "UserKnownHostsFile", options.delete("HostKey")) { yield options }
+      end
 
+      def with_temp_file(options, option, value)
         Utils.temp_file_name("iostreams-sftp-args", "key") do |file_name|
-          options = ssh_options.dup
-          key     = options.delete("IdentityKey")
           # sftp requires that private key is only readable by the current user
-          ::File.open(file_name, "wb", 0o600) { |io| io.write(key) }
+          ::File.open(file_name, "wb", 0o600) { |io| io.write(value) }
 
-          options["IdentityFile"] = file_name
-          yield sftp_args(options)
+          options[option] = file_name
+          yield options
         end
       end
 
@@ -277,7 +310,7 @@ module IOStreams
       end
 
       def build_ssh_options
-        options                = ssh_options.dup
+        options = ssh_options.dup
         options[:logger]       ||= logger if defined?(SemanticLogger)
         options[:port]         ||= port
         options[:max_pkt_size] ||= 65_536

data/lib/io_streams/pgp/writer.rb

@@ -19,8 +19,7 @@ module IOStreams
 
         private
 
-        attr_reader :default_signer_passphrase
-        attr_reader :default_signer
+        attr_reader :default_signer_passphrase, :default_signer
 
         @default_signer_passphrase = nil
         @default_signer            = nil

data/lib/io_streams/pgp.rb

@@ -48,8 +48,8 @@ module IOStreams
     # See `man gpg` for the remaining options
     def self.generate_key(name:,
                           email:,
-                          comment: nil,
                           passphrase:,
+                          comment: nil,
                           key_type: "RSA",
                           key_length: 4096,
                           subkey_type: "RSA",
@@ -291,10 +291,8 @@ module IOStreams
       version_check
       Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |_stdin, out, waith_thr|
         output = out.read.chomp
-        unless waith_thr.value.success?
-          unless output =~ /(public key not found|No public key)/i
-            raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
-          end
+        if !waith_thr.value.success? && !(output !~ /(public key not found|No public key)/i)
+          raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
         end
 
         output.each_line do |line|
@@ -336,9 +334,11 @@ module IOStreams
             match[1]
           end
         else
-          return [] if err =~ /(key not found|No (public|secret) key)/i
+          if err !~ /(key not found|No (public|secret) key)/i
+            raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email || key_id}: #{err}#{out}")
+          end
 
-          raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email || key_id}: #{err}#{out}")
+          []
         end
       end
     end
@@ -382,10 +382,10 @@ module IOStreams
             key_length: match[3].to_s.to_i,
             key_type:   match[2],
             date:       (begin
-                           Date.parse(match[4].to_s)
-                         rescue StandardError
-                           match[4]
-                         end)
+              Date.parse(match[4].to_s)
+            rescue StandardError
+              match[4]
+            end)
           }
         elsif (match = line.match(%r{(pub|sec)\s+(\d+)(.*)/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?}))
           # Matches: pub  2048R/C7F9D9CB 2016-10-26
@@ -396,10 +396,10 @@ module IOStreams
             key_type:   match[3],
             key_id:     match[4],
             date:       (begin
-                           Date.parse(match[5].to_s)
-                         rescue StandardError
-                           match[5]
-                         end)
+              Date.parse(match[5].to_s)
+            rescue StandardError
+              match[5]
+            end)
           }
           # Prior to gpg v2.0.30
           if match[7]

data/lib/io_streams/record/reader.rb

@@ -68,6 +68,8 @@ module IOStreams
       end
 
       def each
+        return to_enum(__method__) unless block_given?
+
         @line_reader.each do |line|
           if @tabular.header?
             @tabular.parse_header(line)

data/lib/io_streams/row/reader.rb

@@ -40,6 +40,8 @@ module IOStreams
       end
 
       def each
+        return to_enum(__method__) unless block_given?
+
         @line_reader.each do |line|
           if @tabular.header?
             columns = @tabular.parse_header(line)

data/lib/io_streams/stream.rb

@@ -56,6 +56,14 @@ module IOStreams
       builder.pipeline
     end
 
+    # Removes the named stream from the current pipeline.
+    # If the stream pipeline has not yet been built it will be built from the file_name if present.
+    # Note: Any options must be set _before_ calling this method.
+    def remove_from_pipeline(stream_name)
+      builder.remove_from_pipeline(stream_name)
+      self
+    end
+
     # Iterate over a file / stream returning one line at a time.
     #
     # Example: Read a line at a time
@@ -151,6 +159,9 @@ module IOStreams
     #     Whether to apply the stream conversions during the copy.
     #     Default: true
     #
+    #   :mode [:line, :array, :hash]
+    #     When convert is `true` then use this mode to convert the contents of the file.
+    #
     # Examples:
     #
     # # Copy and convert streams based on file extensions
@@ -162,11 +173,17 @@ module IOStreams
     # # Advanced copy with custom stream conversions on source and target.
     # source = IOStreams.path("source_file").stream(encoding: "BINARY")
     # IOStreams.path("target_file.pgp").option(:pgp, passphrase: "hello").copy_from(source)
-    def copy_from(source, convert: true)
+    def copy_from(source, convert: true, mode: nil, **args)
       if convert
         stream = IOStreams.new(source)
-        writer do |target|
-          stream.reader { |src| IO.copy_stream(src, target) }
+        if mode
+          writer(mode, **args) do |target|
+            stream.each(mode) { |row| target << row }
+          end
+        else
+          writer(**args) do |target|
+            stream.reader { |src| IO.copy_stream(src, target) }
+          end
         end
       else
         stream = source.is_a?(Stream) ? source.dup : IOStreams.new(source)
@@ -176,9 +193,9 @@ module IOStreams
       end
     end
 
-    def copy_to(target, convert: true)
+    def copy_to(target, **args)
       target = IOStreams.new(target)
-      target.copy_from(self, convert: convert)
+      target.copy_from(self, **args)
     end
 
     # Set/get the original file_name
@@ -365,8 +382,8 @@ module IOStreams
         IOStreams::Row::Writer.stream(
           io,
           original_file_name: builder.file_name,
-          format: builder.format,
-          format_options: builder.format_options,
+          format:             builder.format,
+          format_options:     builder.format_options,
           **args,
           &block
         )
@@ -380,10 +397,11 @@ module IOStreams
         IOStreams::Record::Writer.stream(
           io,
           original_file_name: builder.file_name,
-          format: builder.format,
-          format_options: builder.format_options,
+          format:             builder.format,
+          format_options:     builder.format_options,
           **args,
-          &block)
+          &block
+        )
       end
     end
   end

data/lib/io_streams/tabular/header.rb

@@ -2,6 +2,9 @@ module IOStreams
   class Tabular
     # Process files / streams that start with a header.
     class Header
+      # Column names that begin with this prefix have been rejected and should be ignored.
+      IGNORE_PREFIX = "__rejected__".freeze
+
       attr_accessor :columns, :allowed_columns, :required_columns, :skip_unknown
 
       # Header
@@ -17,8 +20,8 @@ module IOStreams
       #     List of columns to allow.
       #     Default: nil ( Allow all columns )
       #     Note:
-      #       When supplied any columns that are rejected will be returned in the cleansed columns
-      #       as nil so that they can be ignored during processing.
+      #     * So that rejected columns can be identified in subsequent steps, they will be prefixed with `__rejected__`.
+      #       For example, `Unknown Column` would be cleansed as `__rejected__Unknown Column`.
       #
       #   required_columns [Array<String>]
       #     List of columns that must be present, otherwise an Exception is raised.
@@ -44,8 +47,10 @@ module IOStreams
       # - Spaces and '-' are converted to '_'.
       # - All characters except for letters, digits, and '_' are stripped.
       #
-      # Notes
-      # * Raises Tabular::InvalidHeader when there are no non-nil columns left after cleansing.
+      # Notes:
+      # * So that rejected columns can be identified in subsequent steps, they will be prefixed with `__rejected__`.
+      #   For example, `Unknown Column` would be cleansed as `__rejected__Unknown Column`.
+      # * Raises Tabular::InvalidHeader when there are no rejected columns left after cleansing.
       def cleanse!
         return [] if columns.nil? || columns.empty?
 
@@ -56,7 +61,7 @@ module IOStreams
             cleansed
           else
             ignored_columns << column
-            nil
+            "#{IGNORE_PREFIX}#{column}"
           end
         end
 
@@ -122,7 +127,7 @@ module IOStreams
 
       def array_to_hash(row)
         h = {}
-        columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams::Utils.blank?(col) }
+        columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams::Utils.blank?(col) || col.start_with?(IGNORE_PREFIX) }
         h
       end
 
@@ -134,12 +139,7 @@ module IOStreams
           hash = hash.dup
           unmatched.each { |name| hash[cleanse_column(name)] = hash.delete(name) }
         end
-        # Hash#slice as of Ruby 2.5
-        if hash.respond_to?(:slice)
-          hash.slice(*columns)
-        else
-          columns.each_with_object({}) { |column, new_hash| new_hash[column] = hash[column] }
-        end
+        hash.slice(*columns)
       end
 
       def cleanse_column(name)

data/lib/io_streams/tabular/parser/fixed.rb

@@ -146,7 +146,7 @@ module IOStreams
 
           attr_reader :key, :size, :type, :decimals
 
-          def initialize(key: nil, size:, type: :string, decimals: 2)
+          def initialize(size:, key: nil, type: :string, decimals: 2)
             @key      = key
             @size     = size == :remainder ? -1 : size.to_i
             @type     = type.to_sym

data/lib/io_streams/utils.rb

@@ -28,11 +28,9 @@ module IOStreams
     def self.temp_file_name(basename, extension = "")
       result = nil
       ::Dir::Tmpname.create([basename, extension], IOStreams.temp_dir, max_try: MAX_TEMP_FILE_NAME_ATTEMPTS) do |tmpname|
-        begin
-          result = yield(tmpname)
-        ensure
-          ::File.unlink(tmpname) if ::File.exist?(tmpname)
-        end
+        result = yield(tmpname)
+      ensure
+        ::File.unlink(tmpname) if ::File.exist?(tmpname)
       end
       result
     end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.6.2".freeze
+  VERSION = "1.10.0".freeze
 end

data/lib/iostreams.rb

@@ -23,33 +23,41 @@ module IOStreams
     autoload :Reader, "io_streams/bzip2/reader"
     autoload :Writer, "io_streams/bzip2/writer"
   end
+
   module Encode
     autoload :Reader, "io_streams/encode/reader"
     autoload :Writer, "io_streams/encode/writer"
   end
+
   module Gzip
     autoload :Reader, "io_streams/gzip/reader"
     autoload :Writer, "io_streams/gzip/writer"
   end
+
   module Line
     autoload :Reader, "io_streams/line/reader"
     autoload :Writer, "io_streams/line/writer"
   end
+
   module Record
     autoload :Reader, "io_streams/record/reader"
     autoload :Writer, "io_streams/record/writer"
   end
+
   module Row
     autoload :Reader, "io_streams/row/reader"
     autoload :Writer, "io_streams/row/writer"
   end
+
   module SymmetricEncryption
     autoload :Reader, "io_streams/symmetric_encryption/reader"
     autoload :Writer, "io_streams/symmetric_encryption/writer"
   end
+
   module Xlsx
     autoload :Reader, "io_streams/xlsx/reader"
   end
+
   module Zip
     autoload :Reader, "io_streams/zip/reader"
     autoload :Writer, "io_streams/zip/writer"

data/test/builder_test.rb

@@ -237,6 +237,21 @@ class BuilderTest < Minitest::Test
       end
     end
 
+    describe "#remove_from_pipeline" do
+      let(:file_name) { "my/path/abc.bz2.pgp" }
+      it "removes a named stream from the pipeline" do
+        assert_equal({bz2: {}, pgp: {}}, streams.pipeline)
+        streams.remove_from_pipeline(:bz2)
+        assert_equal({pgp: {}}, streams.pipeline)
+      end
+      it "removes a named stream from the pipeline with options" do
+        streams.option(:pgp, passphrase: "unlock-me")
+        assert_equal({bz2: {}, pgp: {passphrase: "unlock-me"}}, streams.pipeline)
+        streams.remove_from_pipeline(:bz2)
+        assert_equal({pgp: {passphrase: "unlock-me"}}, streams.pipeline)
+      end
+    end
+
     describe "#execute" do
       it "directly calls block for an empty stream" do
         string_io = StringIO.new

data/test/line_reader_test.rb

@@ -98,6 +98,13 @@ class LineReaderTest < Minitest::Test
         assert_equal data.size, count
       end
 
+      it "with no block returns enumerator" do
+        lines = IOStreams::Line::Reader.file(file_name) do |io|
+          io.each.first(100)
+        end
+        assert_equal data, lines
+      end
+
       it "each_line stream" do
         lines = []
         count = File.open(file_name) do |file|

data/test/paths/file_test.rb

@@ -5,7 +5,7 @@ module Paths
     describe IOStreams::Paths::File do
       let(:root) { IOStreams::Paths::File.new("/tmp/iostreams").delete_all }
       let(:directory) { root.join("/some_test_dir") }
-      let(:data) { "Hello World" }
+      let(:data) { "Hello World\nHow are you doing?\nOn this fine day" }
       let(:file_path) do
         path = root.join("some_test_dir/test_file.txt")
         path.writer { |io| io << data }
@@ -17,6 +17,20 @@ module Paths
         path
       end
 
+      describe "#each" do
+        it "reads lines" do
+          records = []
+          count   = file_path.each { |line| records << line }
+          assert_equal count, data.lines.size
+          assert_equal data.lines.collect(&:strip), records
+        end
+
+        it "reads lines without block" do
+          records = file_path.each.first(100)
+          assert_equal data.lines.collect(&:strip), records
+        end
+      end
+
       describe "#each_child" do
         it "iterates an empty path" do
           none = nil
@@ -48,6 +62,12 @@ module Paths
           actual   = root.children("**/Test*.TXT", case_sensitive: true).collect(&:to_s)
           refute_equal expected, actual.sort
         end
+
+        it "with no block returns enumerator" do
+          expected = [file_path.to_s, file_path2.to_s]
+          actual   = root.each_child("**/*").first(100).collect(&:to_s)
+          assert_equal expected.sort, actual.sort
+        end
       end
 
       describe "#mkpath" do
@@ -126,15 +146,13 @@ module Paths
 
         it "missing source file" do
           IOStreams.temp_file("iostreams_move_test", ".txt") do |temp_file|
-            begin
-              refute temp_file.exist?
-              target = temp_file.directory.join("move_test.txt")
-              assert_raises Errno::ENOENT do
-                temp_file.move_to(target)
-              end
-              refute target.exist?
-              refute temp_file.exist?
+            refute temp_file.exist?
+            target = temp_file.directory.join("move_test.txt")
+            assert_raises Errno::ENOENT do
+              temp_file.move_to(target)
             end
+            refute target.exist?
+            refute temp_file.exist?
           end
         end
 

data/test/paths/sftp_test.rb

@@ -20,7 +20,13 @@ module Paths
       let(:file_name) { File.join(File.dirname(__FILE__), "..", "files", "text file.txt") }
       let(:raw) { File.read(file_name) }
 
-      let(:root_path) { IOStreams::Paths::SFTP.new(url, username: username, password: password) }
+      let(:root_path) do
+        if ENV["SFTP_HOST_KEY"]
+          IOStreams::Paths::SFTP.new(url, username: username, password: password, ssh_options: {"HostKey" => ENV["SFTP_HOST_KEY"]})
+        else
+          IOStreams::Paths::SFTP.new(url, username: username, password: password)
+        end
+      end
 
       let :existing_path do
         path = root_path.join("test.txt")

data/test/record_reader_test.rb

@@ -46,6 +46,13 @@ class RecordReaderTest < Minitest::Test
         end
         assert_equal expected, rows
       end
+
+      it "with no block returns enumerator" do
+        records = IOStreams::Record::Reader.file(file_name, cleanse_header: false) do |io|
+          io.each.first(100)
+        end
+        assert_equal expected, records
+      end
     end
 
     describe "#collect" do

data/test/row_reader_test.rb

@@ -10,7 +10,7 @@ class RowReaderTest < Minitest::Test
       CSV.read(file_name)
     end
 
-    describe ".open" do
+    describe "#each" do
       it "file" do
         rows  = []
         count = IOStreams::Row::Reader.file(file_name) do |io|
@@ -20,6 +20,13 @@ class RowReaderTest < Minitest::Test
         assert_equal expected.size, count
       end
 
+      it "with no block returns enumerator" do
+        rows = IOStreams::Row::Reader.file(file_name) do |io|
+          io.each.first(100)
+        end
+        assert_equal expected, rows
+      end
+
       it "stream" do
         rows  = []
         count = IOStreams::Line::Reader.file(file_name) do |file|

data/test/stream_test.rb

@@ -45,9 +45,9 @@ class StreamTest < Minitest::Test
       it "reads a zip file" do
         File.open(multiple_zip_file_name, "rb") do |io|
           result = IOStreams::Stream.new(io).
-            file_name(multiple_zip_file_name).
-            option(:zip, entry_file_name: "test.json").
-            read
+                   file_name(multiple_zip_file_name).
+                   option(:zip, entry_file_name: "test.json").
+                   read
           assert_equal contents_test_json, result
         end
       end
@@ -55,8 +55,8 @@ class StreamTest < Minitest::Test
       it "reads a zip file from within a gz file" do
         File.open(zip_gz_file_name, "rb") do |io|
           result = IOStreams::Stream.new(io).
-            file_name(zip_gz_file_name).
-            read
+                   file_name(zip_gz_file_name).
+                   read
           assert_equal contents_test_txt, result
         end
       end

data/test/tabular_test.rb

@@ -58,12 +58,12 @@ class TabularTest < Minitest::Test
           assert_equal header, tabular.header.columns
         end
 
-        it "white listed snake cased alphanumeric columns" do
+        it "allowed list snake cased alphanumeric columns" do
           tabular = IOStreams::Tabular.new(
-            columns:         ["Ard Vark", "password", "robot version", "$$$"],
+            columns:         ["Ard Vark", "Password", "robot version", "$$$"],
             allowed_columns: %w[ard_vark robot_version]
           )
-          expected_header = ["ard_vark", nil, "robot_version", nil]
+          expected_header = ["ard_vark", "__rejected__Password", "robot_version", "__rejected__$$$"]
           cleansed_header = tabular.cleanse_header!
           assert_equal(expected_header, cleansed_header)
         end
@@ -82,13 +82,13 @@ class TabularTest < Minitest::Test
           assert_equal @allowed_columns, tabular.header.allowed_columns
         end
 
-        it "nils columns not in the whitelist" do
+        it "nils columns not in the allowed list" do
           tabular = IOStreams::Tabular.new(columns: ["   first ", "Unknown Column", "thirD   "], allowed_columns: @allowed_columns)
           header  = tabular.cleanse_header!
-          assert_equal ["first", nil, "third"], header
+          assert_equal ["first", "__rejected__Unknown Column", "third"], header
         end
 
-        it "raises exception for columns not in the whitelist" do
+        it "raises exception for columns not in the allowed list" do
           tabular = IOStreams::Tabular.new(columns: ["   first ", "Unknown Column", "thirD   "], allowed_columns: @allowed_columns, skip_unknown: false)
           exc     = assert_raises IOStreams::Errors::InvalidHeader do
             tabular.cleanse_header!
@@ -218,7 +218,7 @@ class TabularTest < Minitest::Test
         end
       end
 
-      it "skips columns not in the whitelist" do
+      it "skips columns not in the allowed list" do
         tabular.header.allowed_columns = %w[first second third fourth fifth]
         tabular.cleanse_header!
         assert hash = tabular.record_parse("1,2,3")

data/test/test_helper.rb

@@ -2,7 +2,6 @@ $LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
 
 require "yaml"
 require "minitest/autorun"
-require "minitest/reporters"
 require "iostreams"
 require "amazing_print"
 require "symmetric-encryption"
@@ -10,8 +9,6 @@ require "symmetric-encryption"
 # Since PGP libraries use UTC for Dates
 ENV["TZ"] = "UTC"
 
-Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
-
 # Test cipher
 SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(
   cipher_name: "aes-128-cbc",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 1.6.2
+  version: 1.10.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-04 00:00:00.000000000 Z
+date: 2021-08-23 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -125,14 +125,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Input and Output streaming for Ruby.