iostreams 1.5.0 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bef30882ae2eaebcfbc240858a6854f44cc54208465868e55fd6296addfc8015
-  data.tar.gz: e9984f367727934b96495a646aa271009681f9972568a0b51e9046edf451e321
+  metadata.gz: 849ceda63eb30f95762a7c985cd215d424e62afd68ab20776e8d16c188dd6aed
+  data.tar.gz: 8e26af86c40bb673ce36855a7fb30d1c4b401edc3eac0b27a71b9760cfe865dd
 SHA512:
-  metadata.gz: 4c30cb2085ce36904551bcfce2d0ec1d4544d79af891bf27660c35daa4081e1f2edaf193332ca7f15ed8324f4aaddef991a906b896d502b1574c53531f27b4d5
-  data.tar.gz: 6066bb59b519568f99121d97ec0ba2637fab650889cc898a788f71d1dedcbce7167fa435206656d3ce607bb141eee5507501825afebbbd7e37e9c2c27c915f0f
+  metadata.gz: 99318c4c64e0133df57b84429b1c2f9caa064abb1405ace5d55208e41b6bf8bb8fa83a75db8ae46d53753f10d566bab53971d95871ed4011bab4571d31bebe8a
+  data.tar.gz: bfba3a033c753e3fe05f798177b8f3c7ee8f566eabaf9223984fa902b288cb3515154f621a4e97f75b6c5bc31da88f284390c2d82c5015d7682ecf08c2a671d3

data/README.md

@@ -1,5 +1,5 @@
 # IOStreams
-[![Gem Version](https://img.shields.io/gem/v/iostreams.svg)](https://rubygems.org/gems/iostreams) [![Build Status](https://travis-ci.org/rocketjob/iostreams.svg?branch=master)](https://travis-ci.org/rocketjob/iostreams) [![Downloads](https://img.shields.io/gem/dt/iostreams.svg)](https://rubygems.org/gems/iostreams) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Gitter chat](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
+[![Gem Version](https://img.shields.io/gem/v/iostreams.svg)](https://rubygems.org/gems/iostreams) [![Downloads](https://img.shields.io/gem/dt/iostreams.svg)](https://rubygems.org/gems/iostreams) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg)
 
 IOStreams is an incredibly powerful streaming library that makes changes to file formats, compression, encryption, 
 or storage mechanism transparent to the application.
@@ -14,6 +14,18 @@ Start with the [IOStreams tutorial](https://iostreams.rocketjob.io/tutorial) to
 
 Next, checkout the remaining [IOStreams documentation](https://iostreams.rocketjob.io/)
 
+## Upgrading to v1.6
+
+The old, deprecated api's are no longer loaded by default with v1.6. To add back the deprecated api support, add
+the following line to your code:
+
+~~~ruby
+IOStreams.include(IOStreams::Deprecated)
+~~~
+
+It is important to move any of the old deprecated apis over to the new api, since they will be removed in a future
+release.
+
 ## Versioning
 
 This project adheres to [Semantic Versioning](http://semver.org/).

data/lib/io_streams/builder.rb

@@ -1,13 +1,15 @@
 module IOStreams
   # Build the streams that need to be applied to a path druing reading or writing.
   class Builder
-    attr_accessor :file_name
+    attr_accessor :file_name, :format_options
     attr_reader :streams, :options
 
     def initialize(file_name = nil)
-      @file_name = file_name
-      @streams   = nil
-      @options   = nil
+      @file_name     = file_name
+      @streams       = nil
+      @options       = nil
+      @format        = nil
+      @format_option = nil
     end
 
     # Supply an option that is only applied once the file name extensions have been parsed.
@@ -88,6 +90,20 @@ module IOStreams
       built_streams.freeze
     end
 
+    # Returns the tabular format if set, otherwise tries to autodetect the format if the file_name has been set
+    # Returns [nil] if no format is set, or if it cannot be determined from the file_name
+    def format
+      @format ||= file_name ? Tabular.format_from_file_name(file_name) : nil
+    end
+
+    def format=(format)
+      unless format.nil? || IOStreams::Tabular.registered_formats.include?(format)
+        raise(ArgumentError, "Invalid format: #{format.inspect}")
+      end
+
+      @format = format
+    end
+
     private
 
     def class_for_stream(type, stream)

data/lib/io_streams/errors.rb

@@ -9,6 +9,9 @@ module IOStreams
     class MissingHeader < Error
     end
 
+    class UnknownFormat < Error
+    end
+
     class TypeMismatch < Error
     end
 
@@ -26,6 +29,15 @@ module IOStreams
     class ValueTooLong < Error
     end
 
+    class MalformedDataError < RuntimeError
+      attr_reader :line_number
+
+      def initialize(message, line_number)
+        @line_number = line_number
+        super("#{message} on line #{line_number}.")
+      end
+    end
+
     class InvalidLayout < Error
     end
   end

data/lib/io_streams/io_streams.rb

@@ -13,8 +13,6 @@ require "uri"
 #   .zip.enc  [ :zip, :enc ]
 #   .gz.enc   [ :gz,  :enc ]
 module IOStreams
-  include Deprecated
-
   # Returns [Path] instance for the supplied complete path with optional scheme.
   #
   # Example:

data/lib/io_streams/line/reader.rb

@@ -38,12 +38,12 @@ module IOStreams
       #     Size of blocks to read from the input stream at a time.
       #     Default: 65536 ( 64K )
       #
-      # TODO:
-      # - Handle embedded line feeds when reading csv files.
-      # - Skip Comment lines. RegExp?
-      # - Skip "empty" / "blank" lines. RegExp?
-      # - Extract header line(s) / first non-comment, non-blank line
-      # - Embedded newline support, RegExp? or Proc?
+      #   embedded_within: [String]
+      #     Supports CSV files where a line may contain an embedded newline.
+      #     For CSV files set `embedded_within: '"'`
+      #
+      # Note:
+      # * When using a line reader and the file_name ends with ".csv" then embedded_within is automatically set to `"`
       def initialize(input_stream, delimiter: nil, buffer_size: 65_536, embedded_within: nil, original_file_name: nil)
         super(input_stream)
 
@@ -86,17 +86,29 @@ module IOStreams
         line_count
       end
 
-      # Reads each line per the @delimeter. It will account for embedded lines provided they are within double quotes.
-      # The embedded_within argument is set in IOStreams::LineReader
+      # Reads each line per the `delimeter`.
+      # Accounts for lines that contain the `delimiter` when the `delimeter` is within the `embedded_within` delimiter.
+      # For Example, CSV files can contain newlines embedded within double quotes.
       def readline
         line = _readline
         if line && @embedded_within
           initial_line_number = @line_number
           while line.count(@embedded_within).odd?
-            raise "Unclosed quoted field on line #{initial_line_number}" if eof? || line.length > @buffer_size * 10
-
+            if eof? || line.length > @buffer_size * 10
+              raise(Errors::MalformedDataError.new(
+                      "Unbalanced delimited field, delimiter: #{@embedded_within}",
+                      initial_line_number
+                    ))
+            end
             line << @delimiter
-            line << _readline
+            next_line = _readline
+            if next_line.nil?
+              raise(Errors::MalformedDataError.new(
+                      "Unbalanced delimited field, delimiter: #{@embedded_within}",
+                      initial_line_number
+                    ))
+            end
+            line << next_line
           end
         end
         line

data/lib/io_streams/path.rb

@@ -153,7 +153,7 @@ module IOStreams
     # Returns [true|false] whether the file is compressed based on its file extensions.
     def compressed?
       # TODO: Look at streams?
-      !(path =~ /\.(zip|gz|gzip|xls.|)\z/i).nil?
+      !(path =~ /\.(zip|gz|gzip|xlsx|xlsm|bz2)\z/i).nil?
     end
 
     # Returns [true|false] whether the file is encrypted based on its file extensions.

data/lib/io_streams/paths/s3.rb

@@ -3,7 +3,10 @@ require "uri"
 module IOStreams
   module Paths
     class S3 < IOStreams::Path
-      attr_reader :bucket_name, :client, :options
+      attr_reader :bucket_name, :options
+
+      # Largest file size supported by the S3 copy object api.
+      S3_COPY_OBJECT_SIZE_LIMIT = 5 * 1024 * 1024 * 1024
 
       # Arguments:
       #
@@ -138,16 +141,17 @@ module IOStreams
 
         @bucket_name = uri.hostname
         key          = uri.path.sub(%r{\A/}, "")
-        if client.is_a?(Hash)
-          client[:access_key_id]     = access_key_id if access_key_id
-          client[:secret_access_key] = secret_access_key if secret_access_key
-          @client                    = ::Aws::S3::Client.new(client)
+
+        if client && !client.is_a?(Hash)
+          @client = client
         else
-          @client = client || ::Aws::S3::Client.new(access_key_id: access_key_id, secret_access_key: secret_access_key)
+          @client_options                     = client.is_a?(Hash) ? client.dup : {}
+          @client_options[:access_key_id]     = access_key_id if access_key_id
+          @client_options[:secret_access_key] = secret_access_key if secret_access_key
         end
-        @options = args
 
-        @options.merge(uri.query) if uri.query
+        @options = args
+        @options.merge!(uri.query.transform_keys(&:to_sym)) if uri.query
 
         super(key)
       end
@@ -187,11 +191,11 @@ module IOStreams
       end
 
       # Make S3 perform direct copies within S3 itself.
-      def copy_to(target_path, convert: true)
-        return super(target_path) if convert
+      def copy_to(target_path, convert: true, **args)
+        return super(target_path, convert: convert, **args) if convert || (size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
 
         target = IOStreams.new(target_path)
-        return super(target) unless target.is_a?(self.class)
+        return super(target, convert: convert, **args) unless target.is_a?(self.class)
 
         source_name = ::File.join(bucket_name, path)
         client.copy_object(options.merge(bucket: target.bucket_name, key: target.path, copy_source: source_name))
@@ -199,11 +203,13 @@ module IOStreams
       end
 
       # Make S3 perform direct copies within S3 itself.
-      def copy_from(source_path, convert: true)
-        return super(source_path) if convert
+      def copy_from(source_path, convert: true, **args)
+        return super(source_path, convert: true, **args) if convert
 
         source = IOStreams.new(source_path)
-        return super(source, **args) unless source.is_a?(self.class)
+        if !source.is_a?(self.class) || (source.size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
+          return super(source, convert: convert, **args)
+        end
 
         source_name = ::File.join(source.bucket_name, source.path)
         client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))
@@ -309,6 +315,11 @@ module IOStreams
       def partial_files_visible?
         false
       end
+
+      # Lazy load S3 client since it takes two seconds to create itself!
+      def client
+        @client ||= ::Aws::S3::Client.new(@client_options)
+      end
     end
   end
 end

data/lib/io_streams/paths/sftp.rb

@@ -26,12 +26,13 @@ module IOStreams
       include SemanticLogger::Loggable if defined?(SemanticLogger)
 
       class << self
-        attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds
+        attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds, :before_password_wait_seconds
       end
 
-      @sftp_bin             = "sftp"
-      @sshpass_bin          = "sshpass"
-      @sshpass_wait_seconds = 5
+      @sftp_bin                     = "sftp"
+      @sshpass_bin                  = "sshpass"
+      @before_password_wait_seconds = 2
+      @sshpass_wait_seconds         = 5
 
       attr_reader :hostname, :username, :ssh_options, :url, :port
 
@@ -46,9 +47,23 @@ module IOStreams
       # password: [String]
       #   Password for the user.
       #
-      # **ssh_options
-      #   Any other options supported by ssh_config.
-      #   `man ssh_config` to see all available options.
+      # ssh_options: [Hash]
+      #   - IdentityKey [String]
+      #     The identity key that this client should use to talk to this host.
+      #     Under the covers this value is written to a file and then the file name is passed as `IdentityFile`
+      #   - HostKey [String]
+      #     The expected SSH Host key that is presented by the remote host.
+      #     Instead of storing the host key in the `known_hosts` file, it can be supplied explicity
+      #     using this option.
+      #     Under the covers this value is written to a file and then the file name is passed as `UserKnownHostsFile`
+      #     Notes:
+      #     - It must contain the entire line that would be stored in `known_hosts`,
+      #       including the hostname, ip address, key type and key value. This value is written as-is into a
+      #       "known_hosts" like file and then passed into sftp using the `UserKnownHostsFile` option.
+      #     - The easiest way to generate the required is to use `ssh-keyscan` and then supply that value in this field.
+      #       For example: `ssh-keyscan hostname`
+      #   - Any other options supported by ssh_config.
+      #     `man ssh_config` to see all available options.
       #
       # Examples:
       #
@@ -167,33 +182,36 @@ module IOStreams
       def sftp_download(remote_file_name, local_file_name)
         with_sftp_args do |args|
           Open3.popen2e(*args) do |writer, reader, waith_thr|
-            begin
-              writer.puts password
-              # Give time for password to be processed and stdin to be passed to sftp process.
-              sleep self.class.sshpass_wait_seconds
-              writer.puts "get #{remote_file_name} #{local_file_name}"
-              writer.puts "bye"
-              writer.close
-              out = reader.read.chomp
-              unless waith_thr.value.success?
-                raise(
-                  Errors::CommunicationsFailure,
-                  "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
-                )
-              end
-
-              out
-            rescue Errno::EPIPE
-              out = begin
-                      reader.read.chomp
-                    rescue StandardError
-                      nil
-                    end
+            # Give time for remote sftp server to get ready to accept the password.
+            sleep self.class.before_password_wait_seconds
+
+            writer.puts password
+
+            # Give time for password to be processed and stdin to be passed to sftp process.
+            sleep self.class.sshpass_wait_seconds
+
+            writer.puts "get #{remote_file_name} #{local_file_name}"
+            writer.puts "bye"
+            writer.close
+            out = reader.read.chomp
+            unless waith_thr.value.success?
               raise(
                 Errors::CommunicationsFailure,
                 "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
               )
             end
+
+            out
+          rescue Errno::EPIPE
+            out = begin
+              reader.read.chomp
+            rescue StandardError
+              nil
+            end
+            raise(
+              Errors::CommunicationsFailure,
+              "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
+            )
           end
         end
       end
@@ -201,48 +219,64 @@ module IOStreams
       def sftp_upload(local_file_name, remote_file_name)
         with_sftp_args do |args|
           Open3.popen2e(*args) do |writer, reader, waith_thr|
-            begin
-              writer.puts(password) if password
-              # Give time for password to be processed and stdin to be passed to sftp process.
-              sleep self.class.sshpass_wait_seconds
-              writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
-              writer.puts "bye"
-              writer.close
-              out = reader.read.chomp
-              unless waith_thr.value.success?
-                raise(
-                  Errors::CommunicationsFailure,
-                  "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
-                )
-              end
-
-              out
-            rescue Errno::EPIPE
-              out = begin
-                      reader.read.chomp
-                    rescue StandardError
-                      nil
-                    end
+            writer.puts(password) if password
+            # Give time for password to be processed and stdin to be passed to sftp process.
+            sleep self.class.sshpass_wait_seconds
+            writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
+            writer.puts "bye"
+            writer.close
+            out = reader.read.chomp
+            unless waith_thr.value.success?
               raise(
                 Errors::CommunicationsFailure,
                 "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
               )
             end
+
+            out
+          rescue Errno::EPIPE
+            out = begin
+              reader.read.chomp
+            rescue StandardError
+              nil
+            end
+            raise(
+              Errors::CommunicationsFailure,
+              "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
+            )
           end
         end
       end
 
       def with_sftp_args
-        return yield sftp_args(ssh_options) unless ssh_options.key?("IdentityKey")
+        return yield sftp_args(ssh_options) if !ssh_options.key?("IdentityKey") && !ssh_options.key?("HostKey")
+
+        with_identity_key(ssh_options.dup) do |options|
+          with_host_key(options) do |options2|
+            yield sftp_args(options2)
+          end
+        end
+      end
+
+      def with_identity_key(options)
+        return yield options unless ssh_options.key?("IdentityKey")
+
+        with_temp_file(options, "IdentityFile", options.delete("IdentityKey")) { yield options }
+      end
+
+      def with_host_key(options)
+        return yield options unless ssh_options.key?("HostKey")
+
+        with_temp_file(options, "UserKnownHostsFile", options.delete("HostKey")) { yield options }
+      end
 
+      def with_temp_file(options, option, value)
         Utils.temp_file_name("iostreams-sftp-args", "key") do |file_name|
-          options = ssh_options.dup
-          key     = options.delete("IdentityKey")
           # sftp requires that private key is only readable by the current user
-          ::File.open(file_name, "wb", 0o600) { |io| io.write(key) }
+          ::File.open(file_name, "wb", 0o600) { |io| io.write(value) }
 
-          options["IdentityFile"] = file_name
-          yield sftp_args(options)
+          options[option] = file_name
+          yield options
         end
       end
 
@@ -272,8 +306,8 @@ module IOStreams
 
       def build_ssh_options
         options = ssh_options.dup
-        options[:logger] ||= logger if defined?(SemanticLogger)
-        options[:port] ||= port
+        options[:logger]       ||= logger if defined?(SemanticLogger)
+        options[:port]         ||= port
         options[:max_pkt_size] ||= 65_536
         options[:password]     ||= @password
         options