iostreams 1.4.0 → 1.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36a51f41f33a1c58f1d5d03ed857784e86de4588ca38aa5a596e479f8661faf3
-  data.tar.gz: 23dd967c21581520675799692296bb901e2dc539f008fa30623f4bd72a2c90f3
+  metadata.gz: a6dd68508835099ef4c9de7f81a9527927d3225138f0bf5ecd00c44194e11858
+  data.tar.gz: 28535e95ca83a4cf0c522de4cd48889a489b4fcefdc8feb9a7bfe0b70124fd7a
 SHA512:
-  metadata.gz: 81d5f7ea50b7a5b06a5d26b758eb0639b0c820735cd0cf1545c6298f244b96b0070d3f8a11deed0ef8398b5b6c461561b3c43661612e3eeda19bdf61abb549ab
-  data.tar.gz: 19f6f2051351533029cd7fca3cd6e7be3a4e5d3ea4ff2ae512b0b98a12b84639ae63e6a4decc0ff3edf1d50d5c14ddc1b869767f2f6c8793fc392f6ad02d939f
+  metadata.gz: 8e4d38ef41234f62fdcfde1ae3a96fe59203dcfc38562843106b081c7292efffdda36b88707674059a24b0fa2996d4c1dbe0627694e80a9b12d23d47195a53d3
+  data.tar.gz: 23e16854c305542ee0976f570444d7b99cadcb6ec460d83bebc73a65cba9c28e3beba485c9943797b174b80bdd783c216d89d539e8aecdbda62e1c31f017efce

data/README.md

@@ -1,5 +1,5 @@
 # IOStreams
-[![Gem Version](https://img.shields.io/gem/v/iostreams.svg)](https://rubygems.org/gems/iostreams) [![Build Status](https://travis-ci.org/rocketjob/iostreams.svg?branch=master)](https://travis-ci.org/rocketjob/iostreams) [![Downloads](https://img.shields.io/gem/dt/iostreams.svg)](https://rubygems.org/gems/iostreams) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Gitter chat](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
+[![Gem Version](https://img.shields.io/gem/v/iostreams.svg)](https://rubygems.org/gems/iostreams) [![Downloads](https://img.shields.io/gem/dt/iostreams.svg)](https://rubygems.org/gems/iostreams) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg)
 
 IOStreams is an incredibly powerful streaming library that makes changes to file formats, compression, encryption, 
 or storage mechanism transparent to the application.
@@ -14,6 +14,18 @@ Start with the [IOStreams tutorial](https://iostreams.rocketjob.io/tutorial) to
 
 Next, checkout the remaining [IOStreams documentation](https://iostreams.rocketjob.io/)
 
+## Upgrading to v1.6
+
+The old, deprecated api's are no longer loaded by default with v1.6. To add back the deprecated api support, add
+the following line to your code:
+
+~~~ruby
+IOStreams.include(IOStreams::Deprecated)
+~~~
+
+It is important to move any of the old deprecated apis over to the new api, since they will be removed in a future
+release.
+
 ## Versioning
 
 This project adheres to [Semantic Versioning](http://semver.org/).

data/lib/io_streams/builder.rb

@@ -1,13 +1,15 @@
 module IOStreams
   # Build the streams that need to be applied to a path druing reading or writing.
   class Builder
-    attr_accessor :file_name
+    attr_accessor :file_name, :format_options
     attr_reader :streams, :options
 
     def initialize(file_name = nil)
-      @file_name = file_name
-      @streams   = nil
-      @options   = nil
+      @file_name     = file_name
+      @streams       = nil
+      @options       = nil
+      @format        = nil
+      @format_option = nil
     end
 
     # Supply an option that is only applied once the file name extensions have been parsed.
@@ -88,11 +90,23 @@ module IOStreams
       built_streams.freeze
     end
 
+    # Returns the tabular format if set, otherwise tries to autodetect the format if the file_name has been set
+    # Returns [nil] if no format is set, or if it cannot be determined from the file_name
+    def format
+      @format ||= file_name ? Tabular.format_from_file_name(file_name) : nil
+    end
+
+    def format=(format)
+      raise(ArgumentError, "Invalid format: #{format.inspect}") unless format.nil? || IOStreams::Tabular.registered_formats.include?(format)
+
+      @format = format
+    end
+
     private
 
     def class_for_stream(type, stream)
       ext = IOStreams.extensions[stream.nil? ? nil : stream.to_sym] ||
-            raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
+        raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
       ext.send("#{type}_class") || raise(ArgumentError, "No #{type} registered for Stream type: #{stream.inspect}")
     end
 

data/lib/io_streams/bzip2/reader.rb

@@ -3,9 +3,7 @@ module IOStreams
     class Reader < IOStreams::Reader
       # Read from a Bzip2 stream, decompressing the contents as it is read
       def self.stream(input_stream, **args)
-        unless defined?(::Bzip2::FFI)
-          Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi")
-        end
+        Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
 
         begin
           io = ::Bzip2::FFI::Reader.new(input_stream, args)

data/lib/io_streams/bzip2/writer.rb

@@ -3,9 +3,7 @@ module IOStreams
     class Writer < IOStreams::Writer
       # Write to a stream, compressing with Bzip2
       def self.stream(input_stream, original_file_name: nil, **args)
-        unless defined?(::Bzip2::FFI)
-          Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi")
-        end
+        Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
 
         begin
           io = ::Bzip2::FFI::Writer.new(input_stream, args)

data/lib/io_streams/encode/reader.rb

@@ -78,8 +78,6 @@ module IOStreams
         block
       end
 
-      private
-
       def self.extract_cleaner(cleaner)
         return if cleaner.nil?
 

data/lib/io_streams/errors.rb

@@ -9,6 +9,9 @@ module IOStreams
     class MissingHeader < Error
     end
 
+    class UnknownFormat < Error
+    end
+
     class TypeMismatch < Error
     end
 
@@ -26,6 +29,15 @@ module IOStreams
     class ValueTooLong < Error
     end
 
+    class MalformedDataError < RuntimeError
+      attr_reader :line_number
+
+      def initialize(message, line_number)
+        @line_number = line_number
+        super("#{message} on line #{line_number}.")
+      end
+    end
+
     class InvalidLayout < Error
     end
   end

data/lib/io_streams/io_streams.rb

@@ -13,8 +13,6 @@ require "uri"
 #   .zip.enc  [ :zip, :enc ]
 #   .gz.enc   [ :gz,  :enc ]
 module IOStreams
-  include Deprecated
-
   # Returns [Path] instance for the supplied complete path with optional scheme.
   #
   # Example:
@@ -298,8 +296,6 @@ module IOStreams
     @schemes[scheme_name.nil? ? nil : scheme_name.to_sym] || raise(ArgumentError, "Unknown Scheme type: #{scheme_name.inspect}")
   end
 
-  private
-
   Extension = Struct.new(:reader_class, :writer_class)
 
   # Hold root paths

data/lib/io_streams/line/reader.rb

@@ -38,12 +38,12 @@ module IOStreams
       #     Size of blocks to read from the input stream at a time.
       #     Default: 65536 ( 64K )
       #
-      # TODO:
-      # - Handle embedded line feeds when reading csv files.
-      # - Skip Comment lines. RegExp?
-      # - Skip "empty" / "blank" lines. RegExp?
-      # - Extract header line(s) / first non-comment, non-blank line
-      # - Embedded newline support, RegExp? or Proc?
+      #   embedded_within: [String]
+      #     Supports CSV files where a line may contain an embedded newline.
+      #     For CSV files set `embedded_within: '"'`
+      #
+      # Note:
+      # * When using a line reader and the file_name ends with ".csv" then embedded_within is automatically set to `"`
       def initialize(input_stream, delimiter: nil, buffer_size: 65_536, embedded_within: nil, original_file_name: nil)
         super(input_stream)
 
@@ -63,11 +63,11 @@ module IOStreams
         # Auto-detect windows/linux line endings if not supplied. \n or \r\n
         @delimiter ||= auto_detect_line_endings
 
-        if @buffer
-          # Change the delimiters encoding to match that of the input stream
-          @delimiter      = @delimiter.encode(@buffer.encoding)
-          @delimiter_size = @delimiter.size
-        end
+        return unless @buffer
+
+        # Change the delimiters encoding to match that of the input stream
+        @delimiter      = @delimiter.encode(@buffer.encoding)
+        @delimiter_size = @delimiter.size
       end
 
       # Iterate over every line in the file/stream passing each line to supplied block in turn.
@@ -86,17 +86,29 @@ module IOStreams
         line_count
       end
 
-      # Reads each line per the @delimeter. It will account for embedded lines provided they are within double quotes.
-      # The embedded_within argument is set in IOStreams::LineReader
+      # Reads each line per the `delimeter`.
+      # Accounts for lines that contain the `delimiter` when the `delimeter` is within the `embedded_within` delimiter.
+      # For Example, CSV files can contain newlines embedded within double quotes.
       def readline
         line = _readline
         if line && @embedded_within
           initial_line_number = @line_number
           while line.count(@embedded_within).odd?
-            raise "Unclosed quoted field on line #{initial_line_number}" if eof? || line.length > @buffer_size * 10
-
+            if eof? || line.length > @buffer_size * 10
+              raise(Errors::MalformedDataError.new(
+                "Unbalanced delimited field, delimiter: #{@embedded_within}",
+                initial_line_number
+              ))
+            end
             line << @delimiter
-            line << _readline
+            next_line = _readline
+            if next_line.nil?
+              raise(Errors::MalformedDataError.new(
+                "Unbalanced delimited field, delimiter: #{@embedded_within}",
+                initial_line_number
+              ))
+            end
+            line << next_line
           end
         end
         line

data/lib/io_streams/path.rb

@@ -82,6 +82,7 @@ module IOStreams
     end
 
     # Cleanup an incomplete write to the target "file" if the copy fails.
+    # rubocop:disable Lint/SuppressedException
     def copy_from(source, **args)
       super(source, **args)
     rescue StandardError => e
@@ -91,6 +92,7 @@ module IOStreams
       end
       raise(e)
     end
+    # rubocop:enable Lint/SuppressedException
 
     # Moves the file by copying it to the new path and then deleting the current path.
     # Returns [IOStreams::Path] the target path.
@@ -151,7 +153,7 @@ module IOStreams
     # Returns [true|false] whether the file is compressed based on its file extensions.
     def compressed?
       # TODO: Look at streams?
-      !(path =~ /\.(zip|gz|gzip|xls.|)\z/i).nil?
+      !(path =~ /\.(zip|gz|gzip|xlsx|xlsm|bz2)\z/i).nil?
     end
 
     # Returns [true|false] whether the file is encrypted based on its file extensions.

data/lib/io_streams/paths/s3.rb

@@ -5,6 +5,9 @@ module IOStreams
     class S3 < IOStreams::Path
       attr_reader :bucket_name, :client, :options
 
+      # Largest file size supported by the S3 copy object api.
+      S3_COPY_OBJECT_SIZE_LIMIT = 5 * 1024 * 1024 * 1024
+
       # Arguments:
       #
       # url: [String]
@@ -188,7 +191,7 @@ module IOStreams
 
       # Make S3 perform direct copies within S3 itself.
       def copy_to(target_path, convert: true)
-        return super(target_path) if convert
+        return super(target_path) if convert || (size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
 
         target = IOStreams.new(target_path)
         return super(target) unless target.is_a?(self.class)
@@ -203,7 +206,7 @@ module IOStreams
         return super(source_path) if convert
 
         source = IOStreams.new(source_path)
-        return super(source, **args) unless source.is_a?(self.class)
+        return super(source) if !source.is_a?(self.class) || (source.size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
 
         source_name = ::File.join(source.bucket_name, source.path)
         client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))

data/lib/io_streams/paths/sftp.rb

@@ -26,12 +26,13 @@ module IOStreams
       include SemanticLogger::Loggable if defined?(SemanticLogger)
 
       class << self
-        attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds
+        attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds, :before_password_wait_seconds
       end
 
-      @sftp_bin             = "sftp"
-      @sshpass_bin          = "sshpass"
-      @sshpass_wait_seconds = 5
+      @sftp_bin                     = "sftp"
+      @sshpass_bin                  = "sshpass"
+      @before_password_wait_seconds = 2
+      @sshpass_wait_seconds         = 5
 
       attr_reader :hostname, :username, :ssh_options, :url, :port
 
@@ -71,7 +72,9 @@ module IOStreams
       #   end
       #
       #   # When using the sftp executable use an identity file instead of a password to authenticate:
-      #   IOStreams.path("sftp://test.com/path/file_name.csv", username: "jack", ssh_options: {IdentityFile: "~/.ssh/private_key"}).reader do |io|
+      #   IOStreams.path("sftp://test.com/path/file_name.csv",
+      #                  username:    "jack",
+      #                  ssh_options: {IdentityFile: "~/.ssh/private_key"}).reader do |io|
       #     puts io.read
       #   end
       def initialize(url, username: nil, password: nil, ssh_options: {})
@@ -122,7 +125,8 @@ module IOStreams
       #   end
       #
       # Example Output:
-      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group", :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
+      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
+      #   :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
       def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
         Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
 
@@ -165,9 +169,14 @@ module IOStreams
         with_sftp_args do |args|
           Open3.popen2e(*args) do |writer, reader, waith_thr|
             begin
+              # Give time for remote sftp server to get ready to accept the password.
+              sleep self.class.before_password_wait_seconds
+
               writer.puts password
+
               # Give time for password to be processed and stdin to be passed to sftp process.
               sleep self.class.sshpass_wait_seconds
+
               writer.puts "get #{remote_file_name} #{local_file_name}"
               writer.puts "bye"
               writer.close
@@ -268,9 +277,9 @@ module IOStreams
       end
 
       def build_ssh_options
-        options = ssh_options.dup
-        options[:logger] ||= logger if defined?(SemanticLogger)
-        options[:port] ||= port
+        options                = ssh_options.dup
+        options[:logger]       ||= logger if defined?(SemanticLogger)
+        options[:port]         ||= port
         options[:max_pkt_size] ||= 65_536
         options[:password]     ||= @password
         options

data/lib/io_streams/pgp.rb

@@ -74,9 +74,10 @@ module IOStreams
 
       raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}") unless status.success?
 
-      if (match = err.match(/gpg: key ([0-9A-F]+)\s+/))
-        match[1]
-      end
+      match = err.match(/gpg: key ([0-9A-F]+)\s+/)
+      return unless match
+
+      match[1]
     end
 
     # Delete all private and public keys for a particular email.
@@ -110,11 +111,6 @@ module IOStreams
       !list_keys(email: email, key_id: key_id, private: private).empty?
     end
 
-    # Deprecated
-    def self.has_key?(**args)
-      key?(**args)
-    end
-
     # Returns [Array<Hash>] the list of keys.
     #   Each Hash consists of:
     #     key_length: [Integer]
@@ -232,7 +228,7 @@ module IOStreams
         err.each_line do |line|
           if line =~ /secret key imported/
             secret = true
-          elsif match = line.match(/key\s+(\w+):\s+(\w+).+\"(.*)<(.*)>\"/)
+          elsif (match = line.match(/key\s+(\w+):\s+(\w+).+\"(.*)<(.*)>\"/))
             results << {
               key_id:  match[1].to_s.strip,
               private: secret,
@@ -265,7 +261,7 @@ module IOStreams
 
       import(key: key)
       set_trust(email: email, key_id: key_id)
-      email
+      email || key_id
     end
 
     # Set the trust level for an existing key.
@@ -347,8 +343,6 @@ module IOStreams
       end
     end
 
-    private
-
     @logger = nil
 
     def self.logger
@@ -356,7 +350,7 @@ module IOStreams
     end
 
     def self.version_check
-      return unless pgp_version.to_f >= 2.3
+      return unless pgp_version.to_f >= 2.4
 
       raise(
         Pgp::UnsupportedVersion,

data/lib/io_streams/stream.rb

@@ -191,11 +191,41 @@ module IOStreams
       end
     end
 
-    # Set/get the original file_name
+    # Set the original file_name
     def file_name=(file_name)
       builder.file_name = file_name
     end
 
+    # Set/get the tabular format_options
+    def format(format = :none)
+      if format == :none
+        builder.format
+      else
+        builder.format = format
+        self
+      end
+    end
+
+    # Set the tabular format
+    def format=(format)
+      builder.format = format
+    end
+
+    # Set/get the tabular format options
+    def format_options(format_options = :none)
+      if format_options == :none
+        builder.format_options
+      else
+        builder.format_options = format_options
+        self
+      end
+    end
+
+    # Set the tabular format_options
+    def format_options=(format_options)
+      builder.format_options = format_options
+    end
+
     # Returns [String] the last component of this path.
     # Returns `nil` if no `file_name` was set.
     #
@@ -293,14 +323,26 @@ module IOStreams
     # Iterate over a file / stream returning each line as an array, one at a time.
     def row_reader(delimiter: nil, embedded_within: nil, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
-        yield IOStreams::Row::Reader.new(io, original_file_name: builder.file_name, **args)
+        yield IOStreams::Row::Reader.new(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args
+        )
       end
     end
 
     # Iterate over a file / stream returning each line as a hash, one at a time.
     def record_reader(delimiter: nil, embedded_within: nil, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
-        yield IOStreams::Record::Reader.new(io, original_file_name: builder.file_name, **args)
+        yield IOStreams::Record::Reader.new(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args
+        )
       end
     end
 
@@ -320,7 +362,14 @@ module IOStreams
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Row::Writer)
 
       line_writer(delimiter: delimiter) do |io|
-        IOStreams::Row::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+        IOStreams::Row::Writer.stream(
+          io,
+          original_file_name: builder.file_name,
+          format: builder.format,
+          format_options: builder.format_options,
+          **args,
+          &block
+        )
       end
     end
 
@@ -328,7 +377,13 @@ module IOStreams
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Record::Writer)
 
       line_writer(delimiter: delimiter) do |io|
-        IOStreams::Record::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+        IOStreams::Record::Writer.stream(
+          io,
+          original_file_name: builder.file_name,
+          format: builder.format,
+          format_options: builder.format_options,
+          **args,
+          &block)
       end
     end
   end