iostreams 1.3.2 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b743c145f120e461d09eed5b9c7476bacbdaa18453200e1ccc57ec9eef2239c
-  data.tar.gz: 7c54a49f344e454ef64a1e19e47f2c665b46e675eda34c03e1604668a6543de4
+  metadata.gz: e4fb750e5c3779000fac8f21803b84a5977f111f481b7d4e11a117149cd0d9e6
+  data.tar.gz: '09cb2dc7ff67afd44d3ebcfefa29f6a1840d1b3dd871c33bc1a06914cd8bee2d'
 SHA512:
-  metadata.gz: 27d4a3875009d1902285a9e01b67032e74e1eac6d295de3400096b01c663de8b14cb594e4ffd8f04154d23cf7cfd4a696dbc33b9651fd200808ff087c00c0dc2
-  data.tar.gz: 92656ad55f37e5ffb3b4588cc93d6d9c988ce31705f888d4e045d86ffcf5047f4185a8de30bc6c262a48ae7f048433edd00f827d155939db7caa858e5e495bf0
+  metadata.gz: 0cf2db14e03b9e81e0e39119b35f293408a9d4b6bf3365abc724d95d7376abf7af73720b8fd8d000b70b2eb7abbae20055753e845a77dec2d8d7297e5b6693ba
+  data.tar.gz: d9fa2194965ef99a99e1ecd0655e84066e6021236d1494336c40693b103d3c7ecb7e5a87e8f179a38c990ad7670d6bd4901b6a7cd8b89757c00a3179d657a1cc

data/README.md

@@ -1,5 +1,5 @@
 # IOStreams
-[![Gem Version](https://img.shields.io/gem/v/iostreams.svg)](https://rubygems.org/gems/iostreams) [![Build Status](https://travis-ci.org/rocketjob/iostreams.svg?branch=master)](https://travis-ci.org/rocketjob/iostreams) [![Downloads](https://img.shields.io/gem/dt/iostreams.svg)](https://rubygems.org/gems/iostreams) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Gitter chat](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
+[![Gem Version](https://img.shields.io/gem/v/iostreams.svg)](https://rubygems.org/gems/iostreams) [![Downloads](https://img.shields.io/gem/dt/iostreams.svg)](https://rubygems.org/gems/iostreams) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Gitter chat](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
 
 IOStreams is an incredibly powerful streaming library that makes changes to file formats, compression, encryption, 
 or storage mechanism transparent to the application.
@@ -14,6 +14,18 @@ Start with the [IOStreams tutorial](https://iostreams.rocketjob.io/tutorial) to
 
 Next, checkout the remaining [IOStreams documentation](https://iostreams.rocketjob.io/)
 
+## Upgrading to v1.6
+
+The old, deprecated api's are no longer loaded by default with v1.6. To add back the deprecated api support, add
+the following line to your code:
+
+~~~ruby
+IOStreams.include(IOStreams::Deprecated)
+~~~
+
+It is important to move any of the old deprecated apis over to the new api, since they will be removed in a future
+release.
+
 ## Versioning
 
 This project adheres to [Semantic Versioning](http://semver.org/).

data/lib/io_streams/builder.rb

@@ -1,13 +1,15 @@
 module IOStreams
   # Build the streams that need to be applied to a path druing reading or writing.
   class Builder
-    attr_accessor :file_name
+    attr_accessor :file_name, :format_options
     attr_reader :streams, :options
 
     def initialize(file_name = nil)
-      @file_name = file_name
-      @streams   = nil
-      @options   = nil
+      @file_name     = file_name
+      @streams       = nil
+      @options       = nil
+      @format        = nil
+      @format_option = nil
     end
 
     # Supply an option that is only applied once the file name extensions have been parsed.
@@ -88,11 +90,23 @@ module IOStreams
       built_streams.freeze
     end
 
+    # Returns the tabular format if set, otherwise tries to autodetect the format if the file_name has been set
+    # Returns [nil] if no format is set, or if it cannot be determined from the file_name
+    def format
+      @format ||= file_name ? Tabular.format_from_file_name(file_name) : nil
+    end
+
+    def format=(format)
+      raise(ArgumentError, "Invalid format: #{format.inspect}") unless format.nil? || IOStreams::Tabular.registered_formats.include?(format)
+
+      @format = format
+    end
+
     private
 
     def class_for_stream(type, stream)
       ext = IOStreams.extensions[stream.nil? ? nil : stream.to_sym] ||
-            raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
+        raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
       ext.send("#{type}_class") || raise(ArgumentError, "No #{type} registered for Stream type: #{stream.inspect}")
     end
 

data/lib/io_streams/bzip2/reader.rb

@@ -2,11 +2,11 @@ module IOStreams
   module Bzip2
     class Reader < IOStreams::Reader
       # Read from a Bzip2 stream, decompressing the contents as it is read
-      def self.stream(input_stream, **_args)
-        Utils.load_soft_dependency("rbzip2", "Bzip2") unless defined?(RBzip2)
+      def self.stream(input_stream, **args)
+        Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
 
         begin
-          io = RBzip2.default_adapter::Decompressor.new(input_stream)
+          io = ::Bzip2::FFI::Reader.new(input_stream, args)
           yield io
         ensure
           io&.close

data/lib/io_streams/bzip2/writer.rb

@@ -2,11 +2,11 @@ module IOStreams
   module Bzip2
     class Writer < IOStreams::Writer
       # Write to a stream, compressing with Bzip2
-      def self.stream(input_stream, original_file_name: nil, **_args)
-        Utils.load_soft_dependency("rbzip2", "Bzip2") unless defined?(RBzip2)
+      def self.stream(input_stream, original_file_name: nil, **args)
+        Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi") unless defined?(::Bzip2::FFI)
 
         begin
-          io = RBzip2.default_adapter::Compressor.new(input_stream)
+          io = ::Bzip2::FFI::Writer.new(input_stream, args)
           yield io
         ensure
           io&.close

data/lib/io_streams/encode/reader.rb

@@ -78,8 +78,6 @@ module IOStreams
         block
       end
 
-      private
-
       def self.extract_cleaner(cleaner)
         return if cleaner.nil?
 

data/lib/io_streams/errors.rb

@@ -9,6 +9,9 @@ module IOStreams
     class MissingHeader < Error
     end
 
+    class UnknownFormat < Error
+    end
+
     class TypeMismatch < Error
     end
 
@@ -26,6 +29,15 @@ module IOStreams
     class ValueTooLong < Error
     end
 
+    class MalformedDataError < RuntimeError
+      attr_reader :line_number
+
+      def initialize(message, line_number)
+        @line_number = line_number
+        super("#{message} on line #{line_number}.")
+      end
+    end
+
     class InvalidLayout < Error
     end
   end

data/lib/io_streams/io_streams.rb

@@ -13,8 +13,6 @@ require "uri"
 #   .zip.enc  [ :zip, :enc ]
 #   .gz.enc   [ :gz,  :enc ]
 module IOStreams
-  include Deprecated
-
   # Returns [Path] instance for the supplied complete path with optional scheme.
   #
   # Example:
@@ -298,8 +296,6 @@ module IOStreams
     @schemes[scheme_name.nil? ? nil : scheme_name.to_sym] || raise(ArgumentError, "Unknown Scheme type: #{scheme_name.inspect}")
   end
 
-  private
-
   Extension = Struct.new(:reader_class, :writer_class)
 
   # Hold root paths

data/lib/io_streams/line/reader.rb

@@ -38,12 +38,12 @@ module IOStreams
       #     Size of blocks to read from the input stream at a time.
       #     Default: 65536 ( 64K )
       #
-      # TODO:
-      # - Handle embedded line feeds when reading csv files.
-      # - Skip Comment lines. RegExp?
-      # - Skip "empty" / "blank" lines. RegExp?
-      # - Extract header line(s) / first non-comment, non-blank line
-      # - Embedded newline support, RegExp? or Proc?
+      #   embedded_within: [String]
+      #     Supports CSV files where a line may contain an embedded newline.
+      #     For CSV files set `embedded_within: '"'`
+      #
+      # Note:
+      # * When using a line reader and the file_name ends with ".csv" then embedded_within is automatically set to `"`
       def initialize(input_stream, delimiter: nil, buffer_size: 65_536, embedded_within: nil, original_file_name: nil)
         super(input_stream)
 
@@ -63,11 +63,11 @@ module IOStreams
         # Auto-detect windows/linux line endings if not supplied. \n or \r\n
         @delimiter ||= auto_detect_line_endings
 
-        if @buffer
-          # Change the delimiters encoding to match that of the input stream
-          @delimiter      = @delimiter.encode(@buffer.encoding)
-          @delimiter_size = @delimiter.size
-        end
+        return unless @buffer
+
+        # Change the delimiters encoding to match that of the input stream
+        @delimiter      = @delimiter.encode(@buffer.encoding)
+        @delimiter_size = @delimiter.size
       end
 
       # Iterate over every line in the file/stream passing each line to supplied block in turn.
@@ -86,17 +86,29 @@ module IOStreams
         line_count
       end
 
-      # Reads each line per the @delimeter. It will account for embedded lines provided they are within double quotes.
-      # The embedded_within argument is set in IOStreams::LineReader
+      # Reads each line per the `delimeter`.
+      # Accounts for lines that contain the `delimiter` when the `delimeter` is within the `embedded_within` delimiter.
+      # For Example, CSV files can contain newlines embedded within double quotes.
       def readline
         line = _readline
         if line && @embedded_within
           initial_line_number = @line_number
           while line.count(@embedded_within).odd?
-            raise "Unclosed quoted field on line #{initial_line_number}" if eof? || line.length > @buffer_size * 10
-
+            if eof? || line.length > @buffer_size * 10
+              raise(Errors::MalformedDataError.new(
+                "Unbalanced delimited field, delimiter: #{@embedded_within}",
+                initial_line_number
+              ))
+            end
             line << @delimiter
-            line << _readline
+            next_line = _readline
+            if next_line.nil?
+              raise(Errors::MalformedDataError.new(
+                "Unbalanced delimited field, delimiter: #{@embedded_within}",
+                initial_line_number
+              ))
+            end
+            line << next_line
           end
         end
         line

data/lib/io_streams/path.rb

@@ -82,6 +82,7 @@ module IOStreams
     end
 
     # Cleanup an incomplete write to the target "file" if the copy fails.
+    # rubocop:disable Lint/SuppressedException
     def copy_from(source, **args)
       super(source, **args)
     rescue StandardError => e
@@ -91,6 +92,7 @@ module IOStreams
       end
       raise(e)
     end
+    # rubocop:enable Lint/SuppressedException
 
     # Moves the file by copying it to the new path and then deleting the current path.
     # Returns [IOStreams::Path] the target path.
@@ -151,7 +153,7 @@ module IOStreams
     # Returns [true|false] whether the file is compressed based on its file extensions.
     def compressed?
       # TODO: Look at streams?
-      !(path =~ /\.(zip|gz|gzip|xls.|)\z/i).nil?
+      !(path =~ /\.(zip|gz|gzip|xlsx|xlsm|bz2)\z/i).nil?
     end
 
     # Returns [true|false] whether the file is encrypted based on its file extensions.

data/lib/io_streams/paths/s3.rb

@@ -3,7 +3,10 @@ require "uri"
 module IOStreams
   module Paths
     class S3 < IOStreams::Path
-      attr_reader :bucket_name, :client
+      attr_reader :bucket_name, :client, :options
+
+      # Largest file size supported by the S3 copy object api.
+      S3_COPY_OBJECT_SIZE_LIMIT = 5 * 1024 * 1024 * 1024
 
       # Arguments:
       #
@@ -179,17 +182,36 @@ module IOStreams
       #
       # Notes:
       # - Can copy across buckets.
+      # - No stream conversions are applied.
       def move_to(target_path)
+        target = copy_to(target_path, convert: false)
+        delete
+        target
+      end
+
+      # Make S3 perform direct copies within S3 itself.
+      def copy_to(target_path, convert: true)
+        return super(target_path) if convert || (size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
+
         target = IOStreams.new(target_path)
         return super(target) unless target.is_a?(self.class)
 
         source_name = ::File.join(bucket_name, path)
-        # TODO: Does/should it also copy metadata?
-        client.copy_object(bucket: target.bucket_name, key: target.path, copy_source: source_name)
-        delete
+        client.copy_object(options.merge(bucket: target.bucket_name, key: target.path, copy_source: source_name))
         target
       end
 
+      # Make S3 perform direct copies within S3 itself.
+      def copy_from(source_path, convert: true)
+        return super(source_path) if convert
+
+        source = IOStreams.new(source_path)
+        return super(source) if !source.is_a?(self.class) || (source.size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
+
+        source_name = ::File.join(source.bucket_name, source.path)
+        client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))
+      end
+
       # S3 logically creates paths when a key is set.
       def mkpath
         self
@@ -220,7 +242,7 @@ module IOStreams
       # Shortcut method if caller has a filename already with no other streams applied:
       def read_file(file_name)
         ::File.open(file_name, "wb") do |file|
-          client.get_object(@options.merge(response_target: file, bucket: bucket_name, key: path))
+          client.get_object(options.merge(response_target: file, bucket: bucket_name, key: path))
         end
       end
 
@@ -248,10 +270,10 @@ module IOStreams
           # Use multipart file upload
           s3  = Aws::S3::Resource.new(client: client)
           obj = s3.bucket(bucket_name).object(path)
-          obj.upload_file(file_name)
+          obj.upload_file(file_name, options)
         else
           ::File.open(file_name, "rb") do |file|
-            client.put_object(@options.merge(bucket: bucket_name, key: path, body: file))
+            client.put_object(options.merge(bucket: bucket_name, key: path, body: file))
           end
         end
       end

data/lib/io_streams/paths/sftp.rb

@@ -71,7 +71,9 @@ module IOStreams
       #   end
       #
       #   # When using the sftp executable use an identity file instead of a password to authenticate:
-      #   IOStreams.path("sftp://test.com/path/file_name.csv", username: "jack", ssh_options: {IdentityFile: "~/.ssh/private_key"}).reader do |io|
+      #   IOStreams.path("sftp://test.com/path/file_name.csv",
+      #                  username:    "jack",
+      #                  ssh_options: {IdentityFile: "~/.ssh/private_key"}).reader do |io|
       #     puts io.read
       #   end
       def initialize(url, username: nil, password: nil, ssh_options: {})
@@ -122,7 +124,8 @@ module IOStreams
       #   end
       #
       # Example Output:
-      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group", :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
+      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
+      #   :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
       def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
         Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
 

data/lib/io_streams/pgp.rb

@@ -74,9 +74,10 @@ module IOStreams
 
       raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}") unless status.success?
 
-      if (match = err.match(/gpg: key ([0-9A-F]+)\s+/))
-        match[1]
-      end
+      match = err.match(/gpg: key ([0-9A-F]+)\s+/)
+      return unless match
+
+      match[1]
     end
 
     # Delete all private and public keys for a particular email.
@@ -110,11 +111,6 @@ module IOStreams
       !list_keys(email: email, key_id: key_id, private: private).empty?
     end
 
-    # Deprecated
-    def self.has_key?(**args)
-      key?(**args)
-    end
-
     # Returns [Array<Hash>] the list of keys.
     #   Each Hash consists of:
     #     key_length: [Integer]
@@ -232,7 +228,7 @@ module IOStreams
         err.each_line do |line|
           if line =~ /secret key imported/
             secret = true
-          elsif match = line.match(/key\s+(\w+):\s+(\w+).+\"(.*)<(.*)>\"/)
+          elsif (match = line.match(/key\s+(\w+):\s+(\w+).+\"(.*)<(.*)>\"/))
             results << {
               key_id:  match[1].to_s.strip,
               private: secret,
@@ -347,8 +343,6 @@ module IOStreams
       end
     end
 
-    private
-
     @logger = nil
 
     def self.logger

data/lib/io_streams/stream.rb

@@ -177,7 +177,7 @@ module IOStreams
     end
 
     def copy_to(target, convert: true)
-      target = IOStreams.path(target) unless target.is_a?(Stream)
+      target = IOStreams.new(target)
       target.copy_from(self, convert: convert)
     end
 
@@ -191,11 +191,41 @@ module IOStreams
       end
     end
 
-    # Set/get the original file_name
+    # Set the original file_name
     def file_name=(file_name)
       builder.file_name = file_name
     end
 
+    # Set/get the tabular format_options
+    def format(format = :none)
+      if format == :none
+        builder.format
+      else
+        builder.format = format
+        self
+      end
+    end
+
+    # Set the tabular format
+    def format=(format)
+      builder.format = format
+    end
+
+    # Set/get the tabular format options
+    def format_options(format_options = :none)
+      if format_options == :none
+        builder.format_options
+      else
+        builder.format_options = format_options
+        self
+      end
+    end
+
+    # Set the tabular format_options
+    def format_options=(format_options)
+      builder.format_options = format_options
+    end
+
     # Returns [String] the last component of this path.
     # Returns `nil` if no `file_name` was set.
     #
@@ -293,14 +323,26 @@ module IOStreams
     # Iterate over a file / stream returning each line as an array, one at a time.
     def row_reader(delimiter: nil, embedded_within: nil, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
-        yield IOStreams::Row::Reader.new(io, original_file_name: builder.file_name, **args)
+        yield IOStreams::Row::Reader.new(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args
+        )
       end
     end
 
     # Iterate over a file / stream returning each line as a hash, one at a time.
     def record_reader(delimiter: nil, embedded_within: nil, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
-        yield IOStreams::Record::Reader.new(io, original_file_name: builder.file_name, **args)
+        yield IOStreams::Record::Reader.new(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args
+        )
       end
     end
 
@@ -320,7 +362,14 @@ module IOStreams
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Row::Writer)
 
       line_writer(delimiter: delimiter) do |io|
-        IOStreams::Row::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+        IOStreams::Row::Writer.stream(
+          io,
+          original_file_name: builder.file_name,
+          format: builder.format,
+          format_options: builder.format_options,
+          **args,
+          &block
+        )
       end
     end
 
@@ -328,7 +377,13 @@ module IOStreams
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Record::Writer)
 
       line_writer(delimiter: delimiter) do |io|
-        IOStreams::Record::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+        IOStreams::Record::Writer.stream(
+          io,
+          original_file_name: builder.file_name,
+          format: builder.format,
+          format_options: builder.format_options,
+          **args,
+          &block)
       end
     end
   end