iostreams 0.16.2 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e9a14b746c83e98c98950f8fe1086e689383e997a2f62688272f419d0a144c36
-  data.tar.gz: 61c6da8d61da48d205f8537bd0a11b0b0cf4a1160ab4e00247f3cea507540072
+  metadata.gz: afb047a3682e7cbb9d953c303ebf2f525ff9975248ac550b6eded486ff73a72b
+  data.tar.gz: b14b13f5c23663b1b173d4ee767a93ea74b381f4c89ae5b44e4610bfbe4fabe4
 SHA512:
-  metadata.gz: 9736cb2bacdb162120a9bd6febe300760c34ea70148a62352244d33ab9142dead20cdc600242f8304ba460fcc9f4364dd2fdc8588ab4fda2dd34802e302037d8
-  data.tar.gz: 64a012432f793855f216be83b6522fe8301eff9f02fdc967782fb5488b85b6df630c44f6c4aefb4541b76b9a8f518856af2315b64ff733516c3b73b2451341f4
+  metadata.gz: 00a5fa08bf88bbcc9cc3c1418f90d2dd55c3da0b129cd472ef571e27f0dec653214f2f1780069801e8fd87c676c70e097fd79e86085fcbd63e3def501f779ec5
+  data.tar.gz: e193d7dacbd65625c9159fb61d13d3d0b005bd40e373047e2ece791c91c5c1798c881a398c0246fda8c424bd875dead9be3c5d3cc00765b0b2977d3decf3a08d

data/lib/io_streams/io_streams.rb

@@ -70,6 +70,15 @@ module IOStreams
   end
 
   # Iterate over a file / stream returning one line at a time.
+  # Embedded lines (within double quotes) will be skipped if
+  #   1. The file name contains .csv
+  #   2. Or the embedded_within argument is set
+  #
+  # Example: Supply custom options
+  #   IOStreams.each_line(file_name, embedded_within: '"') do |line|
+  #     puts line
+  #   end
+  #
   def self.each_line(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
     line_reader(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace, **args) do |line_stream|
       line_stream.each(&block)
@@ -77,6 +86,15 @@ module IOStreams
   end
 
   # Iterate over a file / stream returning one line at a time.
+  # Embedded lines (within double quotes) will be skipped if
+  #   1. The file name contains .csv
+  #   2. Or the embedded_within argument is set
+  #
+  # Example: Supply custom options
+  #   IOStreams.each_row(file_name, embedded_within: '"') do |line|
+  #     puts line
+  #   end
+  #
   def self.each_row(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
     row_reader(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace, **args) do |row_stream|
       row_stream.each(&block)
@@ -90,6 +108,15 @@ module IOStreams
   # Each record / line is returned one at a time so that very large files
   # can be read without having to load the entire file into memory.
   #
+  # Embedded lines (within double quotes) will be skipped if
+  #   1. The file name contains .csv
+  #   2. Or the embedded_within argument is set
+  #
+  # Example: Supply custom options
+  #   IOStreams.each_record(file_name, embedded_within: '"') do |line|
+  #     puts line
+  #   end
+  #
   # Example:
   #   file_name = 'customer_data.csv.pgp'
   #   IOStreams.each_record(file_name) do |hash|
@@ -291,20 +318,21 @@ module IOStreams
   # its extension(s)
   #
   # Example Zip file:
-  #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.zip')
+  #   IOStreams.streams_for_file_name('myfile.zip')
   #   => [ :zip ]
   #
   # Example Encrypted Gzip file:
-  #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv.gz.enc')
+  #   IOStreams.streams_for_file_name('myfile.csv.gz.enc')
   #   => [ :gz, :enc ]
   #
   # Example plain text / binary file:
-  #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv')
-  #   => [ :file ]
+  #   IOStreams.streams_for_file_name('myfile.csv')
+  #   => []
   def self.streams_for_file_name(file_name)
     raise ArgumentError.new('File name cannot be nil') if file_name.nil?
     raise ArgumentError.new("File name must be a string: #{file_name.inspect}, class: #{file_name.class}") unless file_name.is_a?(String)
-    parts      = file_name.split('.')
+
+    parts      = ::File.basename(file_name).split('.')
     extensions = []
     while extension = parts.pop
       sym = extension.downcase.to_sym
@@ -314,12 +342,31 @@ module IOStreams
     extensions
   end
 
+  # Extract URI if any was supplied
+  def self.scheme_for_file_name(file_name)
+    raise ArgumentError.new('File name cannot be nil') if file_name.nil?
+    raise ArgumentError.new("File name must be a string: #{file_name.inspect}, class: #{file_name.class}") unless file_name.is_a?(String)
+
+    if matches = file_name.match(/\A(\w+):\/\//)
+      matches[1].downcase.to_sym
+    end
+  end
+
   # Iterate over a file / stream returning each record/line one at a time.
-  def self.line_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+  # It will apply the embedded_within argument if the file or input_stream contain .csv in its name.
+  def self.line_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, embedded_within: nil, **args, &block)
+
     return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Line::Reader) || file_name_or_io.is_a?(Array)
 
+    # TODO: needs to be improved
+    if embedded_within.nil? && file_name_or_io.is_a?(String)
+      embedded_within = '"' if file_name_or_io.include?('.csv')
+    elsif embedded_within.nil? && file_name
+      embedded_within = '"' if file_name.include?('.csv')
+    end
+
     reader(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace) do |io|
-      IOStreams::Line::Reader.open(io, **args, &block)
+      IOStreams::Line::Reader.open(io, embedded_within: embedded_within, **args, &block)
     end
   end
 
@@ -331,6 +378,7 @@ module IOStreams
     encoding: nil,
     encode_cleaner: nil,
     encode_replace: nil,
+    embedded_within: nil,
     **args,
     &block)
 
@@ -338,12 +386,13 @@ module IOStreams
 
     line_reader(
       file_name_or_io,
-      streams:        streams,
-      delimiter:      delimiter,
-      file_name:      file_name,
-      encoding:       encoding,
-      encode_cleaner: encode_cleaner,
-      encode_replace: encode_replace
+      streams:         streams,
+      delimiter:       delimiter,
+      file_name:       file_name,
+      encoding:        encoding,
+      encode_cleaner:  encode_cleaner,
+      encode_replace:  encode_replace,
+      embedded_within: embedded_within
     ) do |io|
       file_name = file_name_or_io if file_name.nil? && file_name_or_io.is_a?(String)
       IOStreams::Row::Reader.open(io, file_name: file_name, **args, &block)
@@ -358,21 +407,23 @@ module IOStreams
     encoding: nil,
     encode_cleaner: nil,
     encode_replace: nil,
+    embedded_within: nil,
     **args,
     &block)
 
     return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Record::Reader)
 
-    line_reader(
-      file_name_or_io,
-      streams:        streams,
-      delimiter:      delimiter,
-      file_name:      file_name,
-      encoding:       encoding,
-      encode_cleaner: encode_cleaner,
-      encode_replace: encode_replace
+    line_reader(file_name_or_io,
+                streams:         streams,
+                delimiter:       delimiter,
+                file_name:       file_name,
+                encoding:        encoding,
+                encode_cleaner:  encode_cleaner,
+                encode_replace:  encode_replace,
+                embedded_within: embedded_within
     ) do |io|
 
+
       file_name = file_name_or_io if file_name.nil? && file_name_or_io.is_a?(String)
       IOStreams::Record::Reader.open(io, file_name: file_name, **args, &block)
     end
@@ -401,6 +452,16 @@ module IOStreams
     @extensions.delete(extension.to_sym)
   end
 
+  # Register a file extension and the reader and writer streaming classes
+  #
+  # Example:
+  #   # MyXls::Reader and MyXls::Writer must implement .open
+  #   register_extension(:xls, MyXls::Reader, MyXls::Writer)
+  def self.register_scheme(scheme, reader_class, writer_class)
+    raise(ArgumentError, "Invalid scheme #{scheme.inspect}") unless scheme.nil? || scheme.to_s =~ /\A\w+\Z/
+    @schemes[scheme.nil? ? nil : scheme.to_sym] = Extension.new(reader_class, writer_class)
+  end
+
   # Helper method: Returns [true|false] if a value is blank?
   def self.blank?(value)
     if value.nil?
@@ -416,6 +477,7 @@ module IOStreams
 
   # A registry to hold formats for processing files during upload or download
   @extensions = {}
+  @schemes    = {}
 
   # Struct to hold the Stream and options if any
   StreamStruct = Struct.new(:klass, :options)
@@ -438,8 +500,10 @@ module IOStreams
     if streams.nil?
       streams = file_name_or_io.is_a?(String) ? streams_for_file_name(file_name_or_io) : [nil]
     end
+    scheme = scheme_for_file_name(file_name_or_io) if file_name_or_io.is_a?(String)
 
     stream_structs = streams_for(type, streams)
+    stream_structs << stream_struct_for_scheme(type, scheme) if stream_structs.empty? || scheme
 
     # Add encoding stream if any of its options are present
     if encoding || encode_cleaner || encode_replace
@@ -466,7 +530,6 @@ module IOStreams
     if params.is_a?(Symbol)
       [stream_struct_for_stream(type, params)]
     elsif params.is_a?(Array)
-      return [stream_struct_for_stream(type, nil)] if params.empty?
       a = []
       params.each do |stream|
         if stream.is_a?(Hash)
@@ -491,8 +554,14 @@ module IOStreams
     StreamStruct.new(klass, options)
   end
 
+  def self.stream_struct_for_scheme(type, scheme, options = {})
+    ext   = @schemes[scheme.nil? ? nil : scheme.to_sym] || raise(ArgumentError, "Unknown Scheme type: #{scheme.inspect}")
+    klass = ext.send("#{type}_class")
+    StreamStruct.new(klass, options)
+  end
+
   # Default reader/writer when no other streams need to be applied.
-  register_extension(nil, IOStreams::File::Reader, IOStreams::File::Writer)
+  # register_extension(nil, IOStreams::File::Reader, IOStreams::File::Writer)
 
   # Register File extensions
   register_extension(:bz2, IOStreams::Bzip2::Reader, IOStreams::Bzip2::Writer)
@@ -510,10 +579,17 @@ module IOStreams
     register_extension(:enc, SymmetricEncryption::Reader, SymmetricEncryption::Writer)
   end
 
-  # register_scheme(nil,    IOStreams::File::Reader,  IOStreams::File::Writer)
-  # register_scheme(:file,  IOStreams::File::Reader,  IOStreams::File::Writer)
+  # Support URI schemes
+  #
+  # Examples:
+  #    path/file_name
+  #    http://hostname/path/file_name
+  #    https://hostname/path/file_name
+  #    sftp://hostname/path/file_name
+  #    s3://bucket/key
+  register_scheme(nil, IOStreams::File::Reader, IOStreams::File::Writer)
   # register_scheme(:http,  IOStreams::HTTP::Reader,  IOStreams::HTTP::Writer)
   # register_scheme(:https, IOStreams::HTTPS::Reader, IOStreams::HTTPS::Writer)
   # register_scheme(:sftp,  IOStreams::SFTP::Reader,  IOStreams::SFTP::Writer)
-  # register_scheme(:s3,    IOStreams::S3::Reader,    IOStreams::S3::Writer)
+  register_scheme(:s3, IOStreams::S3::Reader, IOStreams::S3::Writer)
 end

data/lib/io_streams/line/reader.rb

@@ -1,7 +1,7 @@
 module IOStreams
   module Line
     class Reader
-      attr_reader :delimiter, :buffer_size, :line_count
+      attr_reader :delimiter, :buffer_size, :line_number
 
       # Prevent denial of service when a delimiter is not found before this number * `buffer_size` characters are read.
       MAX_BLOCKS_MULTIPLIER = 100
@@ -20,7 +20,7 @@ module IOStreams
       # Create a delimited stream reader from the supplied input stream.
       #
       # Lines returned will be in the encoding of the input stream.
-      # To change the encoding of retruned lines, use IOStreams::Encode::Reader.
+      # To change the encoding of returned lines, use IOStreams::Encode::Reader.
       #
       # Parameters
       #   input_stream
@@ -45,14 +45,15 @@ module IOStreams
       # - Skip "empty" / "blank" lines. RegExp?
       # - Extract header line(s) / first non-comment, non-blank line
       # - Embedded newline support, RegExp? or Proc?
-      def initialize(input_stream, delimiter: nil, buffer_size: 65_536)
-        @input_stream = input_stream
-        @buffer_size  = buffer_size
+      def initialize(input_stream, delimiter: nil, buffer_size: 65_536, embedded_within: nil)
+        @embedded_within = embedded_within
+        @input_stream    = input_stream
+        @buffer_size     = buffer_size
 
         # More efficient read buffering only supported when the input stream `#read` method supports it.
         @use_read_cache_buffer = !@input_stream.method(:read).arity.between?(0, 1)
 
-        @line_count        = 0
+        @line_number       = 0
         @eof               = false
         @read_cache_buffer = nil
         @buffer            = nil
@@ -73,14 +74,40 @@ module IOStreams
       # Note:
       # * The line delimiter is _not_ returned.
       def each
+        line_count = 0
         until eof?
           line = readline
-          yield(line) unless line.nil?
+          unless line.nil?
+            yield(line)
+            line_count += 1
+          end
         end
         line_count
       end
 
+      # Reads each line per the @delimeter. It will account for embedded lines provided they are within double quotes.
+      # The embedded_within argument is set in IOStreams::LineReader
       def readline
+        line = _readline
+        if line && @embedded_within
+          initial_line_number = @line_number
+          while line.count(@embedded_within).odd?
+            raise "Unclosed quoted field on line #{initial_line_number}" if eof? || line.length > @buffer_size * 10
+            line << @delimiter
+            line << _readline
+          end
+        end
+        line
+      end
+
+      # Returns whether the end of file has been reached for this stream
+      def eof?
+        @eof && (@buffer.nil? || @buffer.empty?)
+      end
+
+      private
+
+      def _readline
         return if eof?
 
         # Keep reading until it finds the delimiter
@@ -89,29 +116,22 @@ module IOStreams
 
         # Delimiter found?
         if index
-          data    = @buffer.slice(0, index)
-          @buffer = @buffer.slice(index + @delimiter_size, @buffer.size)
-          @line_count += 1
+          data         = @buffer.slice(0, index)
+          @buffer      = @buffer.slice(index + @delimiter_size, @buffer.size)
+          @line_number += 1
         elsif @eof && @buffer.empty?
           data    = nil
           @buffer = nil
         else
           # Last line without delimiter
-          data    = @buffer
-          @buffer = nil
-          @line_count += 1
+          data         = @buffer
+          @buffer      = nil
+          @line_number += 1
         end
 
         data
       end
 
-      # Returns whether the end of file has been reached for this stream
-      def eof?
-        @eof && (@buffer.nil? || @buffer.empty?)
-      end
-
-      private
-
       # Returns [Integer] the number of characters read into the internal buffer
       # Returns 0 on EOF
       def read_block

data/lib/io_streams/record/reader.rb

@@ -32,14 +32,14 @@ module IOStreams
       #     :csv, :hash, :array, :json, :psv, :fixed
       #
       #   For all other parameters, see Tabular::Header.new
-      def initialize(delimited, cleanse_header: true, **args)
+      def initialize(line_reader, cleanse_header: true, **args)
         @tabular        = IOStreams::Tabular.new(**args)
-        @delimited      = delimited
+        @line_reader      = line_reader
         @cleanse_header = cleanse_header
       end
 
       def each
-        @delimited.each do |line|
+        @line_reader.each do |line|
           if @tabular.header?
             @tabular.parse_header(line)
             @tabular.cleanse_header! if @cleanse_header

data/lib/io_streams/s3.rb

@@ -7,19 +7,17 @@ end
 require 'uri'
 module IOStreams
   module S3
+    autoload :Reader, 'io_streams/s3/reader'
+    autoload :Writer, 'io_streams/s3/writer'
+
     # Sample URI: s3://mybucket/user/abc.zip
     def self.parse_uri(uri)
-      # 's3://mybucket/user/abc.zip'
       uri = URI.parse(uri)
-      # Filename and bucket only
-      if uri.scheme.nil?
-        segments = uri.path.split('/')
-        raise "S3 URI must at the very least contain '<bucket_name>/<key>'" if (segments.size == 1) || (segments[0] == '')
-        {
-          bucket: segments.shift,
-          key:    segments.join('/')
-        }
-      end
+      raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == 's3'
+      {
+        bucket: uri.host,
+        key: uri.path.sub(/\A\//, '')
+      }
     end
   end
 end

data/lib/io_streams/s3/reader.rb

@@ -2,63 +2,25 @@ module IOStreams
   module S3
     class Reader
       # Read from a AWS S3 file
-      def self.open(uri = nil, bucket: nil, region: nil, key: nil, &block)
-        options = uri.nil? ? args : parse_uri(uri).merge(args)
+      def self.open(uri, region: nil, **args, &block)
+        raise(ArgumentError, 'file_name must be a URI string') unless uri.is_a?(String)
+
         s3      = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
+        options = IOStreams::S3.parse_uri(uri)
         object  = s3.bucket(options[:bucket]).object(options[:key])
 
-        IO.pipe do |read_io, write_io|
-          object.get(response_target: write_io)
-          write_io.close
-          block.call(read_io)
-        end
-      end
-
-      def self.open2(uri = nil, **args, &block)
-        if !uri.nil? && IOStreams.reader_stream?(uri)
-          raise(ArgumentError, 'S3 can only accept a URI, not an IO stream when reading.')
-        end
-
-        unless defined?(Aws::S3::Resource)
-          begin
-            require 'aws-sdk-s3'
-          rescue LoadError => exc
-            raise(LoadError, "Install gem 'aws-sdk-s3' to read and write AWS S3 files: #{exc.message}")
-          end
-        end
-
-        options = uri.nil? ? args : parse_uri(uri).merge(args)
-
         begin
-          io = new(**options)
-          block.call(io)
-        ensure
-          io.close if io && (io.respond_to?(:closed?) && !io.closed?)
-        end
-      end
-
-      def initialize(region: nil, bucket:, key:)
-        s3      = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
-        @object = s3.bucket(bucket).object(key)
-        @buffer = []
-      end
+          # Since S3 download only supports a push stream, write it to a tempfile first.
+          temp_file = Tempfile.new('rocket_job')
 
-      def read(length = nil, outbuf = nil)
-        # Sufficient data already in the buffer
-        return @buffer.slice!(0, length) if length && (length <= @buffer.length)
+          args[:response_target] = temp_file.to_path
+          object.get(args)
 
-        # Fetch in chunks
-        @object.get do |chunk|
-          @buffer << chunk
-          return @buffer.slice!(0, length) if length && (length <= @buffer.length)
+          block.call(temp_file)
+        ensure
+          temp_file.delete if temp_file
         end
-        @buffer if @buffer.size > 0
       end
-
-      private
-
-      attr_reader :object
-
     end
   end
 end

data/lib/io_streams/s3/writer.rb

@@ -2,11 +2,13 @@ module IOStreams
   module S3
     class Writer
       # Write to AWS S3
-      def self.open(uri = nil, bucket: nil, region: nil, key: nil, &block)
-        options = uri.nil? ? args : parse_uri(uri).merge(args)
+      def self.open(uri, region: nil, **args, &block)
+        raise(ArgumentError, 'file_name must be a URI string') unless uri.is_a?(String)
+
+        options = IOStreams::S3.parse_uri(uri)
         s3      = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
         object  = s3.bucket(options[:bucket]).object(options[:key])
-        object.upload_stream(file_name_or_io, &block)
+        object.upload_stream(args, &block)
       end
     end
   end

data/lib/io_streams/streams.rb

@@ -81,15 +81,15 @@ module IOStreams
     #   .gz.enc   [ :gz,  :enc ]
     #
     # Example Zip file:
-    #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.zip')
+    #   IOStreams.streams_for_file_name('myfile.zip')
     #   => [ :zip ]
     #
     # Example Encrypted Gzip file:
-    #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv.gz.enc')
+    #   IOStreams.streams_for_file_name('myfile.csv.gz.enc')
     #   => [ :gz, :enc ]
     #
     # Example plain text / binary file:
-    #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv')
+    #   IOStreams.streams_for_file_name('myfile.csv')
     #   => [ :file ]
     def streams_for_file_name(file_name)
       raise ArgumentError.new("Cannot auto-detect streams when already a stream: #{file_name.inspect}") if reader_stream?(file_name)

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = '0.16.2'
+  VERSION = '0.17.0'
 end

data/lib/iostreams.rb

@@ -16,10 +16,7 @@ module IOStreams
     autoload :Writer, 'io_streams/gzip/writer'
   end
   autoload :Pgp,      'io_streams/pgp'
-  module S3
-    autoload :Reader, 'io_streams/s3/reader'
-    autoload :Writer, 'io_streams/s3/writer'
-  end
+  autoload :S3,       'io_streams/s3'
   module SFTP
     autoload :Reader, 'io_streams/sftp/reader'
     autoload :Writer, 'io_streams/sftp/writer'

data/test/files/embedded_lines_test.csv

@@ -0,0 +1,7 @@
+name, description, zip
+"
+Jack","Firstname is Jack","234567"
+"John","Firstname
+is John","234568"
+"Zack","Firstname is Zack","234568
+"

data/test/files/unclosed_quote_test.csv

@@ -0,0 +1,4 @@
+name, description, zip
+"Jack","Firstn"ame is Jack","234567"
+"John","Firstname is John","234568"
+"Zack","Firstname is Zack","234568"

data/test/io_streams_test.rb

@@ -95,6 +95,16 @@ class IOStreamsTest < Minitest::Test
       end
     end
 
+    describe '.scheme_for_file_name' do
+      it 'default' do
+        assert_nil IOStreams.scheme_for_file_name('a.xyz')
+      end
+
+      it 's3' do
+        assert_equal :s3, IOStreams.scheme_for_file_name('s3://a.xyz')
+      end
+    end
+
     describe '.each_line' do
       it 'returns a line at a time' do
         lines = []

data/test/line_reader_test.rb

@@ -6,6 +6,14 @@ class LineReaderTest < Minitest::Test
       File.join(File.dirname(__FILE__), 'files', 'text.txt')
     end
 
+    let :csv_file do
+      File.join(File.dirname(__FILE__), 'files', 'embedded_lines_test.csv')
+    end
+
+    let :unclosed_quote_file do
+      File.join(File.dirname(__FILE__), 'files', 'unclosed_quote_test.csv')
+    end
+
     let :data do
       data = []
       File.open(file_name, 'rt') do |file|
@@ -16,6 +24,47 @@ class LineReaderTest < Minitest::Test
       data
     end
 
+    # Test file has embedded new lines in row 2, 3 and 4
+    #
+    #  name, description, zip
+    # "\nJack","Firstname is Jack","234567"
+    # "John","Firstname\n is John","234568"
+    # "Zack","Firstname is Zack","234568\n"
+    #
+    describe 'embedded_within_quotes' do
+      describe 'csv file' do
+
+        it 'fails to keep embedded lines if flag is not set' do
+          lines = []
+          IOStreams::Line::Reader.open(csv_file) do |io|
+            io.each do |line|
+              lines << line
+            end
+          end
+          assert_equal 7, lines.count
+        end
+
+        it 'keeps embedded lines if flag is set' do
+          lines = []
+          IOStreams::Line::Reader.open(csv_file, embedded_within: '"') do |io|
+            io.each do |line|
+              lines << line
+            end
+          end
+          assert_equal 4, lines.count
+        end
+
+        it 'raises error for unclosed quote' do
+          assert_raises(RuntimeError) do
+            IOStreams::Line::Reader.open(unclosed_quote_file, embedded_within: '"') do |io|
+              io.each do |line|
+              end
+            end
+          end
+        end
+      end
+    end
+
     describe '#each' do
       it 'each_line file' do
         lines = []
@@ -41,7 +90,7 @@ class LineReaderTest < Minitest::Test
         it "autodetect delimiter: #{delimiter.inspect}" do
           lines  = []
           stream = StringIO.new(data.join(delimiter))
-          count = IOStreams::Line::Reader.open(stream, buffer_size: 15) do |io|
+          count  = IOStreams::Line::Reader.open(stream, buffer_size: 15) do |io|
             io.each { |line| lines << line }
           end
           assert_equal data, lines
@@ -51,7 +100,7 @@ class LineReaderTest < Minitest::Test
         it "single read autodetect delimiter: #{delimiter.inspect}" do
           lines  = []
           stream = StringIO.new(data.join(delimiter))
-          count = IOStreams::Line::Reader.open(stream) do |io|
+          count  = IOStreams::Line::Reader.open(stream) do |io|
             io.each { |line| lines << line }
           end
           assert_equal data, lines
@@ -63,7 +112,7 @@ class LineReaderTest < Minitest::Test
         it "reads delimited #{delimiter.inspect}" do
           lines  = []
           stream = StringIO.new(data.join(delimiter))
-          count = IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
+          count  = IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
             io.each { |line| lines << line }
           end
           assert_equal data, lines
@@ -75,7 +124,7 @@ class LineReaderTest < Minitest::Test
         delimiter = "\x01"
         lines     = []
         stream    = StringIO.new(data.join(delimiter).encode('ASCII-8BIT'))
-        count = IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
+        count     = IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
           io.each { |line| lines << line }
         end
         assert_equal data, lines

data/test/s3_reader_test.rb

@@ -0,0 +1,41 @@
+require_relative 'test_helper'
+
+class S3ReaderTest < Minitest::Test
+  describe IOStreams::File::Reader do
+    let :file_name do
+      File.join(File.dirname(__FILE__), 'files', 'text.txt')
+    end
+
+    let :raw do
+      File.read(file_name)
+    end
+
+    let :uri do
+      "s3://#{ENV['S3_BUCKET_NAME']}/s3_test/test.txt"
+    end
+
+    let :upload_s3_file do
+      IOStreams::S3::Writer.open(uri) { |io| io << raw }
+    end
+
+    describe '.open' do
+      it 'reads' do
+        unless ENV['S3_BUCKET_NAME']
+          skip "Supply 'S3_BUCKET_NAME' environment variable with S3 bucket name to test with"
+        end
+
+        upload_s3_file
+        result = IOStreams::S3::Reader.open(uri) { |io| io.read }
+        assert_equal raw, result
+      end
+
+      it 'does not support streams' do
+        io_string = StringIO.new('data')
+        assert_raises ArgumentError do
+          IOStreams::S3::Reader.open(io_string) { |io| io.read }
+        end
+      end
+    end
+
+  end
+end

data/test/s3_writer_test.rb

@@ -0,0 +1,41 @@
+require_relative 'test_helper'
+
+class FileWriterTest < Minitest::Test
+  describe IOStreams::File::Writer do
+    let :file_name do
+      File.join(File.dirname(__FILE__), 'files', 'text.txt')
+    end
+
+    let :raw do
+      File.read(file_name)
+    end
+
+    let :uri do
+      "s3://#{ENV['S3_BUCKET_NAME']}/s3_test/test.txt"
+    end
+
+    let :upload_s3_file do
+      IOStreams::S3::Writer.open(uri) { |io| io << raw }
+    end
+
+    describe '.open' do
+      it 'writes' do
+        unless ENV['S3_BUCKET_NAME']
+          skip "Supply 'S3_BUCKET_NAME' environment variable with S3 bucket name to test with"
+        end
+
+        IOStreams::S3::Writer.open(uri) { |io| io.write(raw) }
+        result = IOStreams::S3::Reader.open(uri) { |io| io.read }
+        assert_equal raw, result
+      end
+
+      it 'does not support streams' do
+        io_string = StringIO.new
+        assert_raises ArgumentError do
+          IOStreams::S3::Writer.open(io_string) { |io|  io.write(raw) }
+        end
+      end
+    end
+
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 0.16.2
+  version: 0.17.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-11 00:00:00.000000000 Z
+date: 2019-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -80,6 +80,7 @@ files:
 - test/encode_writer_test.rb
 - test/file_reader_test.rb
 - test/file_writer_test.rb
+- test/files/embedded_lines_test.csv
 - test/files/spreadsheet.xlsx
 - test/files/test.csv
 - test/files/test.json
@@ -88,6 +89,7 @@ files:
 - test/files/text.txt.gz
 - test/files/text.txt.gz.zip
 - test/files/text.zip
+- test/files/unclosed_quote_test.csv
 - test/gzip_reader_test.rb
 - test/gzip_writer_test.rb
 - test/io_streams_test.rb
@@ -100,6 +102,8 @@ files:
 - test/record_writer_test.rb
 - test/row_reader_test.rb
 - test/row_writer_test.rb
+- test/s3_reader_test.rb
+- test/s3_writer_test.rb
 - test/tabular_test.rb
 - test/test_helper.rb
 - test/xlsx_reader_test.rb
@@ -139,6 +143,7 @@ test_files:
 - test/gzip_writer_test.rb
 - test/file_reader_test.rb
 - test/record_reader_test.rb
+- test/s3_writer_test.rb
 - test/pgp_writer_test.rb
 - test/line_writer_test.rb
 - test/row_reader_test.rb
@@ -146,8 +151,10 @@ test_files:
 - test/zip_writer_test.rb
 - test/files/text.zip
 - test/files/spreadsheet.xlsx
+- test/files/embedded_lines_test.csv
 - test/files/test.csv
 - test/files/test.json
+- test/files/unclosed_quote_test.csv
 - test/files/text.txt.bz2
 - test/files/text.txt.gz.zip
 - test/files/text.txt.gz
@@ -160,3 +167,4 @@ test_files:
 - test/pgp_test.rb
 - test/io_streams_test.rb
 - test/record_writer_test.rb
+- test/s3_reader_test.rb