iostreams 0.20.3 → 1.0.0.beta

data/lib/iostreams.rb

@@ -1,12 +1,23 @@
 require 'io_streams/version'
-#@formatter:off
+# @formatter:off
 module IOStreams
-  autoload :BasePath, 'io_streams/base_path'
   autoload :Errors,   'io_streams/errors'
+  autoload :Path,     'io_streams/path'
   autoload :Pgp,      'io_streams/pgp'
-  autoload :S3,       'io_streams/s3'
+  autoload :Reader,   'io_streams/reader'
+  autoload :Stream,   'io_streams/stream'
+  autoload :Streams,  'io_streams/streams'
   autoload :Tabular,  'io_streams/tabular'
   autoload :Utils,    'io_streams/utils'
+  autoload :Writer,   'io_streams/writer'
+
+  module Paths
+    autoload :File,    'io_streams/paths/file'
+    autoload :HTTP,    'io_streams/paths/http'
+    autoload :Matcher, 'io_streams/paths/matcher'
+    autoload :S3,      'io_streams/paths/s3'
+    autoload :SFTP,    'io_streams/paths/sftp'
+  end
 
   module Bzip2
     autoload :Reader, 'io_streams/bzip2/reader'
@@ -16,18 +27,10 @@ module IOStreams
     autoload :Reader, 'io_streams/encode/reader'
     autoload :Writer, 'io_streams/encode/writer'
   end
-  module File
-    autoload :Path,   'io_streams/file/path'
-    autoload :Reader, 'io_streams/file/reader'
-    autoload :Writer, 'io_streams/file/writer'
-  end
   module Gzip
     autoload :Reader, 'io_streams/gzip/reader'
     autoload :Writer, 'io_streams/gzip/writer'
   end
-  module HTTP
-    autoload :Reader, 'io_streams/http/reader'
-  end
   module Line
     autoload :Reader, 'io_streams/line/reader'
     autoload :Writer, 'io_streams/line/writer'
@@ -40,10 +43,6 @@ module IOStreams
     autoload :Reader, 'io_streams/row/reader'
     autoload :Writer, 'io_streams/row/writer'
   end
-  module SFTP
-    autoload :Reader, 'io_streams/sftp/reader'
-    autoload :Writer, 'io_streams/sftp/writer'
-  end
   module SymmetricEncryption
     autoload :Reader, 'io_streams/symmetric_encryption/reader'
     autoload :Writer, 'io_streams/symmetric_encryption/writer'
@@ -56,4 +55,5 @@ module IOStreams
     autoload :Writer, 'io_streams/zip/writer'
   end
 end
+require 'io_streams/deprecated'
 require 'io_streams/io_streams'

data/test/bzip2_reader_test.rb

@@ -10,9 +10,9 @@ class Bzip2ReaderTest < Minitest::Test
       File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
     end
 
-    describe '.open' do
+    describe '.file' do
       it 'file' do
-        result = IOStreams::Bzip2::Reader.open(file_name) do |io|
+        result = IOStreams::Bzip2::Reader.file(file_name) do |io|
           io.read
         end
         assert_equal decompressed, result
@@ -20,7 +20,7 @@ class Bzip2ReaderTest < Minitest::Test
 
       it 'stream' do
         result = File.open(file_name) do |file|
-          IOStreams::Bzip2::Reader.open(file) do |io|
+          IOStreams::Bzip2::Reader.stream(file) do |io|
             io.read
           end
         end

data/test/bzip2_writer_test.rb

@@ -18,9 +18,9 @@ class Bzip2WriterTest < Minitest::Test
       temp_file.delete
     end
 
-    describe '.open' do
+    describe '.file' do
       it 'file' do
-        IOStreams::Bzip2::Writer.open(file_name) do |io|
+        IOStreams::Bzip2::Writer.file(file_name) do |io|
           io.write(decompressed)
         end
 
@@ -34,7 +34,7 @@ class Bzip2WriterTest < Minitest::Test
 
       it 'stream' do
         io_string = StringIO.new(''.b)
-        IOStreams::Bzip2::Writer.open(io_string) do |io|
+        IOStreams::Bzip2::Writer.stream(io_string) do |io|
           io.write(decompressed)
         end
 

data/test/deprecated_test.rb

@@ -0,0 +1,123 @@
+require_relative 'test_helper'
+
+# Test deprecated api
+class DeprecatedTest < Minitest::Test
+  describe IOStreams do
+    let :source_file_name do
+      File.join(__dir__, 'files', 'text.txt')
+    end
+
+    let :data do
+      File.read(source_file_name)
+    end
+
+    let :temp_file do
+      Tempfile.new('iostreams')
+    end
+
+    let :target_file_name do
+      temp_file.path
+    end
+
+    let :bad_data do
+      [
+        "New M\xE9xico,NE".b,
+        'good line',
+        "New M\xE9xico,\x07SF".b
+      ].join("\n").encode('BINARY')
+    end
+
+    let :stripped_data do
+      bad_data.gsub("\xE9".b, '').gsub("\x07", '')
+    end
+
+    let :multiple_zip_file_name do
+      File.join(File.dirname(__FILE__), 'files', 'multiple_files.zip')
+    end
+
+    let :zip_gz_file_name do
+      File.join(File.dirname(__FILE__), 'files', 'text.zip.gz')
+    end
+
+    let :contents_test_txt do
+      File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
+    end
+
+    let :contents_test_json do
+      File.read(File.join(File.dirname(__FILE__), 'files', 'test.json'))
+    end
+
+    after do
+      temp_file.delete
+    end
+
+    describe '.copy' do
+      it 'streams' do
+        size   = IOStreams.reader(source_file_name) do |source_stream|
+          IOStreams.writer(target_file_name) do |target_stream|
+            IOStreams.copy(source_stream, target_stream)
+          end
+        end
+        actual = File.read(target_file_name)
+
+        assert_equal actual, data
+        assert_equal actual.size, size
+      end
+
+      it 'IO stream' do
+        size   = File.open(source_file_name) do |source_stream|
+          IOStreams.writer(target_file_name) do |target_stream|
+            IOStreams.copy(source_stream, target_stream)
+          end
+        end
+        actual = File.read(target_file_name)
+
+        assert_equal actual, data
+        assert_equal actual.size, size
+      end
+
+      it 'files' do
+        size   = IOStreams.copy(source_file_name, target_file_name)
+        actual = File.read(target_file_name)
+
+        assert_equal actual, data
+        assert_equal actual.size, size
+      end
+    end
+
+    describe '.each_line' do
+      it 'returns a line at a time' do
+        lines = []
+        count = IOStreams.each_line(source_file_name) { |line| lines << line }
+        assert_equal data.lines.map(&:strip), lines
+        assert_equal data.lines.count, count
+      end
+
+      it 'strips non-printable characters' do
+        input = StringIO.new(bad_data)
+        lines = []
+        count = IOStreams.each_line(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '') do |line|
+          lines << line
+        end
+        assert_equal stripped_data.lines.map(&:strip), lines
+        assert_equal stripped_data.lines.count, count
+      end
+    end
+
+    describe '.reader' do
+      it 'reads a zip file' do
+        result = IOStreams.reader(multiple_zip_file_name, streams: {zip: {entry_file_name: 'test.json'}}) do |io|
+          io.read
+        end
+        assert_equal contents_test_json, result
+      end
+
+      it 'reads a zip file from within a gz file' do
+        result = IOStreams.reader(zip_gz_file_name) do |io|
+          io.read
+        end
+        assert_equal contents_test_txt, result
+      end
+    end
+  end
+end

data/test/encode_reader_test.rb

@@ -22,7 +22,7 @@ class EncodeReaderTest < Minitest::Test
       describe 'replacement' do
         it 'does not strip invalid characters' do
           input = StringIO.new(bad_data)
-          IOStreams::Encode::Reader.open(input, encoding: 'UTF-8') do |io|
+          IOStreams::Encode::Reader.stream(input, encoding: 'UTF-8') do |io|
             assert_raises ::Encoding::UndefinedConversionError do
               io.read.encoding
             end
@@ -32,7 +32,7 @@ class EncodeReaderTest < Minitest::Test
         it 'strips invalid characters' do
           input = StringIO.new(bad_data)
           data  =
-            IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_replace: '') do |io|
+            IOStreams::Encode::Reader.stream(input, encoding: 'UTF-8', replace: '') do |io|
               io.read
             end
           assert_equal cleansed_data, data
@@ -43,7 +43,7 @@ class EncodeReaderTest < Minitest::Test
         it 'strips non-printable characters' do
           input = StringIO.new(bad_data)
           data  =
-            IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '') do |io|
+            IOStreams::Encode::Reader.stream(input, encoding: 'UTF-8', cleaner: :printable, replace: '') do |io|
               io.read
             end
           assert_equal stripped_data, data

data/test/encode_writer_test.rb

@@ -22,7 +22,7 @@ class EncodeWriterTest < Minitest::Test
       it 'file' do
         temp_file = Tempfile.new('rocket_job')
         file_name = temp_file.to_path
-        IOStreams::Encode::Writer.open(file_name, encoding: 'ASCII-8BIT') do |io|
+        IOStreams::Encode::Writer.file(file_name, encoding: 'ASCII-8BIT') do |io|
           io << bad_data
         end
         result = File.read(file_name, mode: 'rb')
@@ -31,7 +31,7 @@ class EncodeWriterTest < Minitest::Test
 
       it 'stream' do
         io = StringIO.new(''.b)
-        IOStreams::Encode::Writer.open(io, encoding: 'ASCII-8BIT') do |encoded|
+        IOStreams::Encode::Writer.stream(io, encoding: 'ASCII-8BIT') do |encoded|
           encoded << bad_data
         end
         assert_equal 'ASCII-8BIT', io.string.encoding.to_s
@@ -41,7 +41,7 @@ class EncodeWriterTest < Minitest::Test
       it 'stream as utf-8' do
         io = StringIO.new('')
         assert_raises Encoding::UndefinedConversionError do
-          IOStreams::Encode::Writer.open(io, encoding: 'UTF-8') do |encoded|
+          IOStreams::Encode::Writer.stream(io, encoding: 'UTF-8') do |encoded|
             encoded << bad_data
           end
         end
@@ -49,7 +49,7 @@ class EncodeWriterTest < Minitest::Test
 
       it 'stream as utf-8 with replacement' do
         io = StringIO.new('')
-        IOStreams::Encode::Writer.open(io, encoding: 'UTF-8', encode_replace: '?') do |encoded|
+        IOStreams::Encode::Writer.stream(io, encoding: 'UTF-8', replace: '?') do |encoded|
           encoded << bad_data
         end
         assert_equal 'UTF-8', io.string.encoding.to_s
@@ -58,7 +58,7 @@ class EncodeWriterTest < Minitest::Test
 
       it 'stream as utf-8 with replacement and printable cleansing' do
         io = StringIO.new('')
-        IOStreams::Encode::Writer.open(io, encoding: 'UTF-8', encode_replace: '?', encode_cleaner: :printable) do |encoded|
+        IOStreams::Encode::Writer.stream(io, encoding: 'UTF-8', replace: '?', cleaner: :printable) do |encoded|
           encoded << bad_data
         end
         assert_equal 'UTF-8', io.string.encoding.to_s
@@ -70,7 +70,7 @@ class EncodeWriterTest < Minitest::Test
       it 'returns byte count' do
         io_string = StringIO.new(''.b)
         count     = 0
-        IOStreams::Encode::Writer.open(io_string, encoding: 'ASCII-8BIT') do |io|
+        IOStreams::Encode::Writer.stream(io_string, encoding: 'ASCII-8BIT') do |io|
           count += io.write(bad_data)
         end
         assert_equal bad_data, io_string.string

data/test/gzip_reader_test.rb

@@ -12,7 +12,7 @@ class GzipReaderTest < Minitest::Test
 
     describe '.open' do
       it 'file' do
-        result = IOStreams::Gzip::Reader.open(file_name) do |io|
+        result = IOStreams::Gzip::Reader.file(file_name) do |io|
           io.read
         end
         assert_equal decompressed, result
@@ -20,7 +20,7 @@ class GzipReaderTest < Minitest::Test
 
       it 'stream' do
         result = File.open(file_name) do |file|
-          IOStreams::Gzip::Reader.open(file) do |io|
+          IOStreams::Gzip::Reader.stream(file) do |io|
             io.read
           end
         end

data/test/gzip_writer_test.rb

@@ -18,9 +18,9 @@ class GzipWriterTest < Minitest::Test
       temp_file.delete
     end
 
-    describe '.open' do
+    describe '.file' do
       it 'file' do
-        IOStreams::Gzip::Writer.open(file_name) do |io|
+        IOStreams::Gzip::Writer.file(file_name) do |io|
           io.write(decompressed)
         end
         result = Zlib::GzipReader.open(file_name) { |gz| gz.read }
@@ -30,7 +30,7 @@ class GzipWriterTest < Minitest::Test
 
       it 'stream' do
         io_string = StringIO.new(''.b)
-        IOStreams::Gzip::Writer.open(io_string) do |io|
+        IOStreams::Gzip::Writer.stream(io_string) do |io|
           io.write(decompressed)
         end
         io   = StringIO.new(io_string.string)

data/test/io_streams_test.rb

@@ -1,158 +1,65 @@
 require_relative 'test_helper'
 
-class IOStreamsTest < Minitest::Test
-  describe IOStreams do
-    let :source_file_name do
-      File.join(__dir__, 'files', 'text.txt')
-    end
-
-    let :data do
-      File.read(source_file_name)
-    end
-
-    let :temp_file do
-      Tempfile.new('iostreams')
-    end
-
-    let :target_file_name do
-      temp_file.path
-    end
-
-    let :bad_data do
-      [
-        "New M\xE9xico,NE".b,
-        'good line',
-        "New M\xE9xico,\x07SF".b
-      ].join("\n").encode('BINARY')
-    end
-
-    let :stripped_data do
-      bad_data.gsub("\xE9".b, '').gsub("\x07", '')
-    end
-
-    let :multiple_zip_file_name do
-      File.join(File.dirname(__FILE__), 'files', 'multiple_files.zip')
-    end
-
-    let :zip_gz_file_name do
-      File.join(File.dirname(__FILE__), 'files', 'text.zip.gz')
-    end
-
-    let :contents_test_txt do
-      File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
-    end
-
-    let :contents_test_json do
-      File.read(File.join(File.dirname(__FILE__), 'files', 'test.json'))
-    end
-
-    after do
-      temp_file.delete
-    end
-
-    describe '.copy' do
-      it 'streams' do
-        size   = IOStreams.reader(source_file_name) do |source_stream|
-          IOStreams.writer(target_file_name) do |target_stream|
-            IOStreams.copy(source_stream, target_stream)
-          end
+module IOStreams
+  class PathTest < Minitest::Test
+    describe IOStreams do
+      describe '.root' do
+        it 'return default path' do
+          path = ::File.expand_path(::File.join(__dir__, '../tmp/default'))
+          assert_equal path, IOStreams.root.to_s
         end
-        actual = File.read(target_file_name)
 
-        assert_equal actual, data
-        assert_equal actual.size, size
-      end
-
-      it 'IO stream' do
-        size   = File.open(source_file_name) do |source_stream|
-          IOStreams.writer(target_file_name) do |target_stream|
-            IOStreams.copy(source_stream, target_stream)
-          end
+        it 'return downloads path' do
+          path = ::File.expand_path(::File.join(__dir__, '../tmp/downloads'))
+          assert_equal path, IOStreams.root(:downloads).to_s
         end
-        actual = File.read(target_file_name)
-
-        assert_equal actual, data
-        assert_equal actual.size, size
       end
 
-      it 'files' do
-        size   = IOStreams.copy(source_file_name, target_file_name)
-        actual = File.read(target_file_name)
-
-        assert_equal actual, data
-        assert_equal actual.size, size
-      end
-    end
-
-    describe '.streams_for_file_name' do
-      it 'file only' do
-        streams = IOStreams.streams_for_file_name('a.xyz')
-        assert_equal [], streams
-      end
-
-      it 'single stream' do
-        streams = IOStreams.streams_for_file_name('a.gz')
-        assert_equal [:gz], streams
-      end
-
-      it 'multiple streams' do
-        streams = IOStreams.streams_for_file_name('a.xlsx.gz')
-        assert_equal [:xlsx, :gz], streams
-      end
+      describe '.join' do
+        it 'returns path' do
+          assert_equal IOStreams.root.to_s, IOStreams.join.to_s
+        end
 
-      it 'is case-insensitive' do
-        streams = IOStreams.streams_for_file_name('a.GzIp')
-        assert_equal [:gzip], streams
-      end
+        it 'adds path to root' do
+          assert_equal ::File.join(IOStreams.root.to_s, 'test'), IOStreams.join('test').to_s
+        end
 
-      it 'multiple streams are case-insensitive' do
-        streams = IOStreams.streams_for_file_name('a.XlsX.Gz')
-        assert_equal [:xlsx, :gz], streams
-      end
-    end
+        it 'adds paths to root' do
+          assert_equal ::File.join(IOStreams.root.to_s, 'test', 'second', 'third'), IOStreams.join('test', 'second', 'third').to_s
+        end
 
-    describe '.scheme_for_file_name' do
-      it 'default' do
-        assert_nil IOStreams.scheme_for_file_name('a.xyz')
-      end
+        it 'returns path and filename' do
+          path = ::File.join(IOStreams.root.to_s, 'file.xls')
+          assert_equal path, IOStreams.join('file.xls').to_s
+        end
 
-      it 's3' do
-        assert_equal :s3, IOStreams.scheme_for_file_name('s3://a.xyz')
-      end
-    end
+        it 'adds path to root and filename' do
+          path = ::File.join(IOStreams.root.to_s, 'test', 'file.xls')
+          assert_equal path, IOStreams.join('test', 'file.xls').to_s
+        end
 
-    describe '.each_line' do
-      it 'returns a line at a time' do
-        lines = []
-        count = IOStreams.each_line(source_file_name) { |line| lines << line }
-        assert_equal data.lines.map(&:strip), lines
-        assert_equal data.lines.count, count
-      end
+        it 'adds paths to root' do
+          path = ::File.join(IOStreams.root.to_s, 'test', 'second', 'third', 'file.xls')
+          assert_equal path, IOStreams.join('test', 'second', 'third', 'file.xls').to_s
+        end
 
-      it 'strips non-printable characters' do
-        input = StringIO.new(bad_data)
-        lines = []
-        count = IOStreams.each_line(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '') do |line|
-          lines << line
+        it 'return path as sent in when full path' do
+          path = ::File.join(IOStreams.root.to_s, 'file.xls')
+          assert_equal path, IOStreams.join(path).to_s
         end
-        assert_equal stripped_data.lines.map(&:strip), lines
-        assert_equal stripped_data.lines.count, count
       end
-    end
 
-    describe '.reader' do
-      it 'reads a zip file' do
-        result = IOStreams.reader(multiple_zip_file_name, streams: {zip: {entry_file_name: 'test.json'}}) do |io|
-          io.read
+      describe '.path' do
+        it 'default' do
+          path = IOStreams.path('a.xyz')
+          assert path.is_a?(IOStreams::Paths::File), path
         end
-        assert_equal contents_test_json, result
-      end
 
-      it 'reads a zip file from within a gz file' do
-        result = IOStreams.reader(zip_gz_file_name) do |io|
-          io.read
+        it 's3' do
+          skip 'TODO'
+          IOStreams.path('s3://a.xyz')
+          assert_equal :s3, path
         end
-        assert_equal contents_test_txt, result
       end
     end
   end