RubyGems - iostreams - Versions diffs - 0.20.3 → 1.0.0.beta - Mend

iostreams 0.20.3 → 1.0.0.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (90) hide show

checksums.yaml +4 -4
data/lib/io_streams/bzip2/reader.rb +9 -21
data/lib/io_streams/bzip2/writer.rb +9 -21
data/lib/io_streams/deprecated.rb +217 -0
data/lib/io_streams/encode/reader.rb +12 -16
data/lib/io_streams/encode/writer.rb +9 -13
data/lib/io_streams/errors.rb +6 -6
data/lib/io_streams/gzip/reader.rb +7 -14
data/lib/io_streams/gzip/writer.rb +7 -15
data/lib/io_streams/io_streams.rb +182 -524
data/lib/io_streams/line/reader.rb +9 -9
data/lib/io_streams/line/writer.rb +10 -11
data/lib/io_streams/path.rb +190 -0
data/lib/io_streams/paths/file.rb +176 -0
data/lib/io_streams/paths/http.rb +92 -0
data/lib/io_streams/paths/matcher.rb +61 -0
data/lib/io_streams/paths/s3.rb +269 -0
data/lib/io_streams/paths/sftp.rb +99 -0
data/lib/io_streams/pgp.rb +47 -19
data/lib/io_streams/pgp/reader.rb +20 -28
data/lib/io_streams/pgp/writer.rb +24 -46
data/lib/io_streams/reader.rb +28 -0
data/lib/io_streams/record/reader.rb +20 -16
data/lib/io_streams/record/writer.rb +28 -28
data/lib/io_streams/row/reader.rb +22 -26
data/lib/io_streams/row/writer.rb +29 -28
data/lib/io_streams/stream.rb +400 -0
data/lib/io_streams/streams.rb +125 -0
data/lib/io_streams/symmetric_encryption/reader.rb +5 -13
data/lib/io_streams/symmetric_encryption/writer.rb +16 -15
data/lib/io_streams/tabular/header.rb +9 -3
data/lib/io_streams/tabular/parser/array.rb +8 -3
data/lib/io_streams/tabular/parser/csv.rb +6 -2
data/lib/io_streams/tabular/parser/hash.rb +4 -1
data/lib/io_streams/tabular/parser/json.rb +3 -1
data/lib/io_streams/tabular/parser/psv.rb +3 -1
data/lib/io_streams/tabular/utility/csv_row.rb +9 -8
data/lib/io_streams/utils.rb +22 -0
data/lib/io_streams/version.rb +1 -1
data/lib/io_streams/writer.rb +28 -0
data/lib/io_streams/xlsx/reader.rb +7 -19
data/lib/io_streams/zip/reader.rb +7 -26
data/lib/io_streams/zip/writer.rb +21 -38
data/lib/iostreams.rb +15 -15
data/test/bzip2_reader_test.rb +3 -3
data/test/bzip2_writer_test.rb +3 -3
data/test/deprecated_test.rb +123 -0
data/test/encode_reader_test.rb +3 -3
data/test/encode_writer_test.rb +6 -6
data/test/gzip_reader_test.rb +2 -2
data/test/gzip_writer_test.rb +3 -3
data/test/io_streams_test.rb +43 -136
data/test/line_reader_test.rb +20 -20
data/test/line_writer_test.rb +3 -3
data/test/path_test.rb +30 -28
data/test/paths/file_test.rb +206 -0
data/test/paths/http_test.rb +34 -0
data/test/paths/matcher_test.rb +111 -0
data/test/paths/s3_test.rb +207 -0
data/test/pgp_reader_test.rb +8 -8
data/test/pgp_writer_test.rb +13 -13
data/test/record_reader_test.rb +5 -5
data/test/record_writer_test.rb +4 -4
data/test/row_reader_test.rb +5 -5
data/test/row_writer_test.rb +6 -6
data/test/stream_test.rb +116 -0
data/test/streams_test.rb +255 -0
data/test/utils_test.rb +20 -0
data/test/xlsx_reader_test.rb +3 -3
data/test/zip_reader_test.rb +12 -12
data/test/zip_writer_test.rb +5 -5
metadata +33 -45
data/lib/io_streams/base_path.rb +0 -72
data/lib/io_streams/file/path.rb +0 -58
data/lib/io_streams/file/reader.rb +0 -12
data/lib/io_streams/file/writer.rb +0 -22
data/lib/io_streams/http/reader.rb +0 -71
data/lib/io_streams/s3.rb +0 -26
data/lib/io_streams/s3/path.rb +0 -40
data/lib/io_streams/s3/reader.rb +0 -28
data/lib/io_streams/s3/writer.rb +0 -85
data/lib/io_streams/sftp/reader.rb +0 -67
data/lib/io_streams/sftp/writer.rb +0 -68
data/test/base_path_test.rb +0 -35
data/test/file_path_test.rb +0 -97
data/test/file_reader_test.rb +0 -33
data/test/file_writer_test.rb +0 -50
data/test/http_reader_test.rb +0 -38
data/test/s3_reader_test.rb +0 -41
data/test/s3_writer_test.rb +0 -41

data/test/paths/http_test.rb ADDED

@@ -0,0 +1,34 @@
+require_relative '../test_helper'
+module Paths
+  class HTTPTest < Minitest::Test
+    describe IOStreams::Paths::HTTP do
+      let :url do
+        "http://example.com/index.html?count=10"
+      end
+      let :ssl_url do
+        "https://example.com/index.html?count=10"
+      end
+      describe '.open' do
+        it 'reads http' do
+          result = IOStreams::Paths::HTTP.new(url).read
+          assert_includes result, "<html>"
+        end
+        it 'reads https' do
+          result = IOStreams::Paths::HTTP.new(ssl_url).read
+          assert_includes result, "<html>"
+        end
+        it 'does not support streams' do
+          assert_raises URI::InvalidURIError do
+            io = StringIO.new
+            IOStreams::Paths::HTTP.new(io)
+          end
+        end
+      end
+    end
+  end
+end

data/test/paths/matcher_test.rb ADDED

@@ -0,0 +1,111 @@
+require_relative '../test_helper'
+module Paths
+  class MatcherTest < Minitest::Test
+    describe IOStreams::Paths::Matcher do
+      let :cases do
+        [
+          {
+            path:             "/path/work",
+            pattern:          "a/b/c/**/*",
+            expected_path:    "/path/work/a/b/c",
+            expected_pattern: "**/*",
+            recursive:        true,
+            matches:          %w[/path/work/a/b/c/any/file /path/work/a/b/c/other/file /path/work/a/b/c/sub/.name],
+            not_matches:      %w[/path/work/a/b/c/.profile]
+          },
+          {path: "/path/work", pattern: "a/b/c?/**/*", expected_path: "/path/work/a/b", expected_pattern: "c?/**/*", recursive: true},
+          {path: "/path/work", pattern: "**/*", expected_path: "/path/work", expected_pattern: "**/*", recursive: true},
+          # Case-insensitive exists that returns the actual file name.
+          {path: "/path/work", pattern: "a/b/file.txt", expected_path: "/path/work/a/b/file.txt", expected_pattern: nil, recursive: false},
+          {
+            path:             "/path/work",
+            pattern:          "a/b/file*{zip,gz}",
+            expected_path:    "/path/work/a/b",
+            expected_pattern: "file*{zip,gz}",
+            recursive:        false,
+            matches:          %w[/path/work/a/b/file.GZ /path/work/a/b/FILE.ZIP /path/work/a/b/file123.zIp],
+            not_matches:      %w[/path/work/a/b/.profile /path/work/a/b/filter.zip],
+            case_sensitive:   false
+          },
+          {
+            path:             "/path/work",
+            pattern:          "a/b/file*{zip,gz}",
+            expected_path:    "/path/work/a/b",
+            expected_pattern: "file*{zip,gz}",
+            recursive:        false,
+            matches:          %w[/path/work/a/b/file.gz /path/work/a/b/file.zip],
+            not_matches:      %w[/path/work/a/b/file.GZ /path/work/a/b/FILE.ZIP],
+            case_sensitive:   true
+          },
+          {path: "/path/work", pattern: "file.txt", expected_path: "/path/work/file.txt", expected_pattern: nil, recursive: false},
+          {path: "/path/work", pattern: "*", expected_path: "/path/work", expected_pattern: "*", recursive: false},
+        ]
+      end
+      # , case_sensitive: false, hidden: false
+      describe '#recursive?' do
+        it 'identifies recursive paths correctly' do
+          cases.each do |test_case|
+            path    = IOStreams.path(test_case[:path])
+            matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern])
+            assert_equal test_case[:recursive], matcher.recursive?, test_case
+          end
+        end
+      end
+      describe "#path?" do
+        it 'optimizes path correctly' do
+          cases.each do |test_case|
+            path    = IOStreams.path(test_case[:path])
+            matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern])
+            assert_equal test_case[:expected_path], matcher.path.to_s, test_case
+          end
+        end
+      end
+      describe "#pattern" do
+        it 'optimizes pattern correctly' do
+          cases.each do |test_case|
+            path    = IOStreams.path(test_case[:path])
+            matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern])
+            if test_case[:expected_pattern].nil?
+              assert_nil matcher.pattern, test_case
+            else
+              assert_equal test_case[:expected_pattern], matcher.pattern, test_case
+            end
+          end
+        end
+      end
+      describe "#match?" do
+        it 'matches' do
+          cases.each do |test_case|
+            path           = IOStreams.path(test_case[:path])
+            case_sensitive = test_case.fetch(:case_sensitive, false)
+            matcher        = IOStreams::Paths::Matcher.new(path, test_case[:pattern], case_sensitive: case_sensitive)
+            next unless test_case[:matches]
+            test_case[:matches].each do |file_name|
+              assert matcher.match?(file_name), test_case.merge(file_name: file_name)
+            end
+          end
+        end
+        it 'should not match' do
+          cases.each do |test_case|
+            path           = IOStreams.path(test_case[:path])
+            case_sensitive = test_case.fetch(case_sensitive, :false)
+            matcher        = IOStreams::Paths::Matcher.new(path, test_case[:pattern], case_sensitive: case_sensitive)
+            next unless test_case[:not_matches]
+            test_case[:not_matches].each do |file_name|
+              refute matcher.match?(file_name), test_case.merge(file_name: file_name)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/test/paths/s3_test.rb ADDED

@@ -0,0 +1,207 @@
+require_relative '../test_helper'
+module Paths
+  class S3Test < Minitest::Test
+    describe IOStreams::Paths::S3 do
+      before do
+        unless ENV['S3_BUCKET_NAME']
+          skip "Supply 'S3_BUCKET_NAME' environment variable with S3 bucket name to test S3 paths"
+        end
+      end
+      let :file_name do
+        File.join(File.dirname(__FILE__), '..', 'files', 'text.txt')
+      end
+      let :raw do
+        File.read(file_name)
+      end
+      let(:root_path) { IOStreams::Paths::S3.new("s3://#{ENV['S3_BUCKET_NAME']}/iostreams_test") }
+      let :existing_path do
+        path = root_path.join('test.txt')
+        path.write(raw) unless path.exist?
+        path
+      end
+      let :missing_path do
+        root_path.join('unknown.txt')
+      end
+      let :write_path do
+        root_path.join('writer_test.txt').delete
+      end
+      describe '#delete' do
+        it 'existing file' do
+          assert existing_path.delete.is_a?(IOStreams::Paths::S3)
+        end
+        it 'missing file' do
+          assert missing_path.delete.is_a?(IOStreams::Paths::S3)
+        end
+      end
+      describe '#exist?' do
+        it 'existing file' do
+          assert existing_path.exist?
+        end
+        it 'missing file' do
+          refute missing_path.exist?
+        end
+      end
+      describe '#mkpath' do
+        it 'returns self for non-existant path' do
+          assert existing_path.mkpath.is_a?(IOStreams::Paths::S3)
+        end
+        it 'checks for lack of existence' do
+          assert missing_path.mkpath.is_a?(IOStreams::Paths::S3)
+        end
+      end
+      describe '#mkdir' do
+        it 'returns self for non-existant path' do
+          assert existing_path.mkdir.is_a?(IOStreams::Paths::S3)
+        end
+        it 'checks for lack of existence' do
+          assert missing_path.mkdir.is_a?(IOStreams::Paths::S3)
+        end
+      end
+      describe '#reader' do
+        it 'reads' do
+          assert_equal raw, existing_path.reader { |io| io.read }
+        end
+      end
+      describe '#size' do
+        it 'existing file' do
+          assert_equal raw.size, existing_path.size
+        end
+        it 'missing file' do
+          assert_nil missing_path.size
+        end
+      end
+      describe '#writer' do
+        it 'writes' do
+          assert_equal raw.size, write_path.writer { |io| io.write(raw) }
+          assert write_path.exist?
+          assert_equal raw, write_path.read
+        end
+      end
+      describe '#each_child' do
+        # TODO: case_sensitive: false, directories: false, hidden: false
+        let(:abd_file_names) { %w[abd/test1.txt abd/test5.file abd/extra/file.csv] }
+        let(:files_for_test) { abd_file_names + %w[xyz/test2.csv xyz/another.csv] }
+        let :each_root do
+          root_path.join('each_child_test')
+        end
+        let :multiple_paths do
+          files_for_test.collect { |file_name| each_root.join(file_name) }
+        end
+        let :write_raw_data do
+          multiple_paths.each { |path| path.write(raw) unless path.exist? }
+        end
+        it 'existing file returns just the file itself' do
+          # Glorified exists call
+          assert_equal [each_root.join('readme').to_s], each_root.children("readme").collect(&:to_s)
+        end
+        it 'missing file does nothing' do
+          # Glorified exists call
+          assert_equal [], missing_path.children("readme").collect(&:to_s)
+        end
+        it 'returns all the children' do
+          write_raw_data
+          assert_equal multiple_paths.collect(&:to_s).sort, each_root.children("**/*").collect(&:to_s).sort
+        end
+        it 'returns all the children under a sub-dir' do
+          write_raw_data
+          expected = abd_file_names.collect { |file_name| each_root.join(file_name) }
+          assert_equal expected.collect(&:to_s).sort, each_root.children("abd/*").collect(&:to_s).sort
+        end
+        it 'missing path' do
+          count = 0
+          missing_path.each_child { |_| count += 1 }
+          assert_equal 0, count
+        end
+        # Test is here since all the test artifacts have been created already in S3.
+        describe 'IOStreams.each_child' do
+          it 'returns all the children' do
+            write_raw_data
+            children = []
+            IOStreams.each_child(each_root.join("**/*").to_s) { |child| children << child }
+            assert_equal multiple_paths.collect(&:to_s).sort, children.collect(&:to_s).sort
+          end
+        end
+      end
+      describe '#move_to' do
+        it 'moves existing file' do
+          source = root_path.join("move_test_source.txt")
+          begin
+            source.write("Hello World")
+            target   = source.directory.join("move_test_target.txt")
+            response = source.move_to(target)
+            assert_equal target, response
+            assert target.exist?
+            refute source.exist?
+            assert_equal "Hello World", response.read
+            assert_equal target.to_s, response.to_s
+          ensure
+            source&.delete
+            target&.delete
+          end
+        end
+        it 'missing source file' do
+          source = root_path.join("move_test_source.txt")
+          refute source.exist?
+          begin
+            target = source.directory.join("move_test_target.txt")
+            assert_raises Aws::S3::Errors::NoSuchKey do
+              source.move_to(target)
+            end
+            refute target.exist?
+          ensure
+            source&.delete
+            target&.delete
+          end
+        end
+        it 'missing target directories' do
+          source = root_path.join("move_test_source.txt")
+          begin
+            source.write("Hello World")
+            target   = source.directory.join("a/b/c/move_test_target.txt")
+            response = source.move_to(target)
+            assert_equal target, response
+            assert target.exist?
+            refute source.exist?
+            assert_equal "Hello World", response.read
+            assert_equal target.to_s, response.to_s
+          ensure
+            source&.delete
+            target&.delete
+          end
+        end
+      end
+    end
+  end
+end

data/test/pgp_reader_test.rb CHANGED

@@ -15,30 +15,30 @@ class PgpReaderTest < Minitest::Test
       temp_file.delete
     end
-    describe '.open' do
+    describe '.file' do
       it 'reads encrypted file' do
-        IOStreams::Pgp::Writer.open(temp_file.path, recipient: 'receiver@example.org') do |io|
+        IOStreams::Pgp::Writer.file(temp_file.path, recipient: 'receiver@example.org') do |io|
           io.write(decrypted)
         end
-        result = IOStreams::Pgp::Reader.open(temp_file.path, passphrase: 'receiver_passphrase') { |file| file.read }
+        result = IOStreams::Pgp::Reader.file(temp_file.path, passphrase: 'receiver_passphrase') { |file| file.read }
         assert_equal decrypted, result
       end
       it 'fails with bad passphrase' do
         assert_raises IOStreams::Pgp::Failure do
-          IOStreams::Pgp::Reader.open(temp_file.path, passphrase: 'BAD') { |file| file.read }
+          IOStreams::Pgp::Reader.file(temp_file.path, passphrase: 'BAD') { |file| file.read }
         end
       end
       it 'streams input' do
-        IOStreams::Pgp::Writer.open(temp_file.path, recipient: 'receiver@example.org') do |io|
+        io_string = StringIO.new(''.b)
+        IOStreams::Pgp::Writer.stream(io_string, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
           io.write(decrypted)
         end
-        encrypted = IOStreams::File::Reader.open(temp_file.path, &:read)
-        io        = StringIO.new(encrypted)
-        result = IOStreams::Pgp::Reader.open(io, passphrase: 'receiver_passphrase', &:read)
+        io     = StringIO.new(io_string.string)
+        result = IOStreams::Pgp::Reader.stream(io, passphrase: 'receiver_passphrase', &:read)
         assert_equal decrypted, result
       end

data/test/pgp_writer_test.rb CHANGED

@@ -19,13 +19,13 @@ class PgpWriterTest < Minitest::Test
       temp_file.delete
     end
-    describe '.open' do
+    describe '.file' do
       it 'writes encrypted text file' do
-        IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org', binary: false) do |io|
+        IOStreams::Pgp::Writer.file(file_name, recipient: 'receiver@example.org') do |io|
           io.write(decrypted)
         end
-        result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase', binary: false, &:read)
+        result = IOStreams::Pgp::Reader.file(file_name, passphrase: 'receiver_passphrase', &:read)
         assert_equal decrypted, result
       end
@@ -34,28 +34,28 @@ class PgpWriterTest < Minitest::Test
         binary_data      = File.open(binary_file_name, 'rb') { |file| file.read }
         File.open(binary_file_name, 'rb') do |input|
-          IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org') do |output|
-            IOStreams.copy(input, output)
+          IOStreams::Pgp::Writer.file(file_name, recipient: 'receiver@example.org') do |output|
+            IO.copy_stream(input, output)
           end
         end
-        result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase', &:read)
+        result = IOStreams::Pgp::Reader.file(file_name, passphrase: 'receiver_passphrase', &:read)
         assert_equal binary_data, result
       end
       it 'writes and signs encrypted file' do
-        IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
+        IOStreams::Pgp::Writer.file(file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
           io.write(decrypted)
         end
-        result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase', &:read)
+        result = IOStreams::Pgp::Reader.file(file_name, passphrase: 'receiver_passphrase', &:read)
         assert_equal decrypted, result
       end
       it 'fails with bad signer passphrase' do
         skip 'GnuPG v2.1 and above passes when it should not' if IOStreams::Pgp.pgp_version.to_f >= 2.1
         assert_raises IOStreams::Pgp::Failure do
-          IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'BAD') do |io|
+          IOStreams::Pgp::Writer.file(file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'BAD') do |io|
             io.write(decrypted)
           end
         end
@@ -63,7 +63,7 @@ class PgpWriterTest < Minitest::Test
       it 'fails with bad recipient' do
         assert_raises IOStreams::Pgp::Failure do
-          IOStreams::Pgp::Writer.open(file_name, recipient: 'BAD@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
+          IOStreams::Pgp::Writer.file(file_name, recipient: 'BAD@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
             io.write(decrypted)
             # Allow process to terminate
             sleep 1
@@ -74,7 +74,7 @@ class PgpWriterTest < Minitest::Test
       it 'fails with bad signer' do
         assert_raises IOStreams::Pgp::Failure do
-          IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org', signer: 'BAD@example.org', signer_passphrase: 'sender_passphrase') do |io|
+          IOStreams::Pgp::Writer.file(file_name, recipient: 'receiver@example.org', signer: 'BAD@example.org', signer_passphrase: 'sender_passphrase') do |io|
             io.write(decrypted)
           end
         end
@@ -82,12 +82,12 @@ class PgpWriterTest < Minitest::Test
       it 'writes to a stream' do
         io_string = StringIO.new(''.b)
-        IOStreams::Pgp::Writer.open(io_string, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
+        IOStreams::Pgp::Writer.stream(io_string, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
           io.write(decrypted)
         end
         io     = StringIO.new(io_string.string)
-        result = IOStreams::Pgp::Reader.open(io, passphrase: 'receiver_passphrase', &:read)
+        result = IOStreams::Pgp::Reader.stream(io, passphrase: 'receiver_passphrase', &:read)
         assert_equal decrypted, result
       end