iostreams 1.10.3 → 2.0.0

data/test/paths/file_test.rb

@@ -1,202 +0,0 @@
-require_relative "../test_helper"
-
-module Paths
-  class FileTest < Minitest::Test
-    describe IOStreams::Paths::File do
-      let(:root) { IOStreams::Paths::File.new("/tmp/iostreams").delete_all }
-      let(:directory) { root.join("/some_test_dir") }
-      let(:data) { "Hello World\nHow are you doing?\nOn this fine day" }
-      let(:file_path) do
-        path = root.join("some_test_dir/test_file.txt")
-        path.writer { |io| io << data }
-        path
-      end
-      let(:file_path2) do
-        path = root.join("some_test_dir/test_file2.txt")
-        path.writer { |io| io << "Hello World2" }
-        path
-      end
-
-      describe "#each" do
-        it "reads lines" do
-          records = []
-          count   = file_path.each { |line| records << line }
-          assert_equal count, data.lines.size
-          assert_equal data.lines.collect(&:strip), records
-        end
-      end
-
-      describe "#each_child" do
-        it "iterates an empty path" do
-          none = nil
-          directory.join("does_not_exist").mkdir.each_child { |path| none = path }
-          assert_nil none
-        end
-
-        it "iterates a non-existant path" do
-          none = nil
-          directory.join("does_not_exist").each_child { |path| none = path }
-          assert_nil none
-        end
-
-        it "find all files" do
-          expected = [file_path.to_s, file_path2.to_s]
-          actual   = root.children("**/*").collect(&:to_s)
-          assert_equal expected.sort, actual.sort
-        end
-
-        it "find matches case-insensitive" do
-          expected = [file_path.to_s, file_path2.to_s]
-          actual   = root.children("**/Test*.TXT").collect(&:to_s)
-          assert_equal expected, actual.sort
-        end
-
-        it "find matches case-sensitive" do
-          skip "TODO"
-          expected = [file_path.to_s, file_path2.to_s]
-          actual   = root.children("**/Test*.TXT", case_sensitive: true).collect(&:to_s)
-          refute_equal expected, actual.sort
-        end
-
-        it "with no block returns enumerator" do
-          expected = [file_path.to_s, file_path2.to_s]
-          actual   = root.each_child("**/*").first(100).collect(&:to_s)
-          assert_equal expected.sort, actual.sort
-        end
-      end
-
-      describe "#mkpath" do
-        it "makes path skipping file_name" do
-          new_path = directory.join("test_mkpath.xls").mkpath
-          assert ::File.exist?(directory.to_s)
-          refute ::File.exist?(new_path.to_s)
-        end
-      end
-
-      describe "#mkdir" do
-        it "makes entire path that does not have a file name" do
-          new_path = directory.join("more_path").mkdir
-          assert ::File.exist?(directory.to_s)
-          assert ::File.exist?(new_path.to_s)
-        end
-      end
-
-      describe "#exist?" do
-        it "true on existing file or directory" do
-          assert ::File.exist?(file_path.to_s)
-          assert ::File.exist?(directory.to_s)
-
-          assert directory.exist?
-          assert file_path.exist?
-        end
-
-        it "false when not found" do
-          non_existant_directory = directory.join("oh_no")
-          refute ::File.exist?(non_existant_directory.to_s)
-
-          non_existant_file_path = directory.join("abc.txt")
-          refute ::File.exist?(non_existant_file_path.to_s)
-
-          refute non_existant_directory.exist?
-          refute non_existant_file_path.exist?
-        end
-      end
-
-      describe "#size" do
-        it "of file" do
-          assert_equal data.size, file_path.size
-        end
-      end
-
-      describe "#realpath" do
-        it "already a real path" do
-          path = ::File.expand_path(__dir__, "../files/test.csv")
-          assert_equal path, IOStreams::Paths::File.new(path).realpath.to_s
-        end
-
-        it "removes .." do
-          path     = ::File.join(__dir__, "../files/test.csv")
-          realpath = ::File.realpath(path)
-          assert_equal realpath, IOStreams::Paths::File.new(path).realpath.to_s
-        end
-      end
-
-      describe "#move_to" do
-        it "move_to existing file" do
-          IOStreams.temp_file("iostreams_move_test", ".txt") do |temp_file|
-            temp_file.write("Hello World")
-            begin
-              target   = temp_file.directory.join("move_test.txt")
-              response = temp_file.move_to(target)
-              assert_equal target, response
-              assert target.exist?
-              refute temp_file.exist?
-              assert_equal "Hello World", response.read
-              assert_equal target.to_s, response.to_s
-            ensure
-              target&.delete
-            end
-          end
-        end
-
-        it "missing source file" do
-          IOStreams.temp_file("iostreams_move_test", ".txt") do |temp_file|
-            refute temp_file.exist?
-            target = temp_file.directory.join("move_test.txt")
-            assert_raises Errno::ENOENT do
-              temp_file.move_to(target)
-            end
-            refute target.exist?
-            refute temp_file.exist?
-          end
-        end
-
-        it "missing target directories" do
-          IOStreams.temp_file("iostreams_move_test", ".txt") do |temp_file|
-            temp_file.write("Hello World")
-            begin
-              target   = temp_file.directory.join("a/b/c/move_test.txt")
-              response = temp_file.move_to(target)
-              assert_equal target, response
-              assert target.exist?
-              refute temp_file.exist?
-              assert_equal "Hello World", response.read
-              assert_equal target.to_s, response.to_s
-            ensure
-              temp_file.directory.join("a").delete_all
-            end
-          end
-        end
-      end
-
-      describe "#delete" do
-        it "deletes existing file" do
-          assert ::File.exist?(file_path.to_s)
-          file_path.delete
-          refute ::File.exist?(file_path.to_s)
-        end
-
-        it "ignores missing file" do
-          file_path.delete
-          file_path.delete
-        end
-      end
-
-      describe "reader" do
-        it "reads file" do
-          assert_equal data, file_path.read
-        end
-      end
-
-      describe "writer" do
-        it "creates file" do
-          new_file_path = directory.join("new.txt")
-          refute ::File.exist?(new_file_path.to_s)
-          new_file_path.writer { |io| io << data }
-          assert ::File.exist?(new_file_path.to_s)
-          assert_equal data.size, new_file_path.size
-        end
-      end
-    end
-  end
-end

data/test/paths/http_test.rb

@@ -1,34 +0,0 @@
-require_relative "../test_helper"
-
-module Paths
-  class HTTPTest < Minitest::Test
-    describe IOStreams::Paths::HTTP do
-      let :url do
-        "http://example.com/index.html?count=10"
-      end
-
-      let :ssl_url do
-        "https://example.com/index.html?count=10"
-      end
-
-      describe ".open" do
-        it "reads http" do
-          result = IOStreams::Paths::HTTP.new(url).read
-          assert_includes result, "<html>"
-        end
-
-        it "reads https" do
-          result = IOStreams::Paths::HTTP.new(ssl_url).read
-          assert_includes result, "<html>"
-        end
-
-        it "does not support streams" do
-          assert_raises URI::InvalidURIError do
-            io = StringIO.new
-            IOStreams::Paths::HTTP.new(io)
-          end
-        end
-      end
-    end
-  end
-end

data/test/paths/matcher_test.rb

@@ -1,120 +0,0 @@
-require_relative "../test_helper"
-
-module Paths
-  class MatcherTest < Minitest::Test
-    describe IOStreams::Paths::Matcher do
-      let :cases do
-        [
-          {
-            path:             "/path/work",
-            pattern:          "a/b/c/**/*",
-            expected_path:    "/path/work/a/b/c",
-            expected_pattern: "**/*",
-            recursive:        true,
-            matches:          %w[/path/work/a/b/c/any/file /path/work/a/b/c/other/file],
-            not_matches:      %w[/path/work/a/b/c/.profile /path/work/a/b/c/sub/.name]
-          },
-          {path: "/path/work", pattern: "a/b/c?/**", expected_path: "/path/work/a/b", expected_pattern: "c?/**", recursive: true},
-          {path: "/path/work", pattern: "**", expected_path: "/path/work", expected_pattern: "**", recursive: true},
-          # Case-insensitive exists that returns the actual file name.
-          {path: "/path/work", pattern: "a/b/file.txt", expected_path: "/path/work/a/b/file.txt", expected_pattern: nil, recursive: false},
-          {
-            path:             "/path/work",
-            pattern:          "a/b/file*{zip,gz}",
-            expected_path:    "/path/work/a/b",
-            expected_pattern: "file*{zip,gz}",
-            recursive:        false,
-            matches:          %w[/path/work/a/b/file.GZ /path/work/a/b/FILE.ZIP /path/work/a/b/file123.zIp],
-            not_matches:      %w[/path/work/a/b/.profile /path/work/a/b/filter.zip /path/work/a/b/outgoing/filter.zip],
-            case_sensitive:   false
-          },
-          {
-            path:             "/path/work",
-            pattern:          "a/b/*",
-            expected_path:    "/path/work/a/b",
-            expected_pattern: "*",
-            recursive:        false,
-            matches:          %w[/path/work/a/b/file.GZ /path/work/a/b/FILE.ZIP /path/work/a/b/file123.zIp],
-            not_matches:      %w[/path/work/a/b/.profile /path/work/a/b/my/filter.zip /path/work/a/b/outgoing/filter.zip],
-            case_sensitive:   false
-          },
-          {
-            path:             "/path/work",
-            pattern:          "a/b/file*{zip,gz}",
-            expected_path:    "/path/work/a/b",
-            expected_pattern: "file*{zip,gz}",
-            recursive:        false,
-            matches:          %w[/path/work/a/b/file.gz /path/work/a/b/file.zip],
-            not_matches:      %w[/path/work/a/b/file.GZ /path/work/a/b/FILE.ZIP],
-            case_sensitive:   true
-          },
-          {path: "/path/work", pattern: "file.txt", expected_path: "/path/work/file.txt", expected_pattern: nil, recursive: false},
-          {path: "/path/work", pattern: "*", expected_path: "/path/work", expected_pattern: "*", recursive: false}
-        ]
-      end
-      # , case_sensitive: false, hidden: false
-
-      describe "#recursive?" do
-        it "identifies recursive paths correctly" do
-          cases.each do |test_case|
-            path    = IOStreams.path(test_case[:path])
-            matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern])
-            assert_equal test_case[:recursive], matcher.recursive?, test_case
-          end
-        end
-      end
-
-      describe "#path?" do
-        it "optimizes path correctly" do
-          cases.each do |test_case|
-            path    = IOStreams.path(test_case[:path])
-            matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern])
-            assert_equal test_case[:expected_path], matcher.path.to_s, test_case
-          end
-        end
-      end
-
-      describe "#pattern" do
-        it "optimizes pattern correctly" do
-          cases.each do |test_case|
-            path    = IOStreams.path(test_case[:path])
-            matcher = IOStreams::Paths::Matcher.new(path, test_case[:pattern])
-            if test_case[:expected_pattern].nil?
-              assert_nil matcher.pattern, test_case
-            else
-              assert_equal test_case[:expected_pattern], matcher.pattern, test_case
-            end
-          end
-        end
-      end
-
-      describe "#match?" do
-        it "matches" do
-          cases.each do |test_case|
-            path           = IOStreams.path(test_case[:path])
-            case_sensitive = test_case.fetch(:case_sensitive, false)
-            matcher        = IOStreams::Paths::Matcher.new(path, test_case[:pattern], case_sensitive: case_sensitive)
-            next unless test_case[:matches]
-
-            test_case[:matches].each do |file_name|
-              assert matcher.match?(file_name), test_case.merge(file_name: file_name)
-            end
-          end
-        end
-
-        it "should not match" do
-          cases.each_with_index do |test_case, index|
-            path           = IOStreams.path(test_case[:path])
-            case_sensitive = test_case.key?(:case_sensitive) ? test_case[:case_sensitive] : false
-            matcher        = IOStreams::Paths::Matcher.new(path, test_case[:pattern], case_sensitive: case_sensitive)
-            next unless test_case[:not_matches]
-
-            test_case[:not_matches].each do |file_name|
-              refute matcher.match?(file_name), -> { {case_sensitive: case_sensitive, test_case_number: index + 1, failed_file_name: file_name, test_case: test_case}.ai }
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/test/paths/s3_test.rb

@@ -1,220 +0,0 @@
-require_relative "../test_helper"
-
-module Paths
-  class S3Test < Minitest::Test
-    describe IOStreams::Paths::S3 do
-      before do
-        skip "Supply 'S3_BUCKET_NAME' environment variable with S3 bucket name to test S3 paths" unless ENV["S3_BUCKET_NAME"]
-      end
-
-      let :file_name do
-        File.join(File.dirname(__FILE__), "..", "files", "text file.txt")
-      end
-
-      let :raw do
-        File.read(file_name)
-      end
-
-      let(:root_path) { IOStreams::Paths::S3.new("s3://#{ENV['S3_BUCKET_NAME']}/iostreams_test") }
-
-      let :existing_path do
-        path = root_path.join("test.txt")
-        path.write(raw) unless path.exist?
-        path
-      end
-
-      let :missing_path do
-        root_path.join("unknown.txt")
-      end
-
-      let :write_path do
-        root_path.join("writer_test.txt").delete
-      end
-
-      describe "#delete" do
-        it "existing file" do
-          assert existing_path.delete.is_a?(IOStreams::Paths::S3)
-        end
-
-        it "missing file" do
-          assert missing_path.delete.is_a?(IOStreams::Paths::S3)
-        end
-      end
-
-      describe "#exist?" do
-        it "existing file" do
-          assert existing_path.exist?
-        end
-
-        it "missing file" do
-          refute missing_path.exist?
-        end
-      end
-
-      describe "#mkpath" do
-        it "returns self for non-existant path" do
-          assert existing_path.mkpath.is_a?(IOStreams::Paths::S3)
-        end
-
-        it "checks for lack of existence" do
-          assert missing_path.mkpath.is_a?(IOStreams::Paths::S3)
-        end
-      end
-
-      describe "#mkdir" do
-        it "returns self for non-existant path" do
-          assert existing_path.mkdir.is_a?(IOStreams::Paths::S3)
-        end
-
-        it "checks for lack of existence" do
-          assert missing_path.mkdir.is_a?(IOStreams::Paths::S3)
-        end
-      end
-
-      describe "#reader" do
-        it "reads" do
-          assert_equal raw, existing_path.read
-        end
-      end
-
-      describe "#size" do
-        it "existing file" do
-          assert_equal raw.size, existing_path.size
-        end
-
-        it "missing file" do
-          assert_nil missing_path.size
-        end
-      end
-
-      describe "#writer" do
-        it "writes" do
-          assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
-          assert write_path.exist?
-          assert_equal raw, write_path.read
-        end
-      end
-
-      describe "#each_line" do
-        it "reads line by line" do
-          lines = []
-          existing_path.each(:line) { |line| lines << line }
-          assert_equal raw.lines.collect(&:chomp), lines
-        end
-      end
-
-      describe "#each_child" do
-        # TODO: case_sensitive: false, directories: false, hidden: false
-        let(:abd_file_names) { %w[abd/test1.txt abd/test5.file abd/extra/file.csv] }
-        let(:files_for_test) { abd_file_names + %w[xyz/test2.csv xyz/another.csv] }
-
-        let :each_root do
-          root_path.join("each_child_test")
-        end
-
-        let :multiple_paths do
-          files_for_test.collect { |file_name| each_root.join(file_name) }
-        end
-
-        let :write_raw_data do
-          multiple_paths.each { |path| path.write(raw) unless path.exist? }
-        end
-
-        it "existing file returns just the file itself" do
-          # Glorified exists call
-          existing_path
-          assert_equal root_path.join("test.txt").to_s, root_path.children("test.txt").first.to_s
-        end
-
-        it "missing file does nothing" do
-          # Glorified exists call
-          assert_equal [], missing_path.children("readme").collect(&:to_s)
-        end
-
-        it "returns all the children" do
-          write_raw_data
-          assert_equal multiple_paths.collect(&:to_s).sort, each_root.children("**/*").collect(&:to_s).sort
-        end
-
-        it "returns all the children under a sub-dir" do
-          write_raw_data
-          expected = %w[abd/test1.txt abd/test5.file].collect { |file_name| each_root.join(file_name) }
-          assert_equal expected.collect(&:to_s).sort, each_root.children("abd/*").collect(&:to_s).sort
-        end
-
-        it "missing path" do
-          count = 0
-          missing_path.each_child { |_| count += 1 }
-          assert_equal 0, count
-        end
-
-        # Test is here since all the test artifacts have been created already in S3.
-        describe "IOStreams.each_child" do
-          it "returns all the children" do
-            write_raw_data
-            children = []
-            IOStreams.each_child(each_root.join("**/*").to_s) { |child| children << child }
-            assert_equal multiple_paths.collect(&:to_s).sort, children.collect(&:to_s).sort
-          end
-        end
-      end
-
-      describe "#move_to" do
-        it "moves existing file" do
-          source = root_path.join("move_test_source.txt")
-          begin
-            source.write("Hello World")
-            target   = source.directory.join("move_test_target.txt")
-            response = source.move_to(target)
-            assert_equal target, response
-            assert target.exist?
-            refute source.exist?
-            assert_equal "Hello World", response.read
-            assert_equal target.to_s, response.to_s
-          ensure
-            source&.delete
-            target&.delete
-          end
-        end
-
-        it "missing source file" do
-          source = root_path.join("move_test_source.txt")
-          refute source.exist?
-          begin
-            target = source.directory.join("move_test_target.txt")
-            assert_raises Aws::S3::Errors::NoSuchKey do
-              source.move_to(target)
-            end
-            refute target.exist?
-          ensure
-            source&.delete
-            target&.delete
-          end
-        end
-
-        it "missing target directories" do
-          source = root_path.join("move_test_source.txt")
-          begin
-            source.write("Hello World")
-            target   = source.directory.join("a/b/c/move_test_target.txt")
-            response = source.move_to(target)
-            assert_equal target, response
-            assert target.exist?
-            refute source.exist?
-            assert_equal "Hello World", response.read
-            assert_equal target.to_s, response.to_s
-          ensure
-            source&.delete
-            target&.delete
-          end
-        end
-      end
-
-      describe "#partial_files_visible?" do
-        it "visible only after upload" do
-          refute root_path.partial_files_visible?
-        end
-      end
-    end
-  end
-end

data/test/paths/sftp_test.rb

@@ -1,106 +0,0 @@
-require_relative "../test_helper"
-
-module Paths
-  class SFTPTest < Minitest::Test
-    describe IOStreams::Paths::SFTP do
-      before do
-        unless ENV["SFTP_HOSTNAME"]
-          skip "Supply environment variables to test SFTP paths: SFTP_HOSTNAME, SFTP_USERNAME, SFTP_PASSWORD, and optional SFTP_DIR, SFTP_IDENTITY_FILE"
-        end
-      end
-
-      let(:host_name) { ENV["SFTP_HOSTNAME"] }
-      let(:username) { ENV["SFTP_USERNAME"] }
-      let(:password) { ENV["SFTP_PASSWORD"] }
-      let(:ftp_dir) { ENV["SFTP_DIR"] || "iostreams_test" }
-      let(:identity_username) { ENV["SFTP_IDENTITY_USERNAME"] || username }
-
-      let(:url) { File.join("sftp://", host_name, ftp_dir) }
-
-      let(:file_name) { File.join(File.dirname(__FILE__), "..", "files", "text file.txt") }
-      let(:raw) { File.read(file_name) }
-
-      let(:root_path) do
-        if ENV["SFTP_HOST_KEY"]
-          IOStreams::Paths::SFTP.new(url, username: username, password: password, ssh_options: {"HostKey" => ENV["SFTP_HOST_KEY"]})
-        else
-          IOStreams::Paths::SFTP.new(url, username: username, password: password)
-        end
-      end
-
-      let :existing_path do
-        path = root_path.join("test.txt")
-        path.write(raw)
-        path
-      end
-
-      let :missing_path do
-        root_path.join("unknown_path", "test_file.txt")
-      end
-
-      let :missing_file_path do
-        root_path.join("test_file.txt")
-      end
-
-      let :write_path do
-        root_path.join("writer_test.txt")
-      end
-
-      describe "#reader" do
-        it "reads" do
-          assert_equal raw, existing_path.read
-        end
-
-        it "fails when the file does not exist" do
-          assert_raises IOStreams::Errors::CommunicationsFailure do
-            missing_file_path.read
-          end
-        end
-
-        it "fails when the directory does not exist" do
-          assert_raises IOStreams::Errors::CommunicationsFailure do
-            missing_path.read
-          end
-        end
-      end
-
-      describe "#writer" do
-        it "writes" do
-          assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
-          assert_equal raw, write_path.read
-        end
-
-        it "fails when the directory does not exist" do
-          assert_raises IOStreams::Errors::CommunicationsFailure do
-            missing_path.write("Bad path")
-          end
-        end
-
-        describe "use identity file instead of password" do
-          let :root_path do
-            IOStreams::Paths::SFTP.new(url, username: identity_username, ssh_options: {"IdentityFile" => ENV["SFTP_IDENTITY_FILE"]})
-          end
-
-          it "writes" do
-            skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
-            assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
-            assert_equal raw, write_path.read
-          end
-        end
-
-        describe "use identity key instead of password" do
-          let :root_path do
-            key = File.open(ENV["SFTP_IDENTITY_FILE"], "rb", &:read)
-            IOStreams::Paths::SFTP.new(url, username: identity_username, ssh_options: {"IdentityKey" => key})
-          end
-
-          it "writes" do
-            skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
-            assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
-            assert_equal raw, write_path.read
-          end
-        end
-      end
-    end
-  end
-end