iostreams 1.0.0.beta6 → 1.0.0.beta7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 167014d27ff57416cce79209be3025489a6312e208f3edccebb17f535dae9ae1
-  data.tar.gz: 55e1f44b9613010dc81f02bb03d92ee9ec32c2b9d49c4ec99657a9c072700fb7
+  metadata.gz: 4d8a7eaa8009e55339490c405a111986838219094c3df4d0b2303f7e1d16e951
+  data.tar.gz: 3a0635cb14723f81054674759f6c0d1752196a5d8d5c6c69dc96d9910fa63b84
 SHA512:
-  metadata.gz: 7d82649bff9f6ff9c97e58caa499c0cf5ce5090553816a76155374d6123b91fb673426c4eddfed0e5da24fff89e77997e7064b87ec95bf7aac0f1bac7b6f9dd0
-  data.tar.gz: 97d1fa3c4e5771d4f7ad3e9d3b3580f5b4cadbc5d57838277e7e7fe21224d496b5978e788f54981143d579a39131b907c9b069881e8d2b200356d49b4b2362f8
+  metadata.gz: e61d3ed29557eb7abd74737c6dd2b81ecfb2667c0a1f753a4a66fd746816f7083ea7dfc7fd8c86339f0dc671418692ccdef0948875b522d39e16a262c820c188
+  data.tar.gz: 5d6b8cfc44627bd41b5170cc5e0c55ad10089d624bab6409ef864738c114fa614a9d92895c3e29eed20b88ff6627a06834e4aa722fba369fd11018efb38a5a4d

data/lib/io_streams/io_streams.rb

@@ -52,7 +52,7 @@ module IOStreams
 
     elements         = elements.collect(&:to_s)
     path             = ::File.join(*elements)
-    extracted_scheme = path.include?("://") ? URI.parse(path).scheme : nil
+    extracted_scheme = path.include?("://") ? Utils::URI.new(path).scheme : nil
     klass            = scheme(extracted_scheme)
     args.empty? ? klass.new(path) : klass.new(path, **args)
   end

data/lib/io_streams/path.rb

@@ -160,6 +160,14 @@ module IOStreams
       !(path =~ /\.(enc|pgp|gpg)\z/i).nil?
     end
 
+    # Returns [true|false] whether partially created files are visible on this path.
+    #
+    # With local file systems a file that is still being written to is visbile.
+    # On AWS S3 a file is not visible until it is completely written to the bucket.
+    def partial_files_visible?
+      true
+    end
+
     # TODO: Other possible methods:
     # - rename - File.rename
     # - rmtree - delete everything under this path - FileUtils.rm_r

data/lib/io_streams/paths/s3.rb

@@ -133,10 +133,10 @@ module IOStreams
       def initialize(url, client: nil, access_key_id: nil, secret_access_key: nil, **args)
         Utils.load_soft_dependency('aws-sdk-s3', 'AWS S3') unless defined?(::Aws::S3::Client)
 
-        uri = URI.parse(url)
+        uri = Utils::URI.new(url)
         raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == 's3'
 
-        @bucket_name = uri.host
+        @bucket_name = uri.hostname
         key          = uri.path.sub(%r{\A/}, '')
         if client.is_a?(Hash)
           client[:access_key_id]     = access_key_id if access_key_id
@@ -147,7 +147,7 @@ module IOStreams
         end
         @options = args
 
-        URI.decode_www_form(uri.query).each { |key, value| @options[key] = value } if uri.query
+        @options.merge(uri.query) if uri.query
 
         super(key)
       end
@@ -259,8 +259,6 @@ module IOStreams
       # Notes:
       # - Currently all S3 lookups are recursive as of the pattern regardless of whether the pattern includes `**`.
       def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
-        raise(NotImplementedError, "AWS S3 #each_child does not yet return directories") if directories
-
         matcher = Matcher.new(self, pattern, case_sensitive: case_sensitive, hidden: hidden)
 
         # When the pattern includes an exact file name without any pattern characters
@@ -269,22 +267,29 @@ module IOStreams
           return
         end
 
-        prefix = URI.parse(matcher.path.to_s).path.sub(%r{\A/}, '')
+        prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/}, '')
         token  = nil
         loop do
           # Fetches upto 1,000 entries at a time
           resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
           resp.contents.each do |object|
+            next if !directories && object.key.end_with?("/")
+
             file_name = ::File.join("s3://", resp.name, object.key)
             next unless matcher.match?(file_name)
 
-            yield self.class.new(file_name)
+            yield(self.class.new(file_name), object.to_h)
           end
           token = resp.next_continuation_token
           break if token.nil?
         end
         nil
       end
+
+      # On S3 only files that are completely saved are visible.
+      def partial_files_visible?
+        false
+      end
     end
   end
 end

data/lib/io_streams/paths/sftp.rb

@@ -55,7 +55,7 @@ module IOStreams
       #     puts io.read
       #   end
       def initialize(url, username: nil, password: nil, ruby: true, ssh_options: {})
-        uri = URI.parse(url)
+        uri = Utils::URI.new(url)
         raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
 
         @hostname = uri.hostname
@@ -67,8 +67,7 @@ module IOStreams
         # Not Ruby 2.5 yet: transform_keys(&:to_s)
         @ssh_options = {}
         ssh_options.each_pair { |key, value| @ssh_options[key.to_s] = value }
-
-        URI.decode_www_form(uri.query).each { |key, value| @ssh_options[key] = value } if uri.query
+        @ssh_options.merge(uri.query) if uri.query
 
         super(uri.path)
       end

data/lib/io_streams/stream.rb

@@ -108,6 +108,11 @@ module IOStreams
       end
     end
 
+    def copy_to(target, convert: true)
+      target = IOStreams.path(target) unless target.is_a?(Stream)
+      target.copy_from(self, convert: convert)
+    end
+
     # Iterate over a file / stream returning one line at a time.
     # Embedded lines (within double quotes) will be skipped if
     #   1. The file name contains .csv

data/lib/io_streams/utils.rb

@@ -32,5 +32,24 @@ module IOStreams
       end
       result
     end
+
+    class URI
+      attr_reader :scheme, :hostname, :path, :user, :password, :port, :query
+
+      def initialize(url)
+        url       = url.gsub(' ', '%20')
+        uri       = ::URI.parse(url)
+        @scheme   = uri.scheme
+        @hostname = uri.hostname
+        @path     = CGI.unescape(uri.path)
+        @user     = uri.user
+        @password = uri.password
+        @port     = uri.port
+        if uri.query
+          @query = {}
+          ::URI.decode_www_form(uri.query).each { |key, value| @query[key] = value }
+        end
+      end
+    end
   end
 end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.0.0.beta6".freeze
+  VERSION = "1.0.0.beta7".freeze
 end

data/lib/io_streams/zip/reader.rb

@@ -32,6 +32,17 @@ module IOStreams
           fin&.close
         end
 
+        def self.get_entry(zin, entry_file_name)
+          if entry_file_name.nil?
+            zin.get_next_entry
+            return true
+          end
+
+          while entry = zin.get_next_entry
+            return true if entry.name == entry_file_name
+          end
+          false
+        end
       else
         # Read from a zip file or stream, decompressing the contents as it is read
         # The input stream from the first file found in the zip file is passed

data/test/files/text file.txt

@@ -0,0 +1,3 @@
+Hello World
+Line2
+Line3

data/test/paths/s3_test.rb

@@ -10,7 +10,7 @@ module Paths
       end
 
       let :file_name do
-        File.join(File.dirname(__FILE__), '..', 'files', 'text.txt')
+        File.join(File.dirname(__FILE__), '..', 'files', 'text file.txt')
       end
 
       let :raw do
@@ -211,6 +211,12 @@ module Paths
           end
         end
       end
+
+      describe '#partial_files_visible?' do
+        it 'visible only after upload' do
+          refute root_path.partial_files_visible?
+        end
+      end
     end
   end
 end

data/test/paths/sftp_test.rb

@@ -17,7 +17,7 @@ module Paths
 
       let(:url) { File.join("sftp://", host_name, ftp_dir) }
 
-      let(:file_name) { File.join(File.dirname(__FILE__), '..', 'files', 'text.txt') }
+      let(:file_name) { File.join(File.dirname(__FILE__), '..', 'files', 'text file.txt') }
       let(:raw) { File.read(file_name) }
 
       let(:root_path) { IOStreams::Paths::SFTP.new(url, username: username, password: password) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta6
+  version: 1.0.0.beta7
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-06 00:00:00.000000000 Z
+date: 2019-12-09 00:00:00.000000000 Z
 dependencies: []
 description: 
 email:
@@ -76,6 +76,7 @@ files:
 - test/files/spreadsheet.xlsx
 - test/files/test.csv
 - test/files/test.json
+- test/files/text file.txt
 - test/files/text.txt
 - test/files/text.txt.bz2
 - test/files/text.txt.gz
@@ -169,6 +170,7 @@ test_files:
 - test/files/text.txt.gz
 - test/files/text.txt
 - test/files/multiple_files.zip
+- test/files/text file.txt
 - test/gzip_reader_test.rb
 - test/encode_reader_test.rb
 - test/test_helper.rb