iostreams 1.0.0.beta6 → 1.0.0.beta7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/io_streams/io_streams.rb +1 -1
- data/lib/io_streams/path.rb +8 -0
- data/lib/io_streams/paths/s3.rb +12 -7
- data/lib/io_streams/paths/sftp.rb +2 -3
- data/lib/io_streams/stream.rb +5 -0
- data/lib/io_streams/utils.rb +19 -0
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/zip/reader.rb +11 -0
- data/test/files/text file.txt +3 -0
- data/test/paths/s3_test.rb +7 -1
- data/test/paths/sftp_test.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4d8a7eaa8009e55339490c405a111986838219094c3df4d0b2303f7e1d16e951
|
4
|
+
data.tar.gz: 3a0635cb14723f81054674759f6c0d1752196a5d8d5c6c69dc96d9910fa63b84
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e61d3ed29557eb7abd74737c6dd2b81ecfb2667c0a1f753a4a66fd746816f7083ea7dfc7fd8c86339f0dc671418692ccdef0948875b522d39e16a262c820c188
|
7
|
+
data.tar.gz: 5d6b8cfc44627bd41b5170cc5e0c55ad10089d624bab6409ef864738c114fa614a9d92895c3e29eed20b88ff6627a06834e4aa722fba369fd11018efb38a5a4d
|
@@ -52,7 +52,7 @@ module IOStreams
|
|
52
52
|
|
53
53
|
elements = elements.collect(&:to_s)
|
54
54
|
path = ::File.join(*elements)
|
55
|
-
extracted_scheme = path.include?("://") ? URI.
|
55
|
+
extracted_scheme = path.include?("://") ? Utils::URI.new(path).scheme : nil
|
56
56
|
klass = scheme(extracted_scheme)
|
57
57
|
args.empty? ? klass.new(path) : klass.new(path, **args)
|
58
58
|
end
|
data/lib/io_streams/path.rb
CHANGED
@@ -160,6 +160,14 @@ module IOStreams
|
|
160
160
|
!(path =~ /\.(enc|pgp|gpg)\z/i).nil?
|
161
161
|
end
|
162
162
|
|
163
|
+
# Returns [true|false] whether partially created files are visible on this path.
|
164
|
+
#
|
165
|
+
# With local file systems a file that is still being written to is visbile.
|
166
|
+
# On AWS S3 a file is not visible until it is completely written to the bucket.
|
167
|
+
def partial_files_visible?
|
168
|
+
true
|
169
|
+
end
|
170
|
+
|
163
171
|
# TODO: Other possible methods:
|
164
172
|
# - rename - File.rename
|
165
173
|
# - rmtree - delete everything under this path - FileUtils.rm_r
|
data/lib/io_streams/paths/s3.rb
CHANGED
@@ -133,10 +133,10 @@ module IOStreams
|
|
133
133
|
def initialize(url, client: nil, access_key_id: nil, secret_access_key: nil, **args)
|
134
134
|
Utils.load_soft_dependency('aws-sdk-s3', 'AWS S3') unless defined?(::Aws::S3::Client)
|
135
135
|
|
136
|
-
uri = URI.
|
136
|
+
uri = Utils::URI.new(url)
|
137
137
|
raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == 's3'
|
138
138
|
|
139
|
-
@bucket_name = uri.
|
139
|
+
@bucket_name = uri.hostname
|
140
140
|
key = uri.path.sub(%r{\A/}, '')
|
141
141
|
if client.is_a?(Hash)
|
142
142
|
client[:access_key_id] = access_key_id if access_key_id
|
@@ -147,7 +147,7 @@ module IOStreams
|
|
147
147
|
end
|
148
148
|
@options = args
|
149
149
|
|
150
|
-
|
150
|
+
@options.merge(uri.query) if uri.query
|
151
151
|
|
152
152
|
super(key)
|
153
153
|
end
|
@@ -259,8 +259,6 @@ module IOStreams
|
|
259
259
|
# Notes:
|
260
260
|
# - Currently all S3 lookups are recursive as of the pattern regardless of whether the pattern includes `**`.
|
261
261
|
def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
|
262
|
-
raise(NotImplementedError, "AWS S3 #each_child does not yet return directories") if directories
|
263
|
-
|
264
262
|
matcher = Matcher.new(self, pattern, case_sensitive: case_sensitive, hidden: hidden)
|
265
263
|
|
266
264
|
# When the pattern includes an exact file name without any pattern characters
|
@@ -269,22 +267,29 @@ module IOStreams
|
|
269
267
|
return
|
270
268
|
end
|
271
269
|
|
272
|
-
prefix = URI.
|
270
|
+
prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/}, '')
|
273
271
|
token = nil
|
274
272
|
loop do
|
275
273
|
# Fetches upto 1,000 entries at a time
|
276
274
|
resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
|
277
275
|
resp.contents.each do |object|
|
276
|
+
next if !directories && object.key.end_with?("/")
|
277
|
+
|
278
278
|
file_name = ::File.join("s3://", resp.name, object.key)
|
279
279
|
next unless matcher.match?(file_name)
|
280
280
|
|
281
|
-
yield
|
281
|
+
yield(self.class.new(file_name), object.to_h)
|
282
282
|
end
|
283
283
|
token = resp.next_continuation_token
|
284
284
|
break if token.nil?
|
285
285
|
end
|
286
286
|
nil
|
287
287
|
end
|
288
|
+
|
289
|
+
# On S3 only files that are completely saved are visible.
|
290
|
+
def partial_files_visible?
|
291
|
+
false
|
292
|
+
end
|
288
293
|
end
|
289
294
|
end
|
290
295
|
end
|
@@ -55,7 +55,7 @@ module IOStreams
|
|
55
55
|
# puts io.read
|
56
56
|
# end
|
57
57
|
def initialize(url, username: nil, password: nil, ruby: true, ssh_options: {})
|
58
|
-
uri = URI.
|
58
|
+
uri = Utils::URI.new(url)
|
59
59
|
raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
|
60
60
|
|
61
61
|
@hostname = uri.hostname
|
@@ -67,8 +67,7 @@ module IOStreams
|
|
67
67
|
# Not Ruby 2.5 yet: transform_keys(&:to_s)
|
68
68
|
@ssh_options = {}
|
69
69
|
ssh_options.each_pair { |key, value| @ssh_options[key.to_s] = value }
|
70
|
-
|
71
|
-
URI.decode_www_form(uri.query).each { |key, value| @ssh_options[key] = value } if uri.query
|
70
|
+
@ssh_options.merge(uri.query) if uri.query
|
72
71
|
|
73
72
|
super(uri.path)
|
74
73
|
end
|
data/lib/io_streams/stream.rb
CHANGED
@@ -108,6 +108,11 @@ module IOStreams
|
|
108
108
|
end
|
109
109
|
end
|
110
110
|
|
111
|
+
def copy_to(target, convert: true)
|
112
|
+
target = IOStreams.path(target) unless target.is_a?(Stream)
|
113
|
+
target.copy_from(self, convert: convert)
|
114
|
+
end
|
115
|
+
|
111
116
|
# Iterate over a file / stream returning one line at a time.
|
112
117
|
# Embedded lines (within double quotes) will be skipped if
|
113
118
|
# 1. The file name contains .csv
|
data/lib/io_streams/utils.rb
CHANGED
@@ -32,5 +32,24 @@ module IOStreams
|
|
32
32
|
end
|
33
33
|
result
|
34
34
|
end
|
35
|
+
|
36
|
+
class URI
|
37
|
+
attr_reader :scheme, :hostname, :path, :user, :password, :port, :query
|
38
|
+
|
39
|
+
def initialize(url)
|
40
|
+
url = url.gsub(' ', '%20')
|
41
|
+
uri = ::URI.parse(url)
|
42
|
+
@scheme = uri.scheme
|
43
|
+
@hostname = uri.hostname
|
44
|
+
@path = CGI.unescape(uri.path)
|
45
|
+
@user = uri.user
|
46
|
+
@password = uri.password
|
47
|
+
@port = uri.port
|
48
|
+
if uri.query
|
49
|
+
@query = {}
|
50
|
+
::URI.decode_www_form(uri.query).each { |key, value| @query[key] = value }
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
35
54
|
end
|
36
55
|
end
|
data/lib/io_streams/version.rb
CHANGED
@@ -32,6 +32,17 @@ module IOStreams
|
|
32
32
|
fin&.close
|
33
33
|
end
|
34
34
|
|
35
|
+
def self.get_entry(zin, entry_file_name)
|
36
|
+
if entry_file_name.nil?
|
37
|
+
zin.get_next_entry
|
38
|
+
return true
|
39
|
+
end
|
40
|
+
|
41
|
+
while entry = zin.get_next_entry
|
42
|
+
return true if entry.name == entry_file_name
|
43
|
+
end
|
44
|
+
false
|
45
|
+
end
|
35
46
|
else
|
36
47
|
# Read from a zip file or stream, decompressing the contents as it is read
|
37
48
|
# The input stream from the first file found in the zip file is passed
|
data/test/paths/s3_test.rb
CHANGED
@@ -10,7 +10,7 @@ module Paths
|
|
10
10
|
end
|
11
11
|
|
12
12
|
let :file_name do
|
13
|
-
File.join(File.dirname(__FILE__), '..', 'files', 'text.txt')
|
13
|
+
File.join(File.dirname(__FILE__), '..', 'files', 'text file.txt')
|
14
14
|
end
|
15
15
|
|
16
16
|
let :raw do
|
@@ -211,6 +211,12 @@ module Paths
|
|
211
211
|
end
|
212
212
|
end
|
213
213
|
end
|
214
|
+
|
215
|
+
describe '#partial_files_visible?' do
|
216
|
+
it 'visible only after upload' do
|
217
|
+
refute root_path.partial_files_visible?
|
218
|
+
end
|
219
|
+
end
|
214
220
|
end
|
215
221
|
end
|
216
222
|
end
|
data/test/paths/sftp_test.rb
CHANGED
@@ -17,7 +17,7 @@ module Paths
|
|
17
17
|
|
18
18
|
let(:url) { File.join("sftp://", host_name, ftp_dir) }
|
19
19
|
|
20
|
-
let(:file_name) { File.join(File.dirname(__FILE__), '..', 'files', 'text.txt') }
|
20
|
+
let(:file_name) { File.join(File.dirname(__FILE__), '..', 'files', 'text file.txt') }
|
21
21
|
let(:raw) { File.read(file_name) }
|
22
22
|
|
23
23
|
let(:root_path) { IOStreams::Paths::SFTP.new(url, username: username, password: password) }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: iostreams
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.beta7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-12-
|
11
|
+
date: 2019-12-09 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email:
|
@@ -76,6 +76,7 @@ files:
|
|
76
76
|
- test/files/spreadsheet.xlsx
|
77
77
|
- test/files/test.csv
|
78
78
|
- test/files/test.json
|
79
|
+
- test/files/text file.txt
|
79
80
|
- test/files/text.txt
|
80
81
|
- test/files/text.txt.bz2
|
81
82
|
- test/files/text.txt.gz
|
@@ -169,6 +170,7 @@ test_files:
|
|
169
170
|
- test/files/text.txt.gz
|
170
171
|
- test/files/text.txt
|
171
172
|
- test/files/multiple_files.zip
|
173
|
+
- test/files/text file.txt
|
172
174
|
- test/gzip_reader_test.rb
|
173
175
|
- test/encode_reader_test.rb
|
174
176
|
- test/test_helper.rb
|