iostreams 1.0.0.beta5 → 1.0.0.beta6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/io_streams/paths/http.rb +5 -0
- data/lib/io_streams/paths/s3.rb +20 -3
- data/lib/io_streams/paths/sftp.rb +7 -2
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/zip/reader.rb +14 -18
- data/lib/io_streams/zip/writer.rb +10 -40
- data/test/paths/sftp_test.rb +3 -2
- data/test/streams_test.rb +4 -4
- data/test/zip_writer_test.rb +2 -11
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 167014d27ff57416cce79209be3025489a6312e208f3edccebb17f535dae9ae1
|
4
|
+
data.tar.gz: 55e1f44b9613010dc81f02bb03d92ee9ec32c2b9d49c4ec99657a9c072700fb7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7d82649bff9f6ff9c97e58caa499c0cf5ce5090553816a76155374d6123b91fb673426c4eddfed0e5da24fff89e77997e7064b87ec95bf7aac0f1bac7b6f9dd0
|
7
|
+
data.tar.gz: 97d1fa3c4e5771d4f7ad3e9d3b3580f5b4cadbc5d57838277e7e7fe21224d496b5978e788f54981143d579a39131b907c9b069881e8d2b200356d49b4b2362f8
|
data/lib/io_streams/paths/s3.rb
CHANGED
@@ -15,6 +15,12 @@ module IOStreams
|
|
15
15
|
# s3://my-bucket-name/file_name.txt
|
16
16
|
# s3://my-bucket-name/some_path/file_name.csv
|
17
17
|
#
|
18
|
+
# access_key_id: [String]
|
19
|
+
# AWS Access Key Id to use to access this bucket.
|
20
|
+
#
|
21
|
+
# secret_access_key: [String]
|
22
|
+
# AWS Secret Access Key Id to use to access this bucket.
|
23
|
+
#
|
18
24
|
# Writer specific options:
|
19
25
|
#
|
20
26
|
# @option params [String] :acl
|
@@ -124,7 +130,7 @@ module IOStreams
|
|
124
130
|
#
|
125
131
|
# @option params [String] :object_lock_legal_hold_status
|
126
132
|
# The Legal Hold status that you want to apply to the specified object.
|
127
|
-
def initialize(url, client: nil, **args)
|
133
|
+
def initialize(url, client: nil, access_key_id: nil, secret_access_key: nil, **args)
|
128
134
|
Utils.load_soft_dependency('aws-sdk-s3', 'AWS S3') unless defined?(::Aws::S3::Client)
|
129
135
|
|
130
136
|
uri = URI.parse(url)
|
@@ -132,8 +138,14 @@ module IOStreams
|
|
132
138
|
|
133
139
|
@bucket_name = uri.host
|
134
140
|
key = uri.path.sub(%r{\A/}, '')
|
135
|
-
|
136
|
-
|
141
|
+
if client.is_a?(Hash)
|
142
|
+
client[:access_key_id] = access_key_id if access_key_id
|
143
|
+
client[:secret_access_key] = secret_access_key if secret_access_key
|
144
|
+
@client = ::Aws::S3::Client.new(client)
|
145
|
+
else
|
146
|
+
@client = client || ::Aws::S3::Client.new(access_key_id: access_key_id, secret_access_key: secret_access_key)
|
147
|
+
end
|
148
|
+
@options = args
|
137
149
|
|
138
150
|
URI.decode_www_form(uri.query).each { |key, value| @options[key] = value } if uri.query
|
139
151
|
|
@@ -144,6 +156,11 @@ module IOStreams
|
|
144
156
|
::File.join("s3://", bucket_name, path)
|
145
157
|
end
|
146
158
|
|
159
|
+
# Does not support relative file names since there is no concept of current working directory
|
160
|
+
def relative?
|
161
|
+
false
|
162
|
+
end
|
163
|
+
|
147
164
|
def delete
|
148
165
|
client.delete_object(bucket: bucket_name, key: path)
|
149
166
|
self
|
@@ -73,6 +73,11 @@ module IOStreams
|
|
73
73
|
super(uri.path)
|
74
74
|
end
|
75
75
|
|
76
|
+
# Does not support relative file names since there is no concept of current working directory
|
77
|
+
def relative?
|
78
|
+
false
|
79
|
+
end
|
80
|
+
|
76
81
|
def to_s
|
77
82
|
url
|
78
83
|
end
|
@@ -204,10 +209,10 @@ module IOStreams
|
|
204
209
|
options = ssh_options.dup
|
205
210
|
key = options.delete('IdentityKey')
|
206
211
|
# sftp requires that private key is only readable by the current user
|
207
|
-
File.open(file_name, 'wb', 0600) { |io| io.write(key) }
|
212
|
+
::File.open(file_name, 'wb', 0600) { |io| io.write(key) }
|
208
213
|
|
209
214
|
options['IdentityFile'] = file_name
|
210
|
-
yield sftp_args(
|
215
|
+
yield sftp_args(options)
|
211
216
|
end
|
212
217
|
end
|
213
218
|
|
data/lib/io_streams/version.rb
CHANGED
@@ -36,28 +36,24 @@ module IOStreams
|
|
36
36
|
# Read from a zip file or stream, decompressing the contents as it is read
|
37
37
|
# The input stream from the first file found in the zip file is passed
|
38
38
|
# to the supplied block
|
39
|
-
def self.file(file_name, entry_file_name: nil)
|
40
|
-
Utils.load_soft_dependency('rubyzip', 'Zip', 'zip') unless defined?(::Zip)
|
39
|
+
def self.file(file_name, entry_file_name: nil, &block)
|
40
|
+
Utils.load_soft_dependency('rubyzip', 'Read Zip', 'zip') unless defined?(::Zip)
|
41
41
|
|
42
|
-
::Zip::
|
43
|
-
|
44
|
-
|
45
|
-
|
42
|
+
::Zip::File.open(file_name) do |zip_file|
|
43
|
+
if entry_file_name
|
44
|
+
zip_file.get_input_stream(entry_file_name, &block)
|
45
|
+
else
|
46
|
+
result = nil
|
47
|
+
# Return the first file
|
48
|
+
zip_file.each do |entry|
|
49
|
+
result = entry.get_input_stream(&block)
|
50
|
+
break
|
51
|
+
end
|
52
|
+
result
|
53
|
+
end
|
46
54
|
end
|
47
55
|
end
|
48
56
|
end
|
49
|
-
|
50
|
-
def self.get_entry(zin, entry_file_name)
|
51
|
-
if entry_file_name.nil?
|
52
|
-
zin.get_next_entry
|
53
|
-
return true
|
54
|
-
end
|
55
|
-
|
56
|
-
while entry = zin.get_next_entry
|
57
|
-
return true if entry.name == entry_file_name
|
58
|
-
end
|
59
|
-
false
|
60
|
-
end
|
61
57
|
end
|
62
58
|
end
|
63
59
|
end
|
@@ -1,59 +1,29 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Zip
|
3
3
|
class Writer < IOStreams::Writer
|
4
|
-
# Write a single file in Zip format to the supplied output
|
4
|
+
# Write a single file in Zip format to the supplied output stream
|
5
5
|
#
|
6
6
|
# Parameters
|
7
|
-
#
|
8
|
-
#
|
7
|
+
# output_stream [IO]
|
8
|
+
# Output stream to write to
|
9
|
+
#
|
10
|
+
# original_file_name [String]
|
11
|
+
# Since this is a stream the original file name is used to create the entry_file_name if not supplied
|
9
12
|
#
|
10
13
|
# entry_file_name: [String]
|
11
14
|
# Name of the file entry within the Zip file.
|
12
15
|
#
|
13
16
|
# The stream supplied to the block only responds to #write
|
14
|
-
|
15
|
-
# Example:
|
16
|
-
# IOStreams::ZipWriter.open('myfile.zip', zip_file_name: 'myfile.txt') do |io_stream|
|
17
|
-
# io_stream.write("hello world\n")
|
18
|
-
# io_stream.write("and more\n")
|
19
|
-
# end
|
20
|
-
#
|
21
|
-
# Notes:
|
22
|
-
# - Since Zip cannot write to streams, if a stream is supplied, a temp file
|
23
|
-
# is automatically created under the covers
|
24
|
-
def self.file(file_name, original_file_name: file_name, zip_file_name: nil, entry_file_name: zip_file_name, &block)
|
17
|
+
def self.stream(output_stream, original_file_name: nil, zip_file_name: nil, entry_file_name: zip_file_name, &block)
|
25
18
|
# Default the name of the file within the zip to the supplied file_name without the zip extension
|
26
|
-
if entry_file_name.nil? && (original_file_name =~ /\.(zip)\z/i)
|
19
|
+
if entry_file_name.nil? && original_file_name && (original_file_name =~ /\.(zip)\z/i)
|
27
20
|
entry_file_name = original_file_name.to_s[0..-5]
|
28
21
|
end
|
29
22
|
entry_file_name ||= 'file'
|
30
23
|
|
31
|
-
|
32
|
-
end
|
33
|
-
|
34
|
-
private
|
35
|
-
|
36
|
-
if defined?(JRuby)
|
37
|
-
def self.write_file(file_name, entry_file_name)
|
38
|
-
out = Java::JavaIo::FileOutputStream.new(file_name)
|
39
|
-
zout = Java::JavaUtilZip::ZipOutputStream.new(out)
|
40
|
-
zout.put_next_entry(Java::JavaUtilZip::ZipEntry.new(entry_file_name))
|
41
|
-
io = zout.to_io
|
42
|
-
yield(io)
|
43
|
-
ensure
|
44
|
-
io&.close
|
45
|
-
out&.close
|
46
|
-
end
|
47
|
-
else
|
48
|
-
def self.write_file(file_name, entry_file_name)
|
49
|
-
Utils.load_soft_dependency('rubyzip', 'Zip', 'zip') unless defined?(::Zip)
|
24
|
+
Utils.load_soft_dependency('zip_tricks', 'Zip') unless defined?(ZipTricks::Streamer)
|
50
25
|
|
51
|
-
|
52
|
-
zos.put_next_entry(entry_file_name)
|
53
|
-
yield(zos)
|
54
|
-
ensure
|
55
|
-
zos&.close
|
56
|
-
end
|
26
|
+
ZipTricks::Streamer.open(output_stream) { |zip| zip.write_deflated_file(entry_file_name, &block) }
|
57
27
|
end
|
58
28
|
end
|
59
29
|
end
|
data/test/paths/sftp_test.rb
CHANGED
@@ -13,6 +13,7 @@ module Paths
|
|
13
13
|
let(:username) { ENV["SFTP_USERNAME"] }
|
14
14
|
let(:password) { ENV["SFTP_PASSWORD"] }
|
15
15
|
let(:ftp_dir) { ENV["SFTP_DIR"] || "iostreams_test" }
|
16
|
+
let(:identity_username) { ENV["SFTP_IDENTITY_USERNAME"] || username }
|
16
17
|
|
17
18
|
let(:url) { File.join("sftp://", host_name, ftp_dir) }
|
18
19
|
|
@@ -71,7 +72,7 @@ module Paths
|
|
71
72
|
|
72
73
|
describe 'use identity file instead of password' do
|
73
74
|
let :root_path do
|
74
|
-
IOStreams::Paths::SFTP.new(url, username:
|
75
|
+
IOStreams::Paths::SFTP.new(url, username: identity_username, ssh_options: {'IdentityFile' => ENV["SFTP_IDENTITY_FILE"]})
|
75
76
|
end
|
76
77
|
|
77
78
|
it 'writes' do
|
@@ -84,7 +85,7 @@ module Paths
|
|
84
85
|
describe 'use identity key instead of password' do
|
85
86
|
let :root_path do
|
86
87
|
key = File.open(ENV["SFTP_IDENTITY_FILE"], 'rb', &:read)
|
87
|
-
IOStreams::Paths::SFTP.new(url, username:
|
88
|
+
IOStreams::Paths::SFTP.new(url, username: identity_username, ssh_options: {'IdentityKey' => key})
|
88
89
|
end
|
89
90
|
|
90
91
|
it 'writes' do
|
data/test/streams_test.rb
CHANGED
@@ -81,16 +81,16 @@ class StreamsTest < Minitest::Test
|
|
81
81
|
|
82
82
|
it 'returns the reader' do
|
83
83
|
string_io = StringIO.new
|
84
|
-
streams.stream(:
|
84
|
+
streams.stream(:bz2)
|
85
85
|
streams.reader(string_io) do |io|
|
86
|
-
assert io.is_a?(::
|
86
|
+
assert io.is_a?(RBzip2::FFI::Decompressor), io
|
87
87
|
end
|
88
88
|
end
|
89
89
|
|
90
90
|
it 'returns the last reader' do
|
91
91
|
string_io = StringIO.new
|
92
92
|
streams.stream(:encode)
|
93
|
-
streams.stream(:
|
93
|
+
streams.stream(:bz2)
|
94
94
|
streams.reader(string_io) do |io|
|
95
95
|
assert io.is_a?(IOStreams::Encode::Reader), io
|
96
96
|
end
|
@@ -113,7 +113,7 @@ class StreamsTest < Minitest::Test
|
|
113
113
|
string_io = StringIO.new
|
114
114
|
streams.stream(:zip)
|
115
115
|
streams.writer(string_io) do |io|
|
116
|
-
assert io.is_a?(::
|
116
|
+
assert io.is_a?(ZipTricks::Streamer::Writable), io
|
117
117
|
end
|
118
118
|
end
|
119
119
|
|
data/test/zip_writer_test.rb
CHANGED
@@ -24,9 +24,7 @@ class ZipWriterTest < Minitest::Test
|
|
24
24
|
IOStreams::Zip::Writer.file(file_name, entry_file_name: 'text.txt') do |io|
|
25
25
|
io.write(decompressed)
|
26
26
|
end
|
27
|
-
result = ::Zip::
|
28
|
-
zip_file.first.get_input_stream.read
|
29
|
-
end
|
27
|
+
result = IOStreams::Zip::Reader.file(file_name, &:read)
|
30
28
|
assert_equal decompressed, result
|
31
29
|
end
|
32
30
|
|
@@ -36,14 +34,7 @@ class ZipWriterTest < Minitest::Test
|
|
36
34
|
io.write(decompressed)
|
37
35
|
end
|
38
36
|
io = StringIO.new(io_string.string)
|
39
|
-
result =
|
40
|
-
begin
|
41
|
-
zin = ::Zip::InputStream.new(io)
|
42
|
-
zin.get_next_entry
|
43
|
-
result = zin.read
|
44
|
-
ensure
|
45
|
-
zin.close if zin
|
46
|
-
end
|
37
|
+
result = IOStreams::Zip::Reader.stream(io, &:read)
|
47
38
|
assert_equal decompressed, result
|
48
39
|
end
|
49
40
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: iostreams
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.beta6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-12-06 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email:
|