iostreams 1.0.0.beta3 → 1.0.0.beta4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/io_streams/paths/s3.rb +4 -1
- data/lib/io_streams/paths/sftp.rb +91 -33
- data/lib/io_streams/pgp.rb +1 -1
- data/lib/io_streams/version.rb +1 -1
- data/test/paths/s3_test.rb +10 -1
- data/test/paths/sftp_test.rb +31 -10
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1f3b07c0b0634bbc94434c4fb9de6fe4a41de44c9a8a069e8e53922a80a1e560
|
4
|
+
data.tar.gz: d4d7cd4fbb3fedcdf8b6df763d8eace74a075c389895231171e5b41d5fefe975
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2beb8a498df91a5b564858a1f7deb5bcf139a138ceb2373a8d42a0d7c0a60f913d7675a179e558136272eacb2b25d0194aee7cb482732440bda5da6502123fe7
|
7
|
+
data.tar.gz: a6891d61d2910ad1787213c5b0e8c1e5de3f9147c6ab43b267910878ecba8b97fbe753e643ceeaa369f7969b584780dc25a1137e0ba844e01d46fad1b6b97882
|
data/lib/io_streams/paths/s3.rb
CHANGED
@@ -134,6 +134,9 @@ module IOStreams
|
|
134
134
|
key = uri.path.sub(%r{\A/}, '')
|
135
135
|
@client = client || ::Aws::S3::Client.new
|
136
136
|
@options = args
|
137
|
+
|
138
|
+
URI.decode_www_form(uri.query).each { |key, value| @options[key] = value } if uri.query
|
139
|
+
|
137
140
|
super(key)
|
138
141
|
end
|
139
142
|
|
@@ -193,7 +196,7 @@ module IOStreams
|
|
193
196
|
Utils.temp_file_name("iostreams_s3") do |file_name|
|
194
197
|
read_file(file_name)
|
195
198
|
|
196
|
-
::File.open(file_name, 'rb')
|
199
|
+
::File.open(file_name, 'rb', &block)
|
197
200
|
end
|
198
201
|
end
|
199
202
|
|
@@ -13,7 +13,7 @@ module IOStreams
|
|
13
13
|
@sshpass_bin = 'sshpass'
|
14
14
|
@sshpass_wait_seconds = 5
|
15
15
|
|
16
|
-
attr_reader :hostname, :username, :ssh_options, :url
|
16
|
+
attr_reader :hostname, :username, :ssh_options, :url, :port
|
17
17
|
|
18
18
|
# Stream to a remote file over sftp.
|
19
19
|
#
|
@@ -58,13 +58,17 @@ module IOStreams
|
|
58
58
|
uri = URI.parse(url)
|
59
59
|
raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
|
60
60
|
|
61
|
-
@hostname
|
62
|
-
@mkdir
|
63
|
-
@username
|
64
|
-
@url
|
65
|
-
@password
|
66
|
-
@port
|
67
|
-
|
61
|
+
@hostname = uri.hostname
|
62
|
+
@mkdir = false
|
63
|
+
@username = username || uri.user
|
64
|
+
@url = url
|
65
|
+
@password = password || uri.password
|
66
|
+
@port = uri.port || 22
|
67
|
+
# Not Ruby 2.5 yet: transform_keys(&:to_s)
|
68
|
+
@ssh_options = {}
|
69
|
+
ssh_options.each_pair { |key, value| @ssh_options[key.to_s] = value }
|
70
|
+
|
71
|
+
URI.decode_www_form(uri.query).each { |key, value| @ssh_options[key] = value } if uri.query
|
68
72
|
|
69
73
|
super(uri.path)
|
70
74
|
end
|
@@ -152,37 +156,78 @@ module IOStreams
|
|
152
156
|
|
153
157
|
# Use sftp and sshpass executables to download to a local file
|
154
158
|
def sftp_download(remote_file_name, local_file_name)
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
159
|
+
with_sftp_args do |args|
|
160
|
+
Open3.popen2e(*args) do |writer, reader, waith_thr|
|
161
|
+
begin
|
162
|
+
writer.puts password
|
163
|
+
# Give time for password to be processed and stdin to be passed to sftp process.
|
164
|
+
sleep self.class.sshpass_wait_seconds
|
165
|
+
writer.puts "get #{remote_file_name} #{local_file_name}"
|
166
|
+
writer.puts 'bye'
|
167
|
+
writer.close
|
168
|
+
out = reader.read.chomp
|
169
|
+
raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
|
170
|
+
out
|
171
|
+
rescue Errno::EPIPE
|
172
|
+
out = reader.read.chomp rescue nil
|
173
|
+
raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
|
174
|
+
end
|
175
|
+
end
|
165
176
|
end
|
166
177
|
end
|
167
178
|
|
168
179
|
def sftp_upload(local_file_name, remote_file_name)
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
180
|
+
with_sftp_args do |args|
|
181
|
+
Open3.popen2e(*args) do |writer, reader, waith_thr|
|
182
|
+
begin
|
183
|
+
writer.puts(password) if password
|
184
|
+
# Give time for password to be processed and stdin to be passed to sftp process.
|
185
|
+
sleep self.class.sshpass_wait_seconds
|
186
|
+
writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
|
187
|
+
writer.puts 'bye'
|
188
|
+
writer.close
|
189
|
+
out = reader.read.chomp
|
190
|
+
raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
|
191
|
+
out
|
192
|
+
rescue Errno::EPIPE
|
193
|
+
out = reader.read.chomp rescue nil
|
194
|
+
raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
|
195
|
+
end
|
196
|
+
end
|
179
197
|
end
|
180
198
|
end
|
181
199
|
|
182
|
-
def
|
183
|
-
|
184
|
-
|
185
|
-
args
|
200
|
+
def with_sftp_args
|
201
|
+
return yield sftp_args(ssh_options) unless ssh_options.key?('IdentityKey')
|
202
|
+
|
203
|
+
Utils.temp_file_name('iostreams-sftp-args', 'key') do |file_name|
|
204
|
+
options = ssh_options.dup
|
205
|
+
key = options.delete('IdentityKey')
|
206
|
+
# sftp requires that private key is only readable by the current user
|
207
|
+
File.open(file_name, 'wb', 0600) { |io| io.write(key) }
|
208
|
+
|
209
|
+
options['IdentityFile'] = file_name
|
210
|
+
yield sftp_args(ssh_options)
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
def sftp_args(ssh_options)
|
215
|
+
args = [self.class.sshpass_bin, self.class.sftp_bin]
|
216
|
+
# Force sftp to use the password when supplied,
|
217
|
+
# and stop sftp from prompting for a password when none was supplied.
|
218
|
+
if password
|
219
|
+
args << "-oBatchMode=no"
|
220
|
+
args << "-oNumberOfPasswordPrompts=1"
|
221
|
+
args << "-oPubkeyAuthentication=no"
|
222
|
+
else
|
223
|
+
args << "-oBatchMode=yes"
|
224
|
+
args << "-oPasswordAuthentication=no"
|
225
|
+
end
|
226
|
+
args << "-oIdentitiesOnly=yes" if ssh_options.key?('IdentityFile')
|
227
|
+
# Default is ask, but this is non-interactive so make the default fail without asking.
|
228
|
+
args << "-oStrictHostKeyChecking=yes" unless ssh_options.key?('StrictHostKeyChecking')
|
229
|
+
args << "-oLogLevel=#{map_log_level}" unless ssh_options.key?('LogLevel')
|
230
|
+
args << "-oPort=#{port}" unless port == 22
|
186
231
|
ssh_options.each_pair { |key, value| args << "-o#{key}=#{value}" }
|
187
232
|
args << '-b'
|
188
233
|
args << '-'
|
@@ -193,11 +238,24 @@ module IOStreams
|
|
193
238
|
def build_ssh_options
|
194
239
|
options = ssh_options.dup
|
195
240
|
options[:logger] ||= self.logger if defined?(SemanticLogger)
|
196
|
-
options[:port] ||=
|
241
|
+
options[:port] ||= port
|
197
242
|
options[:max_pkt_size] ||= 65_536
|
198
243
|
options[:password] ||= @password
|
199
244
|
options
|
200
245
|
end
|
246
|
+
|
247
|
+
def map_log_level
|
248
|
+
return "INFO" unless defined?(SemanticLogger)
|
249
|
+
|
250
|
+
case logger.level
|
251
|
+
when :trace
|
252
|
+
"DEBUG3"
|
253
|
+
when :warn
|
254
|
+
"ERROR"
|
255
|
+
else
|
256
|
+
logger.level.to_s
|
257
|
+
end
|
258
|
+
end
|
201
259
|
end
|
202
260
|
end
|
203
261
|
end
|
data/lib/io_streams/pgp.rb
CHANGED
@@ -275,7 +275,7 @@ module IOStreams
|
|
275
275
|
# * Invalidated keys must be removed manually.
|
276
276
|
def self.import(key:)
|
277
277
|
version_check
|
278
|
-
command = "#{executable} --import"
|
278
|
+
command = "#{executable} --batch --import"
|
279
279
|
|
280
280
|
out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
|
281
281
|
logger&.debug { "IOStreams::Pgp.import: #{command}\n#{err}#{out}" }
|
data/lib/io_streams/version.rb
CHANGED
data/test/paths/s3_test.rb
CHANGED
@@ -97,6 +97,14 @@ module Paths
|
|
97
97
|
end
|
98
98
|
end
|
99
99
|
|
100
|
+
describe '#each_line' do
|
101
|
+
it 'reads line by line' do
|
102
|
+
lines = []
|
103
|
+
existing_path.each_line {|line| lines << line}
|
104
|
+
assert_equal raw.lines.collect(&:chomp), lines
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
100
108
|
describe '#each_child' do
|
101
109
|
# TODO: case_sensitive: false, directories: false, hidden: false
|
102
110
|
let(:abd_file_names) { %w[abd/test1.txt abd/test5.file abd/extra/file.csv] }
|
@@ -116,7 +124,8 @@ module Paths
|
|
116
124
|
|
117
125
|
it 'existing file returns just the file itself' do
|
118
126
|
# Glorified exists call
|
119
|
-
|
127
|
+
existing_path
|
128
|
+
assert_equal root_path.join("test.txt").to_s, root_path.children("test.txt").first.to_s
|
120
129
|
end
|
121
130
|
|
122
131
|
it 'missing file does nothing' do
|
data/test/paths/sftp_test.rb
CHANGED
@@ -5,25 +5,21 @@ module Paths
|
|
5
5
|
describe IOStreams::Paths::SFTP do
|
6
6
|
before do
|
7
7
|
unless ENV["SFTP_HOSTNAME"]
|
8
|
-
skip "Supply environment variables to test SFTP paths: SFTP_HOSTNAME, SFTP_USERNAME, SFTP_PASSWORD, and optional SFTP_DIR"
|
8
|
+
skip "Supply environment variables to test SFTP paths: SFTP_HOSTNAME, SFTP_USERNAME, SFTP_PASSWORD, and optional SFTP_DIR, SFTP_IDENTITY_FILE"
|
9
9
|
end
|
10
10
|
end
|
11
11
|
|
12
12
|
let(:host_name) { ENV["SFTP_HOSTNAME"] }
|
13
13
|
let(:username) { ENV["SFTP_USERNAME"] }
|
14
14
|
let(:password) { ENV["SFTP_PASSWORD"] }
|
15
|
-
let(:ftp_dir) { ENV["SFTP_DIR"] || "iostreams_test"}
|
16
|
-
let(:url) { File.join("sftp://", host_name, ftp_dir) }
|
15
|
+
let(:ftp_dir) { ENV["SFTP_DIR"] || "iostreams_test" }
|
17
16
|
|
18
|
-
let
|
19
|
-
File.join(File.dirname(__FILE__), '..', 'files', 'text.txt')
|
20
|
-
end
|
17
|
+
let(:url) { File.join("sftp://", host_name, ftp_dir) }
|
21
18
|
|
22
|
-
let
|
23
|
-
|
24
|
-
end
|
19
|
+
let(:file_name) { File.join(File.dirname(__FILE__), '..', 'files', 'text.txt') }
|
20
|
+
let(:raw) { File.read(file_name) }
|
25
21
|
|
26
|
-
let(:root_path) { IOStreams::Paths::SFTP.new(url, username: username, password: password
|
22
|
+
let(:root_path) { IOStreams::Paths::SFTP.new(url, username: username, password: password) }
|
27
23
|
|
28
24
|
let :existing_path do
|
29
25
|
path = root_path.join('test.txt')
|
@@ -72,6 +68,31 @@ module Paths
|
|
72
68
|
missing_path.write("Bad path")
|
73
69
|
end
|
74
70
|
end
|
71
|
+
|
72
|
+
describe 'use identity file instead of password' do
|
73
|
+
let :root_path do
|
74
|
+
IOStreams::Paths::SFTP.new(url, username: username, ssh_options: {'IdentityFile' => ENV["SFTP_IDENTITY_FILE"]} )
|
75
|
+
end
|
76
|
+
|
77
|
+
it 'writes' do
|
78
|
+
skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
|
79
|
+
assert_equal raw.size, write_path.writer { |io| io.write(raw) }
|
80
|
+
assert_equal raw, write_path.read
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
describe 'use identity key instead of password' do
|
85
|
+
let :root_path do
|
86
|
+
key = File.open(ENV["SFTP_IDENTITY_FILE"], 'rb', &:read)
|
87
|
+
IOStreams::Paths::SFTP.new(url, username: username, ssh_options: {'IdentityKey' => key})
|
88
|
+
end
|
89
|
+
|
90
|
+
it 'writes' do
|
91
|
+
skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
|
92
|
+
assert_equal raw.size, write_path.writer { |io| io.write(raw) }
|
93
|
+
assert_equal raw, write_path.read
|
94
|
+
end
|
95
|
+
end
|
75
96
|
end
|
76
97
|
end
|
77
98
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: iostreams
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.beta4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-11-
|
11
|
+
date: 2019-11-19 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email:
|