iostreams 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/Rakefile +7 -7
- data/lib/io_streams/builder.rb +4 -3
- data/lib/io_streams/bzip2/reader.rb +1 -1
- data/lib/io_streams/bzip2/writer.rb +1 -1
- data/lib/io_streams/deprecated.rb +2 -3
- data/lib/io_streams/encode/reader.rb +5 -8
- data/lib/io_streams/encode/writer.rb +1 -1
- data/lib/io_streams/io_streams.rb +5 -2
- data/lib/io_streams/line/reader.rb +2 -1
- data/lib/io_streams/path.rb +4 -4
- data/lib/io_streams/paths/http.rb +8 -10
- data/lib/io_streams/paths/matcher.rb +10 -11
- data/lib/io_streams/paths/s3.rb +6 -6
- data/lib/io_streams/paths/sftp.rb +38 -23
- data/lib/io_streams/pgp.rb +45 -35
- data/lib/io_streams/pgp/reader.rb +4 -6
- data/lib/io_streams/pgp/writer.rb +1 -2
- data/lib/io_streams/reader.rb +2 -2
- data/lib/io_streams/record/writer.rb +2 -4
- data/lib/io_streams/row/writer.rb +3 -5
- data/lib/io_streams/stream.rb +6 -6
- data/lib/io_streams/symmetric_encryption/reader.rb +1 -3
- data/lib/io_streams/symmetric_encryption/writer.rb +2 -6
- data/lib/io_streams/tabular.rb +12 -10
- data/lib/io_streams/tabular/header.rb +4 -4
- data/lib/io_streams/tabular/parser/array.rb +2 -4
- data/lib/io_streams/tabular/parser/csv.rb +3 -5
- data/lib/io_streams/tabular/parser/fixed.rb +4 -3
- data/lib/io_streams/tabular/parser/hash.rb +2 -4
- data/lib/io_streams/tabular/parser/json.rb +2 -4
- data/lib/io_streams/tabular/parser/psv.rb +5 -7
- data/lib/io_streams/tabular/utility/csv_row.rb +9 -17
- data/lib/io_streams/utils.rb +3 -3
- data/lib/io_streams/utils/reliable_http.rb +98 -0
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +1 -1
- data/lib/io_streams/xlsx/reader.rb +5 -5
- data/lib/io_streams/zip/reader.rb +1 -1
- data/lib/io_streams/zip/writer.rb +2 -2
- data/lib/iostreams.rb +34 -34
- data/test/builder_test.rb +74 -74
- data/test/bzip2_reader_test.rb +8 -13
- data/test/bzip2_writer_test.rb +8 -9
- data/test/deprecated_test.rb +25 -29
- data/test/encode_reader_test.rb +14 -18
- data/test/encode_writer_test.rb +29 -30
- data/test/gzip_reader_test.rb +8 -13
- data/test/gzip_writer_test.rb +10 -11
- data/test/io_streams_test.rb +35 -35
- data/test/line_reader_test.rb +35 -39
- data/test/line_writer_test.rb +8 -9
- data/test/minimal_file_reader.rb +1 -1
- data/test/path_test.rb +24 -24
- data/test/paths/file_test.rb +42 -42
- data/test/paths/http_test.rb +5 -5
- data/test/paths/matcher_test.rb +11 -12
- data/test/paths/s3_test.rb +44 -46
- data/test/paths/sftp_test.rb +18 -18
- data/test/pgp_reader_test.rb +13 -15
- data/test/pgp_test.rb +43 -44
- data/test/pgp_writer_test.rb +27 -28
- data/test/record_reader_test.rb +9 -10
- data/test/record_writer_test.rb +10 -11
- data/test/row_reader_test.rb +5 -6
- data/test/row_writer_test.rb +7 -8
- data/test/stream_test.rb +60 -62
- data/test/tabular_test.rb +111 -111
- data/test/test_helper.rb +22 -22
- data/test/utils_test.rb +7 -7
- data/test/xlsx_reader_test.rb +12 -12
- data/test/zip_reader_test.rb +14 -21
- data/test/zip_writer_test.rb +10 -10
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1a141ad8f92a6c1387ad487fc8cf545bcc010ebc862967ffce8f79f71b7ba2d6
|
4
|
+
data.tar.gz: 0210b8d23390ddff6fe31133c24b88b2beb7e5e0f54d6b9df962632dc369a9ec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 810573a43277573365e946205465a4a1cc87c65cf517450dc6f363f5bb7168e66a25cd2d891e346d09c4b96aa9166f369fd2bc9dc21ec2112d87bce270467ff2
|
7
|
+
data.tar.gz: 2304155b27a897f270263283bdf499855a81f80885b9103b6c116f135904a42d73bfb29705a8156f3bde973b7a899eba9b123073e7b140a2e502b2203869ad1f
|
data/README.md
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# iostreams
|
2
|
-
[](https://rubygems.org/gems/iostreams) [](https://travis-ci.org/rocketjob/iostreams) [](https://rubygems.org/gems/iostreams) [](http://opensource.org/licenses/Apache-2.0) ](https://rubygems.org/gems/iostreams) [](https://travis-ci.org/rocketjob/iostreams) [](https://rubygems.org/gems/iostreams) [](http://opensource.org/licenses/Apache-2.0)  [-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
|
3
3
|
|
4
4
|
Input and Output streaming for Ruby.
|
5
5
|
|
data/Rakefile
CHANGED
@@ -1,21 +1,21 @@
|
|
1
|
-
require
|
2
|
-
require_relative
|
1
|
+
require "rake/testtask"
|
2
|
+
require_relative "lib/io_streams/version"
|
3
3
|
|
4
4
|
task :gem do
|
5
|
-
system
|
5
|
+
system "gem build iostreams.gemspec"
|
6
6
|
end
|
7
7
|
|
8
|
-
task :
|
8
|
+
task publish: :gem do
|
9
9
|
system "git tag -a v#{IOStreams::VERSION} -m 'Tagging #{IOStreams::VERSION}'"
|
10
|
-
system
|
10
|
+
system "git push --tags"
|
11
11
|
system "gem push iostreams-#{IOStreams::VERSION}.gem"
|
12
12
|
system "rm iostreams-#{IOStreams::VERSION}.gem"
|
13
13
|
end
|
14
14
|
|
15
15
|
Rake::TestTask.new(:test) do |t|
|
16
|
-
t.pattern =
|
16
|
+
t.pattern = "test/**/*_test.rb"
|
17
17
|
t.verbose = true
|
18
18
|
t.warning = true
|
19
19
|
end
|
20
20
|
|
21
|
-
task :
|
21
|
+
task default: :test
|
data/lib/io_streams/builder.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module IOStreams
|
2
|
-
# Build the streams that need to be applied to a path druing reading or writing.
|
2
|
+
# Build the streams that need to be applied to a path druing reading or writing.
|
3
3
|
class Builder
|
4
4
|
attr_accessor :file_name
|
5
5
|
attr_reader :streams, :options
|
@@ -61,6 +61,7 @@ module IOStreams
|
|
61
61
|
# Return the options set for either a stream or option.
|
62
62
|
def setting(stream)
|
63
63
|
return streams[stream] if streams
|
64
|
+
|
64
65
|
options[stream] if options
|
65
66
|
end
|
66
67
|
|
@@ -96,7 +97,7 @@ module IOStreams
|
|
96
97
|
|
97
98
|
# Returns the streams for the supplied file_name
|
98
99
|
def parse_extensions
|
99
|
-
parts = ::File.basename(file_name).split(
|
100
|
+
parts = ::File.basename(file_name).split(".")
|
100
101
|
extensions = []
|
101
102
|
while extension = parts.pop
|
102
103
|
sym = extension.downcase.to_sym
|
@@ -109,7 +110,7 @@ module IOStreams
|
|
109
110
|
|
110
111
|
# Executes the streams that need to be executed.
|
111
112
|
def execute(type, pipeline, io_stream, &block)
|
112
|
-
raise(ArgumentError,
|
113
|
+
raise(ArgumentError, "IOStreams call is missing mandatory block") if block.nil?
|
113
114
|
|
114
115
|
if pipeline.empty?
|
115
116
|
block.call(io_stream)
|
@@ -3,7 +3,7 @@ module IOStreams
|
|
3
3
|
class Reader < IOStreams::Reader
|
4
4
|
# Read from a Bzip2 stream, decompressing the contents as it is read
|
5
5
|
def self.stream(input_stream, **_args)
|
6
|
-
Utils.load_soft_dependency(
|
6
|
+
Utils.load_soft_dependency("rbzip2", "Bzip2") unless defined?(RBzip2)
|
7
7
|
|
8
8
|
begin
|
9
9
|
io = RBzip2.default_adapter::Decompressor.new(input_stream)
|
@@ -3,7 +3,7 @@ module IOStreams
|
|
3
3
|
class Writer < IOStreams::Writer
|
4
4
|
# Write to a stream, compressing with Bzip2
|
5
5
|
def self.stream(input_stream, original_file_name: nil, **_args)
|
6
|
-
Utils.load_soft_dependency(
|
6
|
+
Utils.load_soft_dependency("rbzip2", "Bzip2") unless defined?(RBzip2)
|
7
7
|
|
8
8
|
begin
|
9
9
|
io = RBzip2.default_adapter::Compressor.new(input_stream)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module IOStreams
|
2
|
-
UTF8_ENCODING = Encoding.find(
|
3
|
-
BINARY_ENCODING = Encoding.find(
|
2
|
+
UTF8_ENCODING = Encoding.find("UTF-8").freeze
|
3
|
+
BINARY_ENCODING = Encoding.find("BINARY").freeze
|
4
4
|
|
5
5
|
# Deprecated IOStreams from v0.x. Do not use, will be removed soon.
|
6
6
|
module Deprecated
|
@@ -9,7 +9,6 @@ module IOStreams
|
|
9
9
|
end
|
10
10
|
|
11
11
|
module ClassMethods
|
12
|
-
|
13
12
|
# DEPRECATED. Use `#path` or `#io`
|
14
13
|
# Examples:
|
15
14
|
# IOStreams.path("data.zip").reader { |f| f.read(100) }
|
@@ -7,9 +7,9 @@ module IOStreams
|
|
7
7
|
# Builtin strip options to apply after encoding the read data.
|
8
8
|
CLEANSE_RULES = {
|
9
9
|
# Strips all non printable characters
|
10
|
-
printable:
|
10
|
+
printable: ->(data, _) { data.gsub!(NOT_PRINTABLE, "") || data },
|
11
11
|
# Replaces non printable characters with the value specified in the `replace` option.
|
12
|
-
replace_non_printable: ->(data, replace) { data.gsub!(NOT_PRINTABLE, replace ||
|
12
|
+
replace_non_printable: ->(data, replace) { data.gsub!(NOT_PRINTABLE, replace || "") || data }
|
13
13
|
}.freeze
|
14
14
|
|
15
15
|
# Read a line at a time from a file or stream
|
@@ -42,7 +42,7 @@ module IOStreams
|
|
42
42
|
# :printable Cleanse all non-printable characters except \r and \n
|
43
43
|
# Proc/lambda Proc to call after every read to cleanse the data
|
44
44
|
# Default: nil
|
45
|
-
def initialize(input_stream, encoding:
|
45
|
+
def initialize(input_stream, encoding: "UTF-8", cleaner: nil, replace: nil)
|
46
46
|
super(input_stream)
|
47
47
|
|
48
48
|
@cleaner = self.class.extract_cleaner(cleaner)
|
@@ -51,11 +51,7 @@ module IOStreams
|
|
51
51
|
@replace = replace
|
52
52
|
|
53
53
|
# More efficient read buffering only supported when the input stream `#read` method supports it.
|
54
|
-
if replace.nil? && !@input_stream.method(:read).arity.between?(0, 1)
|
55
|
-
@read_cache_buffer = ''.encode(@encoding)
|
56
|
-
else
|
57
|
-
@read_cache_buffer = nil
|
58
|
-
end
|
54
|
+
@read_cache_buffer = ("".encode(@encoding) if replace.nil? && !@input_stream.method(:read).arity.between?(0, 1))
|
59
55
|
end
|
60
56
|
|
61
57
|
# Returns [String] data returned from the input stream.
|
@@ -91,6 +87,7 @@ module IOStreams
|
|
91
87
|
when Symbol
|
92
88
|
proc = CLEANSE_RULES[cleaner]
|
93
89
|
raise(ArgumentError, "Invalid cleansing rule #{cleaner.inspect}") unless proc
|
90
|
+
|
94
91
|
proc
|
95
92
|
when Proc
|
96
93
|
cleaner
|
@@ -34,7 +34,7 @@ module IOStreams
|
|
34
34
|
# :printable Cleanse all non-printable characters except \r and \n
|
35
35
|
# Proc/lambda Proc to call after every read to cleanse the data
|
36
36
|
# Default: nil
|
37
|
-
def initialize(output_stream, encoding:
|
37
|
+
def initialize(output_stream, encoding: "UTF-8", cleaner: nil, replace: nil)
|
38
38
|
super(output_stream)
|
39
39
|
|
40
40
|
@cleaner = ::IOStreams::Encode::Reader.send(:extract_cleaner, cleaner)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "uri"
|
2
2
|
|
3
3
|
# Streaming library for Ruby
|
4
4
|
#
|
@@ -111,7 +111,7 @@ module IOStreams
|
|
111
111
|
#
|
112
112
|
# Example:
|
113
113
|
# IOStreams.temp_file
|
114
|
-
def self.temp_file(basename, extension = ""
|
114
|
+
def self.temp_file(basename, extension = "")
|
115
115
|
Utils.temp_file_name(basename, extension) { |file_name| yield(Paths::File.new(file_name).stream(:none)) }
|
116
116
|
end
|
117
117
|
|
@@ -258,6 +258,7 @@ module IOStreams
|
|
258
258
|
# register_extension(:xls, MyXls::Reader, MyXls::Writer)
|
259
259
|
def self.register_extension(extension, reader_class, writer_class)
|
260
260
|
raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.nil? || extension.to_s =~ /\A\w+\Z/
|
261
|
+
|
261
262
|
@extensions[extension.nil? ? nil : extension.to_sym] = Extension.new(reader_class, writer_class)
|
262
263
|
end
|
263
264
|
|
@@ -269,6 +270,7 @@ module IOStreams
|
|
269
270
|
# register_extension(:xls)
|
270
271
|
def self.deregister_extension(extension)
|
271
272
|
raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
|
273
|
+
|
272
274
|
@extensions.delete(extension.to_sym)
|
273
275
|
end
|
274
276
|
|
@@ -284,6 +286,7 @@ module IOStreams
|
|
284
286
|
# register_scheme(:xls, MyXls::Reader, MyXls::Writer)
|
285
287
|
def self.register_scheme(scheme, klass)
|
286
288
|
raise(ArgumentError, "Invalid scheme #{scheme.inspect}") unless scheme.nil? || scheme.to_s =~ /\A\w+\Z/
|
289
|
+
|
287
290
|
@schemes[scheme.nil? ? nil : scheme.to_sym] = klass
|
288
291
|
end
|
289
292
|
|
@@ -94,6 +94,7 @@ module IOStreams
|
|
94
94
|
initial_line_number = @line_number
|
95
95
|
while line.count(@embedded_within).odd?
|
96
96
|
raise "Unclosed quoted field on line #{initial_line_number}" if eof? || line.length > @buffer_size * 10
|
97
|
+
|
97
98
|
line << @delimiter
|
98
99
|
line << _readline
|
99
100
|
end
|
@@ -163,7 +164,7 @@ module IOStreams
|
|
163
164
|
# Take on the encoding from the input stream
|
164
165
|
@buffer = block.dup
|
165
166
|
# Take on the encoding from the first block that was read.
|
166
|
-
@read_cache_buffer =
|
167
|
+
@read_cache_buffer = "".encode(block.encoding) if @use_read_cache_buffer
|
167
168
|
end
|
168
169
|
|
169
170
|
if @buffer.size > MAX_BLOCKS_MULTIPLIER * @buffer_size
|
data/lib/io_streams/path.rb
CHANGED
@@ -3,7 +3,7 @@ module IOStreams
|
|
3
3
|
attr_accessor :path
|
4
4
|
|
5
5
|
def initialize(path)
|
6
|
-
raise(ArgumentError,
|
6
|
+
raise(ArgumentError, "Path cannot be nil") if path.nil?
|
7
7
|
raise(ArgumentError, "Path must be a string: #{path.inspect}, class: #{path.class}") unless path.is_a?(String)
|
8
8
|
|
9
9
|
@path = path.frozen? ? path : path.dup.freeze
|
@@ -30,7 +30,7 @@ module IOStreams
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def absolute?
|
33
|
-
!!(path.strip =~
|
33
|
+
!!(path.strip =~ %r{\A/})
|
34
34
|
end
|
35
35
|
|
36
36
|
# By default realpath just returns self.
|
@@ -84,12 +84,12 @@ module IOStreams
|
|
84
84
|
# Cleanup an incomplete write to the target "file" if the copy fails.
|
85
85
|
def copy_from(source, **args)
|
86
86
|
super(source, **args)
|
87
|
-
rescue StandardError =>
|
87
|
+
rescue StandardError => e
|
88
88
|
begin
|
89
89
|
delete
|
90
90
|
rescue NotImplementedError
|
91
91
|
end
|
92
|
-
raise(
|
92
|
+
raise(e)
|
93
93
|
end
|
94
94
|
|
95
95
|
# Moves the file by copying it to the new path and then deleting the current path.
|
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "net/http"
|
2
|
+
require "uri"
|
3
3
|
module IOStreams
|
4
4
|
module Paths
|
5
5
|
class HTTP < IOStreams::Path
|
@@ -68,21 +68,19 @@ module IOStreams
|
|
68
68
|
result = nil
|
69
69
|
raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
|
70
70
|
|
71
|
-
Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme ==
|
71
|
+
Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
|
72
72
|
request = Net::HTTP::Get.new(uri)
|
73
73
|
request.basic_auth(username, password) if username
|
74
74
|
|
75
75
|
http.request(request) do |response|
|
76
|
-
if response.is_a?(Net::HTTPNotFound)
|
77
|
-
raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}")
|
78
|
-
end
|
76
|
+
raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}") if response.is_a?(Net::HTTPNotFound)
|
79
77
|
if response.is_a?(Net::HTTPUnauthorized)
|
80
78
|
raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
|
81
79
|
end
|
82
80
|
|
83
81
|
if response.is_a?(Net::HTTPRedirection)
|
84
|
-
new_uri = response[
|
85
|
-
return handle_redirects(new_uri, http_redirect_count
|
82
|
+
new_uri = response["location"]
|
83
|
+
return handle_redirects(new_uri, http_redirect_count - 1, &block)
|
86
84
|
end
|
87
85
|
|
88
86
|
unless response.is_a?(Net::HTTPSuccess)
|
@@ -90,8 +88,8 @@ module IOStreams
|
|
90
88
|
end
|
91
89
|
|
92
90
|
# Since Net::HTTP download only supports a push stream, write it to a tempfile first.
|
93
|
-
Utils.temp_file_name(
|
94
|
-
::File.open(file_name,
|
91
|
+
Utils.temp_file_name("iostreams_http") do |file_name|
|
92
|
+
::File.open(file_name, "wb") { |io| response.read_body { |chunk| io.write(chunk) } }
|
95
93
|
# Return a read stream
|
96
94
|
result = ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
|
97
95
|
end
|
@@ -3,7 +3,7 @@ module IOStreams
|
|
3
3
|
# Implement fnmatch logic for any path iterator
|
4
4
|
class Matcher
|
5
5
|
# Characters indicating that pattern matching is required
|
6
|
-
MATCH_START_CHARS = /[*?\[{]
|
6
|
+
MATCH_START_CHARS = /[*?\[{]/.freeze
|
7
7
|
|
8
8
|
attr_reader :path, :pattern, :flags
|
9
9
|
|
@@ -26,7 +26,7 @@ module IOStreams
|
|
26
26
|
|
27
27
|
# Returns whether the relative `file_name` matches
|
28
28
|
def match?(file_name)
|
29
|
-
relative_file_name = file_name.sub(path.to_s,
|
29
|
+
relative_file_name = file_name.sub(path.to_s, "").sub(%r{\A/}, "")
|
30
30
|
::File.fnmatch?(pattern, relative_file_name, flags)
|
31
31
|
end
|
32
32
|
|
@@ -39,23 +39,22 @@ module IOStreams
|
|
39
39
|
private
|
40
40
|
|
41
41
|
def extract_optimized_path(path, pattern)
|
42
|
-
elements = pattern.split(
|
42
|
+
elements = pattern.split("/")
|
43
43
|
index = elements.find_index { |e| e.match(MATCH_START_CHARS) }
|
44
|
-
if index
|
45
|
-
# Cannot optimize path since the very first entry contains a wildcard
|
46
|
-
@path = path || IOStreams.path
|
47
|
-
@pattern = pattern
|
48
|
-
elsif index.nil?
|
44
|
+
if index.nil?
|
49
45
|
# No index means it has no pattern.
|
50
46
|
@path = path.nil? ? IOStreams.path(pattern) : path.join(pattern)
|
51
47
|
@pattern = nil
|
48
|
+
elsif index.zero?
|
49
|
+
# Cannot optimize path since the very first entry contains a wildcard
|
50
|
+
@path = path || IOStreams.path
|
51
|
+
@pattern = pattern
|
52
52
|
else
|
53
|
-
new_path = elements[0..index - 1].join(
|
53
|
+
new_path = elements[0..index - 1].join("/")
|
54
54
|
@path = path.nil? ? IOStreams.path(new_path) : path.join(new_path)
|
55
|
-
@pattern = elements[index..-1].join(
|
55
|
+
@pattern = elements[index..-1].join("/")
|
56
56
|
end
|
57
57
|
end
|
58
|
-
|
59
58
|
end
|
60
59
|
end
|
61
60
|
end
|
data/lib/io_streams/paths/s3.rb
CHANGED
@@ -131,13 +131,13 @@ module IOStreams
|
|
131
131
|
# @option params [String] :object_lock_legal_hold_status
|
132
132
|
# The Legal Hold status that you want to apply to the specified object.
|
133
133
|
def initialize(url, client: nil, access_key_id: nil, secret_access_key: nil, **args)
|
134
|
-
Utils.load_soft_dependency(
|
134
|
+
Utils.load_soft_dependency("aws-sdk-s3", "AWS S3") unless defined?(::Aws::S3::Client)
|
135
135
|
|
136
136
|
uri = Utils::URI.new(url)
|
137
|
-
raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme ==
|
137
|
+
raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == "s3"
|
138
138
|
|
139
139
|
@bucket_name = uri.hostname
|
140
|
-
key = uri.path.sub(%r{\A/},
|
140
|
+
key = uri.path.sub(%r{\A/}, "")
|
141
141
|
if client.is_a?(Hash)
|
142
142
|
client[:access_key_id] = access_key_id if access_key_id
|
143
143
|
client[:secret_access_key] = secret_access_key if secret_access_key
|
@@ -219,7 +219,7 @@ module IOStreams
|
|
219
219
|
|
220
220
|
# Shortcut method if caller has a filename already with no other streams applied:
|
221
221
|
def read_file(file_name)
|
222
|
-
::File.open(file_name,
|
222
|
+
::File.open(file_name, "wb") do |file|
|
223
223
|
client.get_object(@options.merge(response_target: file, bucket: bucket_name, key: path))
|
224
224
|
end
|
225
225
|
end
|
@@ -250,7 +250,7 @@ module IOStreams
|
|
250
250
|
obj = s3.bucket(bucket_name).object(path)
|
251
251
|
obj.upload_file(file_name)
|
252
252
|
else
|
253
|
-
::File.open(file_name,
|
253
|
+
::File.open(file_name, "rb") do |file|
|
254
254
|
client.put_object(@options.merge(bucket: bucket_name, key: path, body: file))
|
255
255
|
end
|
256
256
|
end
|
@@ -267,7 +267,7 @@ module IOStreams
|
|
267
267
|
return
|
268
268
|
end
|
269
269
|
|
270
|
-
prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/},
|
270
|
+
prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/}, "")
|
271
271
|
token = nil
|
272
272
|
loop do
|
273
273
|
# Fetches upto 1,000 entries at a time
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "open3"
|
2
2
|
|
3
3
|
module IOStreams
|
4
4
|
module Paths
|
@@ -29,8 +29,8 @@ module IOStreams
|
|
29
29
|
attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds
|
30
30
|
end
|
31
31
|
|
32
|
-
@sftp_bin =
|
33
|
-
@sshpass_bin =
|
32
|
+
@sftp_bin = "sftp"
|
33
|
+
@sshpass_bin = "sshpass"
|
34
34
|
@sshpass_wait_seconds = 5
|
35
35
|
|
36
36
|
attr_reader :hostname, :username, :ssh_options, :url, :port
|
@@ -76,7 +76,7 @@ module IOStreams
|
|
76
76
|
# end
|
77
77
|
def initialize(url, username: nil, password: nil, ssh_options: {})
|
78
78
|
uri = Utils::URI.new(url)
|
79
|
-
raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme ==
|
79
|
+
raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == "sftp"
|
80
80
|
|
81
81
|
@hostname = uri.hostname
|
82
82
|
@mkdir = false
|
@@ -133,6 +133,7 @@ module IOStreams
|
|
133
133
|
Net::SFTP.start(hostname, username, build_ssh_options) do |sftp|
|
134
134
|
sftp.dir.glob(".", pattern, flags) do |path|
|
135
135
|
next if !directories && !path.file?
|
136
|
+
|
136
137
|
new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password, **ssh_options)
|
137
138
|
yield(new_path, path.attributes.attributes)
|
138
139
|
end
|
@@ -168,13 +169,20 @@ module IOStreams
|
|
168
169
|
# Give time for password to be processed and stdin to be passed to sftp process.
|
169
170
|
sleep self.class.sshpass_wait_seconds
|
170
171
|
writer.puts "get #{remote_file_name} #{local_file_name}"
|
171
|
-
writer.puts
|
172
|
+
writer.puts "bye"
|
172
173
|
writer.close
|
173
174
|
out = reader.read.chomp
|
174
|
-
|
175
|
+
unless waith_thr.value.success?
|
176
|
+
raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
|
177
|
+
end
|
178
|
+
|
175
179
|
out
|
176
180
|
rescue Errno::EPIPE
|
177
|
-
out =
|
181
|
+
out = begin
|
182
|
+
reader.read.chomp
|
183
|
+
rescue StandardError
|
184
|
+
nil
|
185
|
+
end
|
178
186
|
raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
|
179
187
|
end
|
180
188
|
end
|
@@ -189,13 +197,20 @@ module IOStreams
|
|
189
197
|
# Give time for password to be processed and stdin to be passed to sftp process.
|
190
198
|
sleep self.class.sshpass_wait_seconds
|
191
199
|
writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
|
192
|
-
writer.puts
|
200
|
+
writer.puts "bye"
|
193
201
|
writer.close
|
194
202
|
out = reader.read.chomp
|
195
|
-
|
203
|
+
unless waith_thr.value.success?
|
204
|
+
raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
|
205
|
+
end
|
206
|
+
|
196
207
|
out
|
197
208
|
rescue Errno::EPIPE
|
198
|
-
out =
|
209
|
+
out = begin
|
210
|
+
reader.read.chomp
|
211
|
+
rescue StandardError
|
212
|
+
nil
|
213
|
+
end
|
199
214
|
raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
|
200
215
|
end
|
201
216
|
end
|
@@ -203,15 +218,15 @@ module IOStreams
|
|
203
218
|
end
|
204
219
|
|
205
220
|
def with_sftp_args
|
206
|
-
return yield sftp_args(ssh_options) unless ssh_options.key?(
|
221
|
+
return yield sftp_args(ssh_options) unless ssh_options.key?("IdentityKey")
|
207
222
|
|
208
|
-
Utils.temp_file_name(
|
223
|
+
Utils.temp_file_name("iostreams-sftp-args", "key") do |file_name|
|
209
224
|
options = ssh_options.dup
|
210
|
-
key = options.delete(
|
225
|
+
key = options.delete("IdentityKey")
|
211
226
|
# sftp requires that private key is only readable by the current user
|
212
|
-
::File.open(file_name,
|
227
|
+
::File.open(file_name, "wb", 0o600) { |io| io.write(key) }
|
213
228
|
|
214
|
-
options[
|
229
|
+
options["IdentityFile"] = file_name
|
215
230
|
yield sftp_args(options)
|
216
231
|
end
|
217
232
|
end
|
@@ -228,22 +243,22 @@ module IOStreams
|
|
228
243
|
args << "-oBatchMode=yes"
|
229
244
|
args << "-oPasswordAuthentication=no"
|
230
245
|
end
|
231
|
-
args << "-oIdentitiesOnly=yes" if ssh_options.key?(
|
246
|
+
args << "-oIdentitiesOnly=yes" if ssh_options.key?("IdentityFile")
|
232
247
|
# Default is ask, but this is non-interactive so make the default fail without asking.
|
233
|
-
args << "-oStrictHostKeyChecking=yes" unless ssh_options.key?(
|
234
|
-
args << "-oLogLevel=#{map_log_level}" unless ssh_options.key?(
|
248
|
+
args << "-oStrictHostKeyChecking=yes" unless ssh_options.key?("StrictHostKeyChecking")
|
249
|
+
args << "-oLogLevel=#{map_log_level}" unless ssh_options.key?("LogLevel")
|
235
250
|
args << "-oPort=#{port}" unless port == 22
|
236
251
|
ssh_options.each_pair { |key, value| args << "-o#{key}=#{value}" }
|
237
|
-
args <<
|
238
|
-
args <<
|
252
|
+
args << "-b"
|
253
|
+
args << "-"
|
239
254
|
args << "#{username}@#{hostname}"
|
240
255
|
args
|
241
256
|
end
|
242
257
|
|
243
258
|
def build_ssh_options
|
244
|
-
options
|
245
|
-
options[:logger]
|
246
|
-
options[:port]
|
259
|
+
options = ssh_options.dup
|
260
|
+
options[:logger] ||= logger if defined?(SemanticLogger)
|
261
|
+
options[:port] ||= port
|
247
262
|
options[:max_pkt_size] ||= 65_536
|
248
263
|
options[:password] ||= @password
|
249
264
|
options
|