iostreams 1.1.0 → 1.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/Rakefile +7 -7
  4. data/lib/io_streams/builder.rb +4 -3
  5. data/lib/io_streams/bzip2/reader.rb +1 -1
  6. data/lib/io_streams/bzip2/writer.rb +1 -1
  7. data/lib/io_streams/deprecated.rb +2 -3
  8. data/lib/io_streams/encode/reader.rb +5 -8
  9. data/lib/io_streams/encode/writer.rb +1 -1
  10. data/lib/io_streams/io_streams.rb +5 -2
  11. data/lib/io_streams/line/reader.rb +2 -1
  12. data/lib/io_streams/path.rb +4 -4
  13. data/lib/io_streams/paths/http.rb +8 -10
  14. data/lib/io_streams/paths/matcher.rb +10 -11
  15. data/lib/io_streams/paths/s3.rb +6 -6
  16. data/lib/io_streams/paths/sftp.rb +38 -23
  17. data/lib/io_streams/pgp.rb +45 -35
  18. data/lib/io_streams/pgp/reader.rb +4 -6
  19. data/lib/io_streams/pgp/writer.rb +1 -2
  20. data/lib/io_streams/reader.rb +2 -2
  21. data/lib/io_streams/record/writer.rb +2 -4
  22. data/lib/io_streams/row/writer.rb +3 -5
  23. data/lib/io_streams/stream.rb +6 -6
  24. data/lib/io_streams/symmetric_encryption/reader.rb +1 -3
  25. data/lib/io_streams/symmetric_encryption/writer.rb +2 -6
  26. data/lib/io_streams/tabular.rb +12 -10
  27. data/lib/io_streams/tabular/header.rb +4 -4
  28. data/lib/io_streams/tabular/parser/array.rb +2 -4
  29. data/lib/io_streams/tabular/parser/csv.rb +3 -5
  30. data/lib/io_streams/tabular/parser/fixed.rb +4 -3
  31. data/lib/io_streams/tabular/parser/hash.rb +2 -4
  32. data/lib/io_streams/tabular/parser/json.rb +2 -4
  33. data/lib/io_streams/tabular/parser/psv.rb +5 -7
  34. data/lib/io_streams/tabular/utility/csv_row.rb +9 -17
  35. data/lib/io_streams/utils.rb +3 -3
  36. data/lib/io_streams/utils/reliable_http.rb +98 -0
  37. data/lib/io_streams/version.rb +1 -1
  38. data/lib/io_streams/writer.rb +1 -1
  39. data/lib/io_streams/xlsx/reader.rb +5 -5
  40. data/lib/io_streams/zip/reader.rb +1 -1
  41. data/lib/io_streams/zip/writer.rb +2 -2
  42. data/lib/iostreams.rb +34 -34
  43. data/test/builder_test.rb +74 -74
  44. data/test/bzip2_reader_test.rb +8 -13
  45. data/test/bzip2_writer_test.rb +8 -9
  46. data/test/deprecated_test.rb +25 -29
  47. data/test/encode_reader_test.rb +14 -18
  48. data/test/encode_writer_test.rb +29 -30
  49. data/test/gzip_reader_test.rb +8 -13
  50. data/test/gzip_writer_test.rb +10 -11
  51. data/test/io_streams_test.rb +35 -35
  52. data/test/line_reader_test.rb +35 -39
  53. data/test/line_writer_test.rb +8 -9
  54. data/test/minimal_file_reader.rb +1 -1
  55. data/test/path_test.rb +24 -24
  56. data/test/paths/file_test.rb +42 -42
  57. data/test/paths/http_test.rb +5 -5
  58. data/test/paths/matcher_test.rb +11 -12
  59. data/test/paths/s3_test.rb +44 -46
  60. data/test/paths/sftp_test.rb +18 -18
  61. data/test/pgp_reader_test.rb +13 -15
  62. data/test/pgp_test.rb +43 -44
  63. data/test/pgp_writer_test.rb +27 -28
  64. data/test/record_reader_test.rb +9 -10
  65. data/test/record_writer_test.rb +10 -11
  66. data/test/row_reader_test.rb +5 -6
  67. data/test/row_writer_test.rb +7 -8
  68. data/test/stream_test.rb +60 -62
  69. data/test/tabular_test.rb +111 -111
  70. data/test/test_helper.rb +22 -22
  71. data/test/utils_test.rb +7 -7
  72. data/test/xlsx_reader_test.rb +12 -12
  73. data/test/zip_reader_test.rb +14 -21
  74. data/test/zip_writer_test.rb +10 -10
  75. metadata +4 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1a047f1f33b360bf1aa4f028da098e5d2364e4e4ebc4f4c603c445e6f43f99d8
4
- data.tar.gz: 42e7095dcdb7707ec774f6d5efe104418fcb632af97d2978e736ddd3ec7c72ae
3
+ metadata.gz: 1a141ad8f92a6c1387ad487fc8cf545bcc010ebc862967ffce8f79f71b7ba2d6
4
+ data.tar.gz: 0210b8d23390ddff6fe31133c24b88b2beb7e5e0f54d6b9df962632dc369a9ec
5
5
  SHA512:
6
- metadata.gz: ef9b14f94069a48f81cfb52960a115f0bfb83bcebde57850002d52148e2cb495a0eed9a9f1a6505e1210dd259cc1ab8555e36cb7e9769410a48ca11daa4c4b9c
7
- data.tar.gz: b8b0bc72a97bbb7174c32e3d852ce171e6443d246950f788016ed4309e539d644703765fd2767c36ea4df7b8218d7b14b84c94a8630d5214a1bc60b0f4659472
6
+ metadata.gz: 810573a43277573365e946205465a4a1cc87c65cf517450dc6f363f5bb7168e66a25cd2d891e346d09c4b96aa9166f369fd2bc9dc21ec2112d87bce270467ff2
7
+ data.tar.gz: 2304155b27a897f270263283bdf499855a81f80885b9103b6c116f135904a42d73bfb29705a8156f3bde973b7a899eba9b123073e7b140a2e502b2203869ad1f
data/README.md CHANGED
@@ -1,5 +1,5 @@
1
1
  # iostreams
2
- [![Gem Version](https://img.shields.io/gem/v/iostreams.svg)](https://rubygems.org/gems/iostreams) [![Build Status](https://travis-ci.org/rocketjob/iostreams.svg?branch=master)](https://travis-ci.org/rocketjob/iostreams) [![Downloads](https://img.shields.io/gem/dt/iostreams.svg)](https://rubygems.org/gems/iostreams) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Beta-yellow.svg) [![Gitter chat](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
2
+ [![Gem Version](https://img.shields.io/gem/v/iostreams.svg)](https://rubygems.org/gems/iostreams) [![Build Status](https://travis-ci.org/rocketjob/iostreams.svg?branch=master)](https://travis-ci.org/rocketjob/iostreams) [![Downloads](https://img.shields.io/gem/dt/iostreams.svg)](https://rubygems.org/gems/iostreams) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Gitter chat](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
3
3
 
4
4
  Input and Output streaming for Ruby.
5
5
 
data/Rakefile CHANGED
@@ -1,21 +1,21 @@
1
- require 'rake/testtask'
2
- require_relative 'lib/io_streams/version'
1
+ require "rake/testtask"
2
+ require_relative "lib/io_streams/version"
3
3
 
4
4
  task :gem do
5
- system 'gem build iostreams.gemspec'
5
+ system "gem build iostreams.gemspec"
6
6
  end
7
7
 
8
- task :publish => :gem do
8
+ task publish: :gem do
9
9
  system "git tag -a v#{IOStreams::VERSION} -m 'Tagging #{IOStreams::VERSION}'"
10
- system 'git push --tags'
10
+ system "git push --tags"
11
11
  system "gem push iostreams-#{IOStreams::VERSION}.gem"
12
12
  system "rm iostreams-#{IOStreams::VERSION}.gem"
13
13
  end
14
14
 
15
15
  Rake::TestTask.new(:test) do |t|
16
- t.pattern = 'test/**/*_test.rb'
16
+ t.pattern = "test/**/*_test.rb"
17
17
  t.verbose = true
18
18
  t.warning = true
19
19
  end
20
20
 
21
- task :default => :test
21
+ task default: :test
@@ -1,5 +1,5 @@
1
1
  module IOStreams
2
- # Build the streams that need to be applied to a path druing reading or writing.
2
+ # Build the streams that need to be applied to a path druing reading or writing.
3
3
  class Builder
4
4
  attr_accessor :file_name
5
5
  attr_reader :streams, :options
@@ -61,6 +61,7 @@ module IOStreams
61
61
  # Return the options set for either a stream or option.
62
62
  def setting(stream)
63
63
  return streams[stream] if streams
64
+
64
65
  options[stream] if options
65
66
  end
66
67
 
@@ -96,7 +97,7 @@ module IOStreams
96
97
 
97
98
  # Returns the streams for the supplied file_name
98
99
  def parse_extensions
99
- parts = ::File.basename(file_name).split('.')
100
+ parts = ::File.basename(file_name).split(".")
100
101
  extensions = []
101
102
  while extension = parts.pop
102
103
  sym = extension.downcase.to_sym
@@ -109,7 +110,7 @@ module IOStreams
109
110
 
110
111
  # Executes the streams that need to be executed.
111
112
  def execute(type, pipeline, io_stream, &block)
112
- raise(ArgumentError, 'IOStreams call is missing mandatory block') if block.nil?
113
+ raise(ArgumentError, "IOStreams call is missing mandatory block") if block.nil?
113
114
 
114
115
  if pipeline.empty?
115
116
  block.call(io_stream)
@@ -3,7 +3,7 @@ module IOStreams
3
3
  class Reader < IOStreams::Reader
4
4
  # Read from a Bzip2 stream, decompressing the contents as it is read
5
5
  def self.stream(input_stream, **_args)
6
- Utils.load_soft_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
6
+ Utils.load_soft_dependency("rbzip2", "Bzip2") unless defined?(RBzip2)
7
7
 
8
8
  begin
9
9
  io = RBzip2.default_adapter::Decompressor.new(input_stream)
@@ -3,7 +3,7 @@ module IOStreams
3
3
  class Writer < IOStreams::Writer
4
4
  # Write to a stream, compressing with Bzip2
5
5
  def self.stream(input_stream, original_file_name: nil, **_args)
6
- Utils.load_soft_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
6
+ Utils.load_soft_dependency("rbzip2", "Bzip2") unless defined?(RBzip2)
7
7
 
8
8
  begin
9
9
  io = RBzip2.default_adapter::Compressor.new(input_stream)
@@ -1,6 +1,6 @@
1
1
  module IOStreams
2
- UTF8_ENCODING = Encoding.find('UTF-8').freeze
3
- BINARY_ENCODING = Encoding.find('BINARY').freeze
2
+ UTF8_ENCODING = Encoding.find("UTF-8").freeze
3
+ BINARY_ENCODING = Encoding.find("BINARY").freeze
4
4
 
5
5
  # Deprecated IOStreams from v0.x. Do not use, will be removed soon.
6
6
  module Deprecated
@@ -9,7 +9,6 @@ module IOStreams
9
9
  end
10
10
 
11
11
  module ClassMethods
12
-
13
12
  # DEPRECATED. Use `#path` or `#io`
14
13
  # Examples:
15
14
  # IOStreams.path("data.zip").reader { |f| f.read(100) }
@@ -7,9 +7,9 @@ module IOStreams
7
7
  # Builtin strip options to apply after encoding the read data.
8
8
  CLEANSE_RULES = {
9
9
  # Strips all non printable characters
10
- printable: ->(data, _) { data.gsub!(NOT_PRINTABLE, '') || data },
10
+ printable: ->(data, _) { data.gsub!(NOT_PRINTABLE, "") || data },
11
11
  # Replaces non printable characters with the value specified in the `replace` option.
12
- replace_non_printable: ->(data, replace) { data.gsub!(NOT_PRINTABLE, replace || '') || data }
12
+ replace_non_printable: ->(data, replace) { data.gsub!(NOT_PRINTABLE, replace || "") || data }
13
13
  }.freeze
14
14
 
15
15
  # Read a line at a time from a file or stream
@@ -42,7 +42,7 @@ module IOStreams
42
42
  # :printable Cleanse all non-printable characters except \r and \n
43
43
  # Proc/lambda Proc to call after every read to cleanse the data
44
44
  # Default: nil
45
- def initialize(input_stream, encoding: 'UTF-8', cleaner: nil, replace: nil)
45
+ def initialize(input_stream, encoding: "UTF-8", cleaner: nil, replace: nil)
46
46
  super(input_stream)
47
47
 
48
48
  @cleaner = self.class.extract_cleaner(cleaner)
@@ -51,11 +51,7 @@ module IOStreams
51
51
  @replace = replace
52
52
 
53
53
  # More efficient read buffering only supported when the input stream `#read` method supports it.
54
- if replace.nil? && !@input_stream.method(:read).arity.between?(0, 1)
55
- @read_cache_buffer = ''.encode(@encoding)
56
- else
57
- @read_cache_buffer = nil
58
- end
54
+ @read_cache_buffer = ("".encode(@encoding) if replace.nil? && !@input_stream.method(:read).arity.between?(0, 1))
59
55
  end
60
56
 
61
57
  # Returns [String] data returned from the input stream.
@@ -91,6 +87,7 @@ module IOStreams
91
87
  when Symbol
92
88
  proc = CLEANSE_RULES[cleaner]
93
89
  raise(ArgumentError, "Invalid cleansing rule #{cleaner.inspect}") unless proc
90
+
94
91
  proc
95
92
  when Proc
96
93
  cleaner
@@ -34,7 +34,7 @@ module IOStreams
34
34
  # :printable Cleanse all non-printable characters except \r and \n
35
35
  # Proc/lambda Proc to call after every read to cleanse the data
36
36
  # Default: nil
37
- def initialize(output_stream, encoding: 'UTF-8', cleaner: nil, replace: nil)
37
+ def initialize(output_stream, encoding: "UTF-8", cleaner: nil, replace: nil)
38
38
  super(output_stream)
39
39
 
40
40
  @cleaner = ::IOStreams::Encode::Reader.send(:extract_cleaner, cleaner)
@@ -1,4 +1,4 @@
1
- require 'uri'
1
+ require "uri"
2
2
 
3
3
  # Streaming library for Ruby
4
4
  #
@@ -111,7 +111,7 @@ module IOStreams
111
111
  #
112
112
  # Example:
113
113
  # IOStreams.temp_file
114
- def self.temp_file(basename, extension = "", &block)
114
+ def self.temp_file(basename, extension = "")
115
115
  Utils.temp_file_name(basename, extension) { |file_name| yield(Paths::File.new(file_name).stream(:none)) }
116
116
  end
117
117
 
@@ -258,6 +258,7 @@ module IOStreams
258
258
  # register_extension(:xls, MyXls::Reader, MyXls::Writer)
259
259
  def self.register_extension(extension, reader_class, writer_class)
260
260
  raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.nil? || extension.to_s =~ /\A\w+\Z/
261
+
261
262
  @extensions[extension.nil? ? nil : extension.to_sym] = Extension.new(reader_class, writer_class)
262
263
  end
263
264
 
@@ -269,6 +270,7 @@ module IOStreams
269
270
  # register_extension(:xls)
270
271
  def self.deregister_extension(extension)
271
272
  raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
273
+
272
274
  @extensions.delete(extension.to_sym)
273
275
  end
274
276
 
@@ -284,6 +286,7 @@ module IOStreams
284
286
  # register_scheme(:xls, MyXls::Reader, MyXls::Writer)
285
287
  def self.register_scheme(scheme, klass)
286
288
  raise(ArgumentError, "Invalid scheme #{scheme.inspect}") unless scheme.nil? || scheme.to_s =~ /\A\w+\Z/
289
+
287
290
  @schemes[scheme.nil? ? nil : scheme.to_sym] = klass
288
291
  end
289
292
 
@@ -94,6 +94,7 @@ module IOStreams
94
94
  initial_line_number = @line_number
95
95
  while line.count(@embedded_within).odd?
96
96
  raise "Unclosed quoted field on line #{initial_line_number}" if eof? || line.length > @buffer_size * 10
97
+
97
98
  line << @delimiter
98
99
  line << _readline
99
100
  end
@@ -163,7 +164,7 @@ module IOStreams
163
164
  # Take on the encoding from the input stream
164
165
  @buffer = block.dup
165
166
  # Take on the encoding from the first block that was read.
166
- @read_cache_buffer = ''.encode(block.encoding) if @use_read_cache_buffer
167
+ @read_cache_buffer = "".encode(block.encoding) if @use_read_cache_buffer
167
168
  end
168
169
 
169
170
  if @buffer.size > MAX_BLOCKS_MULTIPLIER * @buffer_size
@@ -3,7 +3,7 @@ module IOStreams
3
3
  attr_accessor :path
4
4
 
5
5
  def initialize(path)
6
- raise(ArgumentError, 'Path cannot be nil') if path.nil?
6
+ raise(ArgumentError, "Path cannot be nil") if path.nil?
7
7
  raise(ArgumentError, "Path must be a string: #{path.inspect}, class: #{path.class}") unless path.is_a?(String)
8
8
 
9
9
  @path = path.frozen? ? path : path.dup.freeze
@@ -30,7 +30,7 @@ module IOStreams
30
30
  end
31
31
 
32
32
  def absolute?
33
- !!(path.strip =~ /\A\//)
33
+ !!(path.strip =~ %r{\A/})
34
34
  end
35
35
 
36
36
  # By default realpath just returns self.
@@ -84,12 +84,12 @@ module IOStreams
84
84
  # Cleanup an incomplete write to the target "file" if the copy fails.
85
85
  def copy_from(source, **args)
86
86
  super(source, **args)
87
- rescue StandardError => exc
87
+ rescue StandardError => e
88
88
  begin
89
89
  delete
90
90
  rescue NotImplementedError
91
91
  end
92
- raise(exc)
92
+ raise(e)
93
93
  end
94
94
 
95
95
  # Moves the file by copying it to the new path and then deleting the current path.
@@ -1,5 +1,5 @@
1
- require 'net/http'
2
- require 'uri'
1
+ require "net/http"
2
+ require "uri"
3
3
  module IOStreams
4
4
  module Paths
5
5
  class HTTP < IOStreams::Path
@@ -68,21 +68,19 @@ module IOStreams
68
68
  result = nil
69
69
  raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
70
70
 
71
- Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
71
+ Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
72
72
  request = Net::HTTP::Get.new(uri)
73
73
  request.basic_auth(username, password) if username
74
74
 
75
75
  http.request(request) do |response|
76
- if response.is_a?(Net::HTTPNotFound)
77
- raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}")
78
- end
76
+ raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}") if response.is_a?(Net::HTTPNotFound)
79
77
  if response.is_a?(Net::HTTPUnauthorized)
80
78
  raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
81
79
  end
82
80
 
83
81
  if response.is_a?(Net::HTTPRedirection)
84
- new_uri = response['location']
85
- return handle_redirects(new_uri, http_redirect_count: http_redirect_count - 1, &block)
82
+ new_uri = response["location"]
83
+ return handle_redirects(new_uri, http_redirect_count - 1, &block)
86
84
  end
87
85
 
88
86
  unless response.is_a?(Net::HTTPSuccess)
@@ -90,8 +88,8 @@ module IOStreams
90
88
  end
91
89
 
92
90
  # Since Net::HTTP download only supports a push stream, write it to a tempfile first.
93
- Utils.temp_file_name('iostreams_http') do |file_name|
94
- ::File.open(file_name, 'wb') { |io| response.read_body { |chunk| io.write(chunk) } }
91
+ Utils.temp_file_name("iostreams_http") do |file_name|
92
+ ::File.open(file_name, "wb") { |io| response.read_body { |chunk| io.write(chunk) } }
95
93
  # Return a read stream
96
94
  result = ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
97
95
  end
@@ -3,7 +3,7 @@ module IOStreams
3
3
  # Implement fnmatch logic for any path iterator
4
4
  class Matcher
5
5
  # Characters indicating that pattern matching is required
6
- MATCH_START_CHARS = /[*?\[{]/
6
+ MATCH_START_CHARS = /[*?\[{]/.freeze
7
7
 
8
8
  attr_reader :path, :pattern, :flags
9
9
 
@@ -26,7 +26,7 @@ module IOStreams
26
26
 
27
27
  # Returns whether the relative `file_name` matches
28
28
  def match?(file_name)
29
- relative_file_name = file_name.sub(path.to_s, '').sub(%r{\A/}, '')
29
+ relative_file_name = file_name.sub(path.to_s, "").sub(%r{\A/}, "")
30
30
  ::File.fnmatch?(pattern, relative_file_name, flags)
31
31
  end
32
32
 
@@ -39,23 +39,22 @@ module IOStreams
39
39
  private
40
40
 
41
41
  def extract_optimized_path(path, pattern)
42
- elements = pattern.split('/')
42
+ elements = pattern.split("/")
43
43
  index = elements.find_index { |e| e.match(MATCH_START_CHARS) }
44
- if index == 0
45
- # Cannot optimize path since the very first entry contains a wildcard
46
- @path = path || IOStreams.path
47
- @pattern = pattern
48
- elsif index.nil?
44
+ if index.nil?
49
45
  # No index means it has no pattern.
50
46
  @path = path.nil? ? IOStreams.path(pattern) : path.join(pattern)
51
47
  @pattern = nil
48
+ elsif index.zero?
49
+ # Cannot optimize path since the very first entry contains a wildcard
50
+ @path = path || IOStreams.path
51
+ @pattern = pattern
52
52
  else
53
- new_path = elements[0..index - 1].join('/')
53
+ new_path = elements[0..index - 1].join("/")
54
54
  @path = path.nil? ? IOStreams.path(new_path) : path.join(new_path)
55
- @pattern = elements[index..-1].join('/')
55
+ @pattern = elements[index..-1].join("/")
56
56
  end
57
57
  end
58
-
59
58
  end
60
59
  end
61
60
  end
@@ -131,13 +131,13 @@ module IOStreams
131
131
  # @option params [String] :object_lock_legal_hold_status
132
132
  # The Legal Hold status that you want to apply to the specified object.
133
133
  def initialize(url, client: nil, access_key_id: nil, secret_access_key: nil, **args)
134
- Utils.load_soft_dependency('aws-sdk-s3', 'AWS S3') unless defined?(::Aws::S3::Client)
134
+ Utils.load_soft_dependency("aws-sdk-s3", "AWS S3") unless defined?(::Aws::S3::Client)
135
135
 
136
136
  uri = Utils::URI.new(url)
137
- raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == 's3'
137
+ raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == "s3"
138
138
 
139
139
  @bucket_name = uri.hostname
140
- key = uri.path.sub(%r{\A/}, '')
140
+ key = uri.path.sub(%r{\A/}, "")
141
141
  if client.is_a?(Hash)
142
142
  client[:access_key_id] = access_key_id if access_key_id
143
143
  client[:secret_access_key] = secret_access_key if secret_access_key
@@ -219,7 +219,7 @@ module IOStreams
219
219
 
220
220
  # Shortcut method if caller has a filename already with no other streams applied:
221
221
  def read_file(file_name)
222
- ::File.open(file_name, 'wb') do |file|
222
+ ::File.open(file_name, "wb") do |file|
223
223
  client.get_object(@options.merge(response_target: file, bucket: bucket_name, key: path))
224
224
  end
225
225
  end
@@ -250,7 +250,7 @@ module IOStreams
250
250
  obj = s3.bucket(bucket_name).object(path)
251
251
  obj.upload_file(file_name)
252
252
  else
253
- ::File.open(file_name, 'rb') do |file|
253
+ ::File.open(file_name, "rb") do |file|
254
254
  client.put_object(@options.merge(bucket: bucket_name, key: path, body: file))
255
255
  end
256
256
  end
@@ -267,7 +267,7 @@ module IOStreams
267
267
  return
268
268
  end
269
269
 
270
- prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/}, '')
270
+ prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/}, "")
271
271
  token = nil
272
272
  loop do
273
273
  # Fetches upto 1,000 entries at a time
@@ -1,4 +1,4 @@
1
- require 'open3'
1
+ require "open3"
2
2
 
3
3
  module IOStreams
4
4
  module Paths
@@ -29,8 +29,8 @@ module IOStreams
29
29
  attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds
30
30
  end
31
31
 
32
- @sftp_bin = 'sftp'
33
- @sshpass_bin = 'sshpass'
32
+ @sftp_bin = "sftp"
33
+ @sshpass_bin = "sshpass"
34
34
  @sshpass_wait_seconds = 5
35
35
 
36
36
  attr_reader :hostname, :username, :ssh_options, :url, :port
@@ -76,7 +76,7 @@ module IOStreams
76
76
  # end
77
77
  def initialize(url, username: nil, password: nil, ssh_options: {})
78
78
  uri = Utils::URI.new(url)
79
- raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
79
+ raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == "sftp"
80
80
 
81
81
  @hostname = uri.hostname
82
82
  @mkdir = false
@@ -133,6 +133,7 @@ module IOStreams
133
133
  Net::SFTP.start(hostname, username, build_ssh_options) do |sftp|
134
134
  sftp.dir.glob(".", pattern, flags) do |path|
135
135
  next if !directories && !path.file?
136
+
136
137
  new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password, **ssh_options)
137
138
  yield(new_path, path.attributes.attributes)
138
139
  end
@@ -168,13 +169,20 @@ module IOStreams
168
169
  # Give time for password to be processed and stdin to be passed to sftp process.
169
170
  sleep self.class.sshpass_wait_seconds
170
171
  writer.puts "get #{remote_file_name} #{local_file_name}"
171
- writer.puts 'bye'
172
+ writer.puts "bye"
172
173
  writer.close
173
174
  out = reader.read.chomp
174
- raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
175
+ unless waith_thr.value.success?
176
+ raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
177
+ end
178
+
175
179
  out
176
180
  rescue Errno::EPIPE
177
- out = reader.read.chomp rescue nil
181
+ out = begin
182
+ reader.read.chomp
183
+ rescue StandardError
184
+ nil
185
+ end
178
186
  raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
179
187
  end
180
188
  end
@@ -189,13 +197,20 @@ module IOStreams
189
197
  # Give time for password to be processed and stdin to be passed to sftp process.
190
198
  sleep self.class.sshpass_wait_seconds
191
199
  writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
192
- writer.puts 'bye'
200
+ writer.puts "bye"
193
201
  writer.close
194
202
  out = reader.read.chomp
195
- raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
203
+ unless waith_thr.value.success?
204
+ raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
205
+ end
206
+
196
207
  out
197
208
  rescue Errno::EPIPE
198
- out = reader.read.chomp rescue nil
209
+ out = begin
210
+ reader.read.chomp
211
+ rescue StandardError
212
+ nil
213
+ end
199
214
  raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
200
215
  end
201
216
  end
@@ -203,15 +218,15 @@ module IOStreams
203
218
  end
204
219
 
205
220
  def with_sftp_args
206
- return yield sftp_args(ssh_options) unless ssh_options.key?('IdentityKey')
221
+ return yield sftp_args(ssh_options) unless ssh_options.key?("IdentityKey")
207
222
 
208
- Utils.temp_file_name('iostreams-sftp-args', 'key') do |file_name|
223
+ Utils.temp_file_name("iostreams-sftp-args", "key") do |file_name|
209
224
  options = ssh_options.dup
210
- key = options.delete('IdentityKey')
225
+ key = options.delete("IdentityKey")
211
226
  # sftp requires that private key is only readable by the current user
212
- ::File.open(file_name, 'wb', 0600) { |io| io.write(key) }
227
+ ::File.open(file_name, "wb", 0o600) { |io| io.write(key) }
213
228
 
214
- options['IdentityFile'] = file_name
229
+ options["IdentityFile"] = file_name
215
230
  yield sftp_args(options)
216
231
  end
217
232
  end
@@ -228,22 +243,22 @@ module IOStreams
228
243
  args << "-oBatchMode=yes"
229
244
  args << "-oPasswordAuthentication=no"
230
245
  end
231
- args << "-oIdentitiesOnly=yes" if ssh_options.key?('IdentityFile')
246
+ args << "-oIdentitiesOnly=yes" if ssh_options.key?("IdentityFile")
232
247
  # Default is ask, but this is non-interactive so make the default fail without asking.
233
- args << "-oStrictHostKeyChecking=yes" unless ssh_options.key?('StrictHostKeyChecking')
234
- args << "-oLogLevel=#{map_log_level}" unless ssh_options.key?('LogLevel')
248
+ args << "-oStrictHostKeyChecking=yes" unless ssh_options.key?("StrictHostKeyChecking")
249
+ args << "-oLogLevel=#{map_log_level}" unless ssh_options.key?("LogLevel")
235
250
  args << "-oPort=#{port}" unless port == 22
236
251
  ssh_options.each_pair { |key, value| args << "-o#{key}=#{value}" }
237
- args << '-b'
238
- args << '-'
252
+ args << "-b"
253
+ args << "-"
239
254
  args << "#{username}@#{hostname}"
240
255
  args
241
256
  end
242
257
 
243
258
  def build_ssh_options
244
- options = ssh_options.dup
245
- options[:logger] ||= self.logger if defined?(SemanticLogger)
246
- options[:port] ||= port
259
+ options = ssh_options.dup
260
+ options[:logger] ||= logger if defined?(SemanticLogger)
261
+ options[:port] ||= port
247
262
  options[:max_pkt_size] ||= 65_536
248
263
  options[:password] ||= @password
249
264
  options