iostreams 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/README.md +1 -1
 - data/Rakefile +7 -7
 - data/lib/io_streams/builder.rb +4 -3
 - data/lib/io_streams/bzip2/reader.rb +1 -1
 - data/lib/io_streams/bzip2/writer.rb +1 -1
 - data/lib/io_streams/deprecated.rb +2 -3
 - data/lib/io_streams/encode/reader.rb +5 -8
 - data/lib/io_streams/encode/writer.rb +1 -1
 - data/lib/io_streams/io_streams.rb +5 -2
 - data/lib/io_streams/line/reader.rb +2 -1
 - data/lib/io_streams/path.rb +4 -4
 - data/lib/io_streams/paths/http.rb +8 -10
 - data/lib/io_streams/paths/matcher.rb +10 -11
 - data/lib/io_streams/paths/s3.rb +6 -6
 - data/lib/io_streams/paths/sftp.rb +38 -23
 - data/lib/io_streams/pgp.rb +45 -35
 - data/lib/io_streams/pgp/reader.rb +4 -6
 - data/lib/io_streams/pgp/writer.rb +1 -2
 - data/lib/io_streams/reader.rb +2 -2
 - data/lib/io_streams/record/writer.rb +2 -4
 - data/lib/io_streams/row/writer.rb +3 -5
 - data/lib/io_streams/stream.rb +6 -6
 - data/lib/io_streams/symmetric_encryption/reader.rb +1 -3
 - data/lib/io_streams/symmetric_encryption/writer.rb +2 -6
 - data/lib/io_streams/tabular.rb +12 -10
 - data/lib/io_streams/tabular/header.rb +4 -4
 - data/lib/io_streams/tabular/parser/array.rb +2 -4
 - data/lib/io_streams/tabular/parser/csv.rb +3 -5
 - data/lib/io_streams/tabular/parser/fixed.rb +4 -3
 - data/lib/io_streams/tabular/parser/hash.rb +2 -4
 - data/lib/io_streams/tabular/parser/json.rb +2 -4
 - data/lib/io_streams/tabular/parser/psv.rb +5 -7
 - data/lib/io_streams/tabular/utility/csv_row.rb +9 -17
 - data/lib/io_streams/utils.rb +3 -3
 - data/lib/io_streams/utils/reliable_http.rb +98 -0
 - data/lib/io_streams/version.rb +1 -1
 - data/lib/io_streams/writer.rb +1 -1
 - data/lib/io_streams/xlsx/reader.rb +5 -5
 - data/lib/io_streams/zip/reader.rb +1 -1
 - data/lib/io_streams/zip/writer.rb +2 -2
 - data/lib/iostreams.rb +34 -34
 - data/test/builder_test.rb +74 -74
 - data/test/bzip2_reader_test.rb +8 -13
 - data/test/bzip2_writer_test.rb +8 -9
 - data/test/deprecated_test.rb +25 -29
 - data/test/encode_reader_test.rb +14 -18
 - data/test/encode_writer_test.rb +29 -30
 - data/test/gzip_reader_test.rb +8 -13
 - data/test/gzip_writer_test.rb +10 -11
 - data/test/io_streams_test.rb +35 -35
 - data/test/line_reader_test.rb +35 -39
 - data/test/line_writer_test.rb +8 -9
 - data/test/minimal_file_reader.rb +1 -1
 - data/test/path_test.rb +24 -24
 - data/test/paths/file_test.rb +42 -42
 - data/test/paths/http_test.rb +5 -5
 - data/test/paths/matcher_test.rb +11 -12
 - data/test/paths/s3_test.rb +44 -46
 - data/test/paths/sftp_test.rb +18 -18
 - data/test/pgp_reader_test.rb +13 -15
 - data/test/pgp_test.rb +43 -44
 - data/test/pgp_writer_test.rb +27 -28
 - data/test/record_reader_test.rb +9 -10
 - data/test/record_writer_test.rb +10 -11
 - data/test/row_reader_test.rb +5 -6
 - data/test/row_writer_test.rb +7 -8
 - data/test/stream_test.rb +60 -62
 - data/test/tabular_test.rb +111 -111
 - data/test/test_helper.rb +22 -22
 - data/test/utils_test.rb +7 -7
 - data/test/xlsx_reader_test.rb +12 -12
 - data/test/zip_reader_test.rb +14 -21
 - data/test/zip_writer_test.rb +10 -10
 - metadata +4 -3
 
    
        checksums.yaml
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            ---
         
     | 
| 
       2 
2 
     | 
    
         
             
            SHA256:
         
     | 
| 
       3 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       4 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: 1a141ad8f92a6c1387ad487fc8cf545bcc010ebc862967ffce8f79f71b7ba2d6
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: 0210b8d23390ddff6fe31133c24b88b2beb7e5e0f54d6b9df962632dc369a9ec
         
     | 
| 
       5 
5 
     | 
    
         
             
            SHA512:
         
     | 
| 
       6 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       7 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: 810573a43277573365e946205465a4a1cc87c65cf517450dc6f363f5bb7168e66a25cd2d891e346d09c4b96aa9166f369fd2bc9dc21ec2112d87bce270467ff2
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: 2304155b27a897f270263283bdf499855a81f80885b9103b6c116f135904a42d73bfb29705a8156f3bde973b7a899eba9b123073e7b140a2e502b2203869ad1f
         
     | 
    
        data/README.md
    CHANGED
    
    | 
         @@ -1,5 +1,5 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            # iostreams
         
     | 
| 
       2 
     | 
    
         
            -
            [](https://rubygems.org/gems/iostreams) [](https://travis-ci.org/rocketjob/iostreams) [](https://rubygems.org/gems/iostreams) [](http://opensource.org/licenses/Apache-2.0) ](https://rubygems.org/gems/iostreams) [](https://travis-ci.org/rocketjob/iostreams) [](https://rubygems.org/gems/iostreams) [](http://opensource.org/licenses/Apache-2.0)  [-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
         
     | 
| 
       3 
3 
     | 
    
         | 
| 
       4 
4 
     | 
    
         
             
            Input and Output streaming for Ruby.
         
     | 
| 
       5 
5 
     | 
    
         | 
    
        data/Rakefile
    CHANGED
    
    | 
         @@ -1,21 +1,21 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require  
     | 
| 
       2 
     | 
    
         
            -
            require_relative  
     | 
| 
      
 1 
     | 
    
         
            +
            require "rake/testtask"
         
     | 
| 
      
 2 
     | 
    
         
            +
            require_relative "lib/io_streams/version"
         
     | 
| 
       3 
3 
     | 
    
         | 
| 
       4 
4 
     | 
    
         
             
            task :gem do
         
     | 
| 
       5 
     | 
    
         
            -
              system  
     | 
| 
      
 5 
     | 
    
         
            +
              system "gem build iostreams.gemspec"
         
     | 
| 
       6 
6 
     | 
    
         
             
            end
         
     | 
| 
       7 
7 
     | 
    
         | 
| 
       8 
     | 
    
         
            -
            task : 
     | 
| 
      
 8 
     | 
    
         
            +
            task publish: :gem do
         
     | 
| 
       9 
9 
     | 
    
         
             
              system "git tag -a v#{IOStreams::VERSION} -m 'Tagging #{IOStreams::VERSION}'"
         
     | 
| 
       10 
     | 
    
         
            -
              system  
     | 
| 
      
 10 
     | 
    
         
            +
              system "git push --tags"
         
     | 
| 
       11 
11 
     | 
    
         
             
              system "gem push iostreams-#{IOStreams::VERSION}.gem"
         
     | 
| 
       12 
12 
     | 
    
         
             
              system "rm iostreams-#{IOStreams::VERSION}.gem"
         
     | 
| 
       13 
13 
     | 
    
         
             
            end
         
     | 
| 
       14 
14 
     | 
    
         | 
| 
       15 
15 
     | 
    
         
             
            Rake::TestTask.new(:test) do |t|
         
     | 
| 
       16 
     | 
    
         
            -
              t.pattern =  
     | 
| 
      
 16 
     | 
    
         
            +
              t.pattern = "test/**/*_test.rb"
         
     | 
| 
       17 
17 
     | 
    
         
             
              t.verbose = true
         
     | 
| 
       18 
18 
     | 
    
         
             
              t.warning = true
         
     | 
| 
       19 
19 
     | 
    
         
             
            end
         
     | 
| 
       20 
20 
     | 
    
         | 
| 
       21 
     | 
    
         
            -
            task : 
     | 
| 
      
 21 
     | 
    
         
            +
            task default: :test
         
     | 
    
        data/lib/io_streams/builder.rb
    CHANGED
    
    | 
         @@ -1,5 +1,5 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            module IOStreams
         
     | 
| 
       2 
     | 
    
         
            -
            # Build the streams that need to be applied to a path druing reading or writing.
         
     | 
| 
      
 2 
     | 
    
         
            +
              # Build the streams that need to be applied to a path druing reading or writing.
         
     | 
| 
       3 
3 
     | 
    
         
             
              class Builder
         
     | 
| 
       4 
4 
     | 
    
         
             
                attr_accessor :file_name
         
     | 
| 
       5 
5 
     | 
    
         
             
                attr_reader :streams, :options
         
     | 
| 
         @@ -61,6 +61,7 @@ module IOStreams 
     | 
|
| 
       61 
61 
     | 
    
         
             
                # Return the options set for either a stream or option.
         
     | 
| 
       62 
62 
     | 
    
         
             
                def setting(stream)
         
     | 
| 
       63 
63 
     | 
    
         
             
                  return streams[stream] if streams
         
     | 
| 
      
 64 
     | 
    
         
            +
             
     | 
| 
       64 
65 
     | 
    
         
             
                  options[stream] if options
         
     | 
| 
       65 
66 
     | 
    
         
             
                end
         
     | 
| 
       66 
67 
     | 
    
         | 
| 
         @@ -96,7 +97,7 @@ module IOStreams 
     | 
|
| 
       96 
97 
     | 
    
         | 
| 
       97 
98 
     | 
    
         
             
                # Returns the streams for the supplied file_name
         
     | 
| 
       98 
99 
     | 
    
         
             
                def parse_extensions
         
     | 
| 
       99 
     | 
    
         
            -
                  parts      = ::File.basename(file_name).split( 
     | 
| 
      
 100 
     | 
    
         
            +
                  parts      = ::File.basename(file_name).split(".")
         
     | 
| 
       100 
101 
     | 
    
         
             
                  extensions = []
         
     | 
| 
       101 
102 
     | 
    
         
             
                  while extension = parts.pop
         
     | 
| 
       102 
103 
     | 
    
         
             
                    sym = extension.downcase.to_sym
         
     | 
| 
         @@ -109,7 +110,7 @@ module IOStreams 
     | 
|
| 
       109 
110 
     | 
    
         | 
| 
       110 
111 
     | 
    
         
             
                # Executes the streams that need to be executed.
         
     | 
| 
       111 
112 
     | 
    
         
             
                def execute(type, pipeline, io_stream, &block)
         
     | 
| 
       112 
     | 
    
         
            -
                  raise(ArgumentError,  
     | 
| 
      
 113 
     | 
    
         
            +
                  raise(ArgumentError, "IOStreams call is missing mandatory block") if block.nil?
         
     | 
| 
       113 
114 
     | 
    
         | 
| 
       114 
115 
     | 
    
         
             
                  if pipeline.empty?
         
     | 
| 
       115 
116 
     | 
    
         
             
                    block.call(io_stream)
         
     | 
| 
         @@ -3,7 +3,7 @@ module IOStreams 
     | 
|
| 
       3 
3 
     | 
    
         
             
                class Reader < IOStreams::Reader
         
     | 
| 
       4 
4 
     | 
    
         
             
                  # Read from a Bzip2 stream, decompressing the contents as it is read
         
     | 
| 
       5 
5 
     | 
    
         
             
                  def self.stream(input_stream, **_args)
         
     | 
| 
       6 
     | 
    
         
            -
                    Utils.load_soft_dependency( 
     | 
| 
      
 6 
     | 
    
         
            +
                    Utils.load_soft_dependency("rbzip2", "Bzip2") unless defined?(RBzip2)
         
     | 
| 
       7 
7 
     | 
    
         | 
| 
       8 
8 
     | 
    
         
             
                    begin
         
     | 
| 
       9 
9 
     | 
    
         
             
                      io = RBzip2.default_adapter::Decompressor.new(input_stream)
         
     | 
| 
         @@ -3,7 +3,7 @@ module IOStreams 
     | 
|
| 
       3 
3 
     | 
    
         
             
                class Writer < IOStreams::Writer
         
     | 
| 
       4 
4 
     | 
    
         
             
                  # Write to a stream, compressing with Bzip2
         
     | 
| 
       5 
5 
     | 
    
         
             
                  def self.stream(input_stream, original_file_name: nil, **_args)
         
     | 
| 
       6 
     | 
    
         
            -
                    Utils.load_soft_dependency( 
     | 
| 
      
 6 
     | 
    
         
            +
                    Utils.load_soft_dependency("rbzip2", "Bzip2") unless defined?(RBzip2)
         
     | 
| 
       7 
7 
     | 
    
         | 
| 
       8 
8 
     | 
    
         
             
                    begin
         
     | 
| 
       9 
9 
     | 
    
         
             
                      io = RBzip2.default_adapter::Compressor.new(input_stream)
         
     | 
| 
         @@ -1,6 +1,6 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            module IOStreams
         
     | 
| 
       2 
     | 
    
         
            -
              UTF8_ENCODING   = Encoding.find( 
     | 
| 
       3 
     | 
    
         
            -
              BINARY_ENCODING = Encoding.find( 
     | 
| 
      
 2 
     | 
    
         
            +
              UTF8_ENCODING   = Encoding.find("UTF-8").freeze
         
     | 
| 
      
 3 
     | 
    
         
            +
              BINARY_ENCODING = Encoding.find("BINARY").freeze
         
     | 
| 
       4 
4 
     | 
    
         | 
| 
       5 
5 
     | 
    
         
             
              # Deprecated IOStreams from v0.x. Do not use, will be removed soon.
         
     | 
| 
       6 
6 
     | 
    
         
             
              module Deprecated
         
     | 
| 
         @@ -9,7 +9,6 @@ module IOStreams 
     | 
|
| 
       9 
9 
     | 
    
         
             
                end
         
     | 
| 
       10 
10 
     | 
    
         | 
| 
       11 
11 
     | 
    
         
             
                module ClassMethods
         
     | 
| 
       12 
     | 
    
         
            -
             
     | 
| 
       13 
12 
     | 
    
         
             
                  # DEPRECATED. Use `#path` or `#io`
         
     | 
| 
       14 
13 
     | 
    
         
             
                  # Examples:
         
     | 
| 
       15 
14 
     | 
    
         
             
                  #   IOStreams.path("data.zip").reader { |f| f.read(100) }
         
     | 
| 
         @@ -7,9 +7,9 @@ module IOStreams 
     | 
|
| 
       7 
7 
     | 
    
         
             
                  # Builtin strip options to apply after encoding the read data.
         
     | 
| 
       8 
8 
     | 
    
         
             
                  CLEANSE_RULES = {
         
     | 
| 
       9 
9 
     | 
    
         
             
                    # Strips all non printable characters
         
     | 
| 
       10 
     | 
    
         
            -
                    printable: 
     | 
| 
      
 10 
     | 
    
         
            +
                    printable:             ->(data, _) { data.gsub!(NOT_PRINTABLE, "") || data },
         
     | 
| 
       11 
11 
     | 
    
         
             
                    # Replaces non printable characters with the value specified in the `replace` option.
         
     | 
| 
       12 
     | 
    
         
            -
                    replace_non_printable: ->(data, replace) { data.gsub!(NOT_PRINTABLE, replace ||  
     | 
| 
      
 12 
     | 
    
         
            +
                    replace_non_printable: ->(data, replace) { data.gsub!(NOT_PRINTABLE, replace || "") || data }
         
     | 
| 
       13 
13 
     | 
    
         
             
                  }.freeze
         
     | 
| 
       14 
14 
     | 
    
         | 
| 
       15 
15 
     | 
    
         
             
                  # Read a line at a time from a file or stream
         
     | 
| 
         @@ -42,7 +42,7 @@ module IOStreams 
     | 
|
| 
       42 
42 
     | 
    
         
             
                  #     :printable Cleanse all non-printable characters except \r and \n
         
     | 
| 
       43 
43 
     | 
    
         
             
                  #     Proc/lambda    Proc to call after every read to cleanse the data
         
     | 
| 
       44 
44 
     | 
    
         
             
                  #     Default: nil
         
     | 
| 
       45 
     | 
    
         
            -
                  def initialize(input_stream, encoding:  
     | 
| 
      
 45 
     | 
    
         
            +
                  def initialize(input_stream, encoding: "UTF-8", cleaner: nil, replace: nil)
         
     | 
| 
       46 
46 
     | 
    
         
             
                    super(input_stream)
         
     | 
| 
       47 
47 
     | 
    
         | 
| 
       48 
48 
     | 
    
         
             
                    @cleaner          = self.class.extract_cleaner(cleaner)
         
     | 
| 
         @@ -51,11 +51,7 @@ module IOStreams 
     | 
|
| 
       51 
51 
     | 
    
         
             
                    @replace          = replace
         
     | 
| 
       52 
52 
     | 
    
         | 
| 
       53 
53 
     | 
    
         
             
                    # More efficient read buffering only supported when the input stream `#read` method supports it.
         
     | 
| 
       54 
     | 
    
         
            -
                    if replace.nil? && !@input_stream.method(:read).arity.between?(0, 1)
         
     | 
| 
       55 
     | 
    
         
            -
                      @read_cache_buffer = ''.encode(@encoding)
         
     | 
| 
       56 
     | 
    
         
            -
                    else
         
     | 
| 
       57 
     | 
    
         
            -
                      @read_cache_buffer = nil
         
     | 
| 
       58 
     | 
    
         
            -
                    end
         
     | 
| 
      
 54 
     | 
    
         
            +
                    @read_cache_buffer = ("".encode(@encoding) if replace.nil? && !@input_stream.method(:read).arity.between?(0, 1))
         
     | 
| 
       59 
55 
     | 
    
         
             
                  end
         
     | 
| 
       60 
56 
     | 
    
         | 
| 
       61 
57 
     | 
    
         
             
                  # Returns [String] data returned from the input stream.
         
     | 
| 
         @@ -91,6 +87,7 @@ module IOStreams 
     | 
|
| 
       91 
87 
     | 
    
         
             
                    when Symbol
         
     | 
| 
       92 
88 
     | 
    
         
             
                      proc = CLEANSE_RULES[cleaner]
         
     | 
| 
       93 
89 
     | 
    
         
             
                      raise(ArgumentError, "Invalid cleansing rule #{cleaner.inspect}") unless proc
         
     | 
| 
      
 90 
     | 
    
         
            +
             
     | 
| 
       94 
91 
     | 
    
         
             
                      proc
         
     | 
| 
       95 
92 
     | 
    
         
             
                    when Proc
         
     | 
| 
       96 
93 
     | 
    
         
             
                      cleaner
         
     | 
| 
         @@ -34,7 +34,7 @@ module IOStreams 
     | 
|
| 
       34 
34 
     | 
    
         
             
                  #     :printable Cleanse all non-printable characters except \r and \n
         
     | 
| 
       35 
35 
     | 
    
         
             
                  #     Proc/lambda    Proc to call after every read to cleanse the data
         
     | 
| 
       36 
36 
     | 
    
         
             
                  #     Default: nil
         
     | 
| 
       37 
     | 
    
         
            -
                  def initialize(output_stream, encoding:  
     | 
| 
      
 37 
     | 
    
         
            +
                  def initialize(output_stream, encoding: "UTF-8", cleaner: nil, replace: nil)
         
     | 
| 
       38 
38 
     | 
    
         
             
                    super(output_stream)
         
     | 
| 
       39 
39 
     | 
    
         | 
| 
       40 
40 
     | 
    
         
             
                    @cleaner          = ::IOStreams::Encode::Reader.send(:extract_cleaner, cleaner)
         
     | 
| 
         @@ -1,4 +1,4 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require  
     | 
| 
      
 1 
     | 
    
         
            +
            require "uri"
         
     | 
| 
       2 
2 
     | 
    
         | 
| 
       3 
3 
     | 
    
         
             
            # Streaming library for Ruby
         
     | 
| 
       4 
4 
     | 
    
         
             
            #
         
     | 
| 
         @@ -111,7 +111,7 @@ module IOStreams 
     | 
|
| 
       111 
111 
     | 
    
         
             
              #
         
     | 
| 
       112 
112 
     | 
    
         
             
              # Example:
         
     | 
| 
       113 
113 
     | 
    
         
             
              #   IOStreams.temp_file
         
     | 
| 
       114 
     | 
    
         
            -
              def self.temp_file(basename, extension = "" 
     | 
| 
      
 114 
     | 
    
         
            +
              def self.temp_file(basename, extension = "")
         
     | 
| 
       115 
115 
     | 
    
         
             
                Utils.temp_file_name(basename, extension) { |file_name| yield(Paths::File.new(file_name).stream(:none)) }
         
     | 
| 
       116 
116 
     | 
    
         
             
              end
         
     | 
| 
       117 
117 
     | 
    
         | 
| 
         @@ -258,6 +258,7 @@ module IOStreams 
     | 
|
| 
       258 
258 
     | 
    
         
             
              #   register_extension(:xls, MyXls::Reader, MyXls::Writer)
         
     | 
| 
       259 
259 
     | 
    
         
             
              def self.register_extension(extension, reader_class, writer_class)
         
     | 
| 
       260 
260 
     | 
    
         
             
                raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.nil? || extension.to_s =~ /\A\w+\Z/
         
     | 
| 
      
 261 
     | 
    
         
            +
             
     | 
| 
       261 
262 
     | 
    
         
             
                @extensions[extension.nil? ? nil : extension.to_sym] = Extension.new(reader_class, writer_class)
         
     | 
| 
       262 
263 
     | 
    
         
             
              end
         
     | 
| 
       263 
264 
     | 
    
         | 
| 
         @@ -269,6 +270,7 @@ module IOStreams 
     | 
|
| 
       269 
270 
     | 
    
         
             
              #   register_extension(:xls)
         
     | 
| 
       270 
271 
     | 
    
         
             
              def self.deregister_extension(extension)
         
     | 
| 
       271 
272 
     | 
    
         
             
                raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
         
     | 
| 
      
 273 
     | 
    
         
            +
             
     | 
| 
       272 
274 
     | 
    
         
             
                @extensions.delete(extension.to_sym)
         
     | 
| 
       273 
275 
     | 
    
         
             
              end
         
     | 
| 
       274 
276 
     | 
    
         | 
| 
         @@ -284,6 +286,7 @@ module IOStreams 
     | 
|
| 
       284 
286 
     | 
    
         
             
              #   register_scheme(:xls, MyXls::Reader, MyXls::Writer)
         
     | 
| 
       285 
287 
     | 
    
         
             
              def self.register_scheme(scheme, klass)
         
     | 
| 
       286 
288 
     | 
    
         
             
                raise(ArgumentError, "Invalid scheme #{scheme.inspect}") unless scheme.nil? || scheme.to_s =~ /\A\w+\Z/
         
     | 
| 
      
 289 
     | 
    
         
            +
             
     | 
| 
       287 
290 
     | 
    
         
             
                @schemes[scheme.nil? ? nil : scheme.to_sym] = klass
         
     | 
| 
       288 
291 
     | 
    
         
             
              end
         
     | 
| 
       289 
292 
     | 
    
         | 
| 
         @@ -94,6 +94,7 @@ module IOStreams 
     | 
|
| 
       94 
94 
     | 
    
         
             
                      initial_line_number = @line_number
         
     | 
| 
       95 
95 
     | 
    
         
             
                      while line.count(@embedded_within).odd?
         
     | 
| 
       96 
96 
     | 
    
         
             
                        raise "Unclosed quoted field on line #{initial_line_number}" if eof? || line.length > @buffer_size * 10
         
     | 
| 
      
 97 
     | 
    
         
            +
             
     | 
| 
       97 
98 
     | 
    
         
             
                        line << @delimiter
         
     | 
| 
       98 
99 
     | 
    
         
             
                        line << _readline
         
     | 
| 
       99 
100 
     | 
    
         
             
                      end
         
     | 
| 
         @@ -163,7 +164,7 @@ module IOStreams 
     | 
|
| 
       163 
164 
     | 
    
         
             
                      # Take on the encoding from the input stream
         
     | 
| 
       164 
165 
     | 
    
         
             
                      @buffer            = block.dup
         
     | 
| 
       165 
166 
     | 
    
         
             
                      # Take on the encoding from the first block that was read.
         
     | 
| 
       166 
     | 
    
         
            -
                      @read_cache_buffer =  
     | 
| 
      
 167 
     | 
    
         
            +
                      @read_cache_buffer = "".encode(block.encoding) if @use_read_cache_buffer
         
     | 
| 
       167 
168 
     | 
    
         
             
                    end
         
     | 
| 
       168 
169 
     | 
    
         | 
| 
       169 
170 
     | 
    
         
             
                    if @buffer.size > MAX_BLOCKS_MULTIPLIER * @buffer_size
         
     | 
    
        data/lib/io_streams/path.rb
    CHANGED
    
    | 
         @@ -3,7 +3,7 @@ module IOStreams 
     | 
|
| 
       3 
3 
     | 
    
         
             
                attr_accessor :path
         
     | 
| 
       4 
4 
     | 
    
         | 
| 
       5 
5 
     | 
    
         
             
                def initialize(path)
         
     | 
| 
       6 
     | 
    
         
            -
                  raise(ArgumentError,  
     | 
| 
      
 6 
     | 
    
         
            +
                  raise(ArgumentError, "Path cannot be nil") if path.nil?
         
     | 
| 
       7 
7 
     | 
    
         
             
                  raise(ArgumentError, "Path must be a string: #{path.inspect}, class: #{path.class}") unless path.is_a?(String)
         
     | 
| 
       8 
8 
     | 
    
         | 
| 
       9 
9 
     | 
    
         
             
                  @path      = path.frozen? ? path : path.dup.freeze
         
     | 
| 
         @@ -30,7 +30,7 @@ module IOStreams 
     | 
|
| 
       30 
30 
     | 
    
         
             
                end
         
     | 
| 
       31 
31 
     | 
    
         | 
| 
       32 
32 
     | 
    
         
             
                def absolute?
         
     | 
| 
       33 
     | 
    
         
            -
                  !!(path.strip =~  
     | 
| 
      
 33 
     | 
    
         
            +
                  !!(path.strip =~ %r{\A/})
         
     | 
| 
       34 
34 
     | 
    
         
             
                end
         
     | 
| 
       35 
35 
     | 
    
         | 
| 
       36 
36 
     | 
    
         
             
                # By default realpath just returns self.
         
     | 
| 
         @@ -84,12 +84,12 @@ module IOStreams 
     | 
|
| 
       84 
84 
     | 
    
         
             
                # Cleanup an incomplete write to the target "file" if the copy fails.
         
     | 
| 
       85 
85 
     | 
    
         
             
                def copy_from(source, **args)
         
     | 
| 
       86 
86 
     | 
    
         
             
                  super(source, **args)
         
     | 
| 
       87 
     | 
    
         
            -
                rescue StandardError =>  
     | 
| 
      
 87 
     | 
    
         
            +
                rescue StandardError => e
         
     | 
| 
       88 
88 
     | 
    
         
             
                  begin
         
     | 
| 
       89 
89 
     | 
    
         
             
                    delete
         
     | 
| 
       90 
90 
     | 
    
         
             
                  rescue NotImplementedError
         
     | 
| 
       91 
91 
     | 
    
         
             
                  end
         
     | 
| 
       92 
     | 
    
         
            -
                  raise( 
     | 
| 
      
 92 
     | 
    
         
            +
                  raise(e)
         
     | 
| 
       93 
93 
     | 
    
         
             
                end
         
     | 
| 
       94 
94 
     | 
    
         | 
| 
       95 
95 
     | 
    
         
             
                # Moves the file by copying it to the new path and then deleting the current path.
         
     | 
| 
         @@ -1,5 +1,5 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require  
     | 
| 
       2 
     | 
    
         
            -
            require  
     | 
| 
      
 1 
     | 
    
         
            +
            require "net/http"
         
     | 
| 
      
 2 
     | 
    
         
            +
            require "uri"
         
     | 
| 
       3 
3 
     | 
    
         
             
            module IOStreams
         
     | 
| 
       4 
4 
     | 
    
         
             
              module Paths
         
     | 
| 
       5 
5 
     | 
    
         
             
                class HTTP < IOStreams::Path
         
     | 
| 
         @@ -68,21 +68,19 @@ module IOStreams 
     | 
|
| 
       68 
68 
     | 
    
         
             
                    result = nil
         
     | 
| 
       69 
69 
     | 
    
         
             
                    raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
         
     | 
| 
       70 
70 
     | 
    
         | 
| 
       71 
     | 
    
         
            -
                    Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme ==  
     | 
| 
      
 71 
     | 
    
         
            +
                    Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
         
     | 
| 
       72 
72 
     | 
    
         
             
                      request = Net::HTTP::Get.new(uri)
         
     | 
| 
       73 
73 
     | 
    
         
             
                      request.basic_auth(username, password) if username
         
     | 
| 
       74 
74 
     | 
    
         | 
| 
       75 
75 
     | 
    
         
             
                      http.request(request) do |response|
         
     | 
| 
       76 
     | 
    
         
            -
                        if response.is_a?(Net::HTTPNotFound)
         
     | 
| 
       77 
     | 
    
         
            -
                          raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}")
         
     | 
| 
       78 
     | 
    
         
            -
                        end
         
     | 
| 
      
 76 
     | 
    
         
            +
                        raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}") if response.is_a?(Net::HTTPNotFound)
         
     | 
| 
       79 
77 
     | 
    
         
             
                        if response.is_a?(Net::HTTPUnauthorized)
         
     | 
| 
       80 
78 
     | 
    
         
             
                          raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
         
     | 
| 
       81 
79 
     | 
    
         
             
                        end
         
     | 
| 
       82 
80 
     | 
    
         | 
| 
       83 
81 
     | 
    
         
             
                        if response.is_a?(Net::HTTPRedirection)
         
     | 
| 
       84 
     | 
    
         
            -
                          new_uri = response[ 
     | 
| 
       85 
     | 
    
         
            -
                          return handle_redirects(new_uri, http_redirect_count 
     | 
| 
      
 82 
     | 
    
         
            +
                          new_uri = response["location"]
         
     | 
| 
      
 83 
     | 
    
         
            +
                          return handle_redirects(new_uri, http_redirect_count - 1, &block)
         
     | 
| 
       86 
84 
     | 
    
         
             
                        end
         
     | 
| 
       87 
85 
     | 
    
         | 
| 
       88 
86 
     | 
    
         
             
                        unless response.is_a?(Net::HTTPSuccess)
         
     | 
| 
         @@ -90,8 +88,8 @@ module IOStreams 
     | 
|
| 
       90 
88 
     | 
    
         
             
                        end
         
     | 
| 
       91 
89 
     | 
    
         | 
| 
       92 
90 
     | 
    
         
             
                        # Since Net::HTTP download only supports a push stream, write it to a tempfile first.
         
     | 
| 
       93 
     | 
    
         
            -
                        Utils.temp_file_name( 
     | 
| 
       94 
     | 
    
         
            -
                          ::File.open(file_name,  
     | 
| 
      
 91 
     | 
    
         
            +
                        Utils.temp_file_name("iostreams_http") do |file_name|
         
     | 
| 
      
 92 
     | 
    
         
            +
                          ::File.open(file_name, "wb") { |io| response.read_body { |chunk| io.write(chunk) } }
         
     | 
| 
       95 
93 
     | 
    
         
             
                          # Return a read stream
         
     | 
| 
       96 
94 
     | 
    
         
             
                          result = ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
         
     | 
| 
       97 
95 
     | 
    
         
             
                        end
         
     | 
| 
         @@ -3,7 +3,7 @@ module IOStreams 
     | 
|
| 
       3 
3 
     | 
    
         
             
                # Implement fnmatch logic for any path iterator
         
     | 
| 
       4 
4 
     | 
    
         
             
                class Matcher
         
     | 
| 
       5 
5 
     | 
    
         
             
                  # Characters indicating that pattern matching is required
         
     | 
| 
       6 
     | 
    
         
            -
                  MATCH_START_CHARS = /[*?\[{] 
     | 
| 
      
 6 
     | 
    
         
            +
                  MATCH_START_CHARS = /[*?\[{]/.freeze
         
     | 
| 
       7 
7 
     | 
    
         | 
| 
       8 
8 
     | 
    
         
             
                  attr_reader :path, :pattern, :flags
         
     | 
| 
       9 
9 
     | 
    
         | 
| 
         @@ -26,7 +26,7 @@ module IOStreams 
     | 
|
| 
       26 
26 
     | 
    
         | 
| 
       27 
27 
     | 
    
         
             
                  # Returns whether the relative `file_name` matches
         
     | 
| 
       28 
28 
     | 
    
         
             
                  def match?(file_name)
         
     | 
| 
       29 
     | 
    
         
            -
                    relative_file_name = file_name.sub(path.to_s,  
     | 
| 
      
 29 
     | 
    
         
            +
                    relative_file_name = file_name.sub(path.to_s, "").sub(%r{\A/}, "")
         
     | 
| 
       30 
30 
     | 
    
         
             
                    ::File.fnmatch?(pattern, relative_file_name, flags)
         
     | 
| 
       31 
31 
     | 
    
         
             
                  end
         
     | 
| 
       32 
32 
     | 
    
         | 
| 
         @@ -39,23 +39,22 @@ module IOStreams 
     | 
|
| 
       39 
39 
     | 
    
         
             
                  private
         
     | 
| 
       40 
40 
     | 
    
         | 
| 
       41 
41 
     | 
    
         
             
                  def extract_optimized_path(path, pattern)
         
     | 
| 
       42 
     | 
    
         
            -
                    elements = pattern.split( 
     | 
| 
      
 42 
     | 
    
         
            +
                    elements = pattern.split("/")
         
     | 
| 
       43 
43 
     | 
    
         
             
                    index    = elements.find_index { |e| e.match(MATCH_START_CHARS) }
         
     | 
| 
       44 
     | 
    
         
            -
                    if index 
     | 
| 
       45 
     | 
    
         
            -
                      # Cannot optimize path since the very first entry contains a wildcard
         
     | 
| 
       46 
     | 
    
         
            -
                      @path    = path || IOStreams.path
         
     | 
| 
       47 
     | 
    
         
            -
                      @pattern = pattern
         
     | 
| 
       48 
     | 
    
         
            -
                    elsif index.nil?
         
     | 
| 
      
 44 
     | 
    
         
            +
                    if index.nil?
         
     | 
| 
       49 
45 
     | 
    
         
             
                      # No index means it has no pattern.
         
     | 
| 
       50 
46 
     | 
    
         
             
                      @path    = path.nil? ? IOStreams.path(pattern) : path.join(pattern)
         
     | 
| 
       51 
47 
     | 
    
         
             
                      @pattern = nil
         
     | 
| 
      
 48 
     | 
    
         
            +
                    elsif index.zero?
         
     | 
| 
      
 49 
     | 
    
         
            +
                      # Cannot optimize path since the very first entry contains a wildcard
         
     | 
| 
      
 50 
     | 
    
         
            +
                      @path    = path || IOStreams.path
         
     | 
| 
      
 51 
     | 
    
         
            +
                      @pattern = pattern
         
     | 
| 
       52 
52 
     | 
    
         
             
                    else
         
     | 
| 
       53 
     | 
    
         
            -
                      new_path = elements[0..index - 1].join( 
     | 
| 
      
 53 
     | 
    
         
            +
                      new_path = elements[0..index - 1].join("/")
         
     | 
| 
       54 
54 
     | 
    
         
             
                      @path    = path.nil? ? IOStreams.path(new_path) : path.join(new_path)
         
     | 
| 
       55 
     | 
    
         
            -
                      @pattern = elements[index..-1].join( 
     | 
| 
      
 55 
     | 
    
         
            +
                      @pattern = elements[index..-1].join("/")
         
     | 
| 
       56 
56 
     | 
    
         
             
                    end
         
     | 
| 
       57 
57 
     | 
    
         
             
                  end
         
     | 
| 
       58 
     | 
    
         
            -
             
     | 
| 
       59 
58 
     | 
    
         
             
                end
         
     | 
| 
       60 
59 
     | 
    
         
             
              end
         
     | 
| 
       61 
60 
     | 
    
         
             
            end
         
     | 
    
        data/lib/io_streams/paths/s3.rb
    CHANGED
    
    | 
         @@ -131,13 +131,13 @@ module IOStreams 
     | 
|
| 
       131 
131 
     | 
    
         
             
                  # @option params [String] :object_lock_legal_hold_status
         
     | 
| 
       132 
132 
     | 
    
         
             
                  #   The Legal Hold status that you want to apply to the specified object.
         
     | 
| 
       133 
133 
     | 
    
         
             
                  def initialize(url, client: nil, access_key_id: nil, secret_access_key: nil, **args)
         
     | 
| 
       134 
     | 
    
         
            -
                    Utils.load_soft_dependency( 
     | 
| 
      
 134 
     | 
    
         
            +
                    Utils.load_soft_dependency("aws-sdk-s3", "AWS S3") unless defined?(::Aws::S3::Client)
         
     | 
| 
       135 
135 
     | 
    
         | 
| 
       136 
136 
     | 
    
         
             
                    uri = Utils::URI.new(url)
         
     | 
| 
       137 
     | 
    
         
            -
                    raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme ==  
     | 
| 
      
 137 
     | 
    
         
            +
                    raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == "s3"
         
     | 
| 
       138 
138 
     | 
    
         | 
| 
       139 
139 
     | 
    
         
             
                    @bucket_name = uri.hostname
         
     | 
| 
       140 
     | 
    
         
            -
                    key          = uri.path.sub(%r{\A/},  
     | 
| 
      
 140 
     | 
    
         
            +
                    key          = uri.path.sub(%r{\A/}, "")
         
     | 
| 
       141 
141 
     | 
    
         
             
                    if client.is_a?(Hash)
         
     | 
| 
       142 
142 
     | 
    
         
             
                      client[:access_key_id]     = access_key_id if access_key_id
         
     | 
| 
       143 
143 
     | 
    
         
             
                      client[:secret_access_key] = secret_access_key if secret_access_key
         
     | 
| 
         @@ -219,7 +219,7 @@ module IOStreams 
     | 
|
| 
       219 
219 
     | 
    
         | 
| 
       220 
220 
     | 
    
         
             
                  # Shortcut method if caller has a filename already with no other streams applied:
         
     | 
| 
       221 
221 
     | 
    
         
             
                  def read_file(file_name)
         
     | 
| 
       222 
     | 
    
         
            -
                    ::File.open(file_name,  
     | 
| 
      
 222 
     | 
    
         
            +
                    ::File.open(file_name, "wb") do |file|
         
     | 
| 
       223 
223 
     | 
    
         
             
                      client.get_object(@options.merge(response_target: file, bucket: bucket_name, key: path))
         
     | 
| 
       224 
224 
     | 
    
         
             
                    end
         
     | 
| 
       225 
225 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -250,7 +250,7 @@ module IOStreams 
     | 
|
| 
       250 
250 
     | 
    
         
             
                      obj = s3.bucket(bucket_name).object(path)
         
     | 
| 
       251 
251 
     | 
    
         
             
                      obj.upload_file(file_name)
         
     | 
| 
       252 
252 
     | 
    
         
             
                    else
         
     | 
| 
       253 
     | 
    
         
            -
                      ::File.open(file_name,  
     | 
| 
      
 253 
     | 
    
         
            +
                      ::File.open(file_name, "rb") do |file|
         
     | 
| 
       254 
254 
     | 
    
         
             
                        client.put_object(@options.merge(bucket: bucket_name, key: path, body: file))
         
     | 
| 
       255 
255 
     | 
    
         
             
                      end
         
     | 
| 
       256 
256 
     | 
    
         
             
                    end
         
     | 
| 
         @@ -267,7 +267,7 @@ module IOStreams 
     | 
|
| 
       267 
267 
     | 
    
         
             
                      return
         
     | 
| 
       268 
268 
     | 
    
         
             
                    end
         
     | 
| 
       269 
269 
     | 
    
         | 
| 
       270 
     | 
    
         
            -
                    prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/},  
     | 
| 
      
 270 
     | 
    
         
            +
                    prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/}, "")
         
     | 
| 
       271 
271 
     | 
    
         
             
                    token  = nil
         
     | 
| 
       272 
272 
     | 
    
         
             
                    loop do
         
     | 
| 
       273 
273 
     | 
    
         
             
                      # Fetches upto 1,000 entries at a time
         
     | 
| 
         @@ -1,4 +1,4 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require  
     | 
| 
      
 1 
     | 
    
         
            +
            require "open3"
         
     | 
| 
       2 
2 
     | 
    
         | 
| 
       3 
3 
     | 
    
         
             
            module IOStreams
         
     | 
| 
       4 
4 
     | 
    
         
             
              module Paths
         
     | 
| 
         @@ -29,8 +29,8 @@ module IOStreams 
     | 
|
| 
       29 
29 
     | 
    
         
             
                    attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds
         
     | 
| 
       30 
30 
     | 
    
         
             
                  end
         
     | 
| 
       31 
31 
     | 
    
         | 
| 
       32 
     | 
    
         
            -
                  @sftp_bin             =  
     | 
| 
       33 
     | 
    
         
            -
                  @sshpass_bin          =  
     | 
| 
      
 32 
     | 
    
         
            +
                  @sftp_bin             = "sftp"
         
     | 
| 
      
 33 
     | 
    
         
            +
                  @sshpass_bin          = "sshpass"
         
     | 
| 
       34 
34 
     | 
    
         
             
                  @sshpass_wait_seconds = 5
         
     | 
| 
       35 
35 
     | 
    
         | 
| 
       36 
36 
     | 
    
         
             
                  attr_reader :hostname, :username, :ssh_options, :url, :port
         
     | 
| 
         @@ -76,7 +76,7 @@ module IOStreams 
     | 
|
| 
       76 
76 
     | 
    
         
             
                  #   end
         
     | 
| 
       77 
77 
     | 
    
         
             
                  def initialize(url, username: nil, password: nil, ssh_options: {})
         
     | 
| 
       78 
78 
     | 
    
         
             
                    uri = Utils::URI.new(url)
         
     | 
| 
       79 
     | 
    
         
            -
                    raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme ==  
     | 
| 
      
 79 
     | 
    
         
            +
                    raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == "sftp"
         
     | 
| 
       80 
80 
     | 
    
         | 
| 
       81 
81 
     | 
    
         
             
                    @hostname = uri.hostname
         
     | 
| 
       82 
82 
     | 
    
         
             
                    @mkdir    = false
         
     | 
| 
         @@ -133,6 +133,7 @@ module IOStreams 
     | 
|
| 
       133 
133 
     | 
    
         
             
                    Net::SFTP.start(hostname, username, build_ssh_options) do |sftp|
         
     | 
| 
       134 
134 
     | 
    
         
             
                      sftp.dir.glob(".", pattern, flags) do |path|
         
     | 
| 
       135 
135 
     | 
    
         
             
                        next if !directories && !path.file?
         
     | 
| 
      
 136 
     | 
    
         
            +
             
     | 
| 
       136 
137 
     | 
    
         
             
                        new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password, **ssh_options)
         
     | 
| 
       137 
138 
     | 
    
         
             
                        yield(new_path, path.attributes.attributes)
         
     | 
| 
       138 
139 
     | 
    
         
             
                      end
         
     | 
| 
         @@ -168,13 +169,20 @@ module IOStreams 
     | 
|
| 
       168 
169 
     | 
    
         
             
                          # Give time for password to be processed and stdin to be passed to sftp process.
         
     | 
| 
       169 
170 
     | 
    
         
             
                          sleep self.class.sshpass_wait_seconds
         
     | 
| 
       170 
171 
     | 
    
         
             
                          writer.puts "get #{remote_file_name} #{local_file_name}"
         
     | 
| 
       171 
     | 
    
         
            -
                          writer.puts  
     | 
| 
      
 172 
     | 
    
         
            +
                          writer.puts "bye"
         
     | 
| 
       172 
173 
     | 
    
         
             
                          writer.close
         
     | 
| 
       173 
174 
     | 
    
         
             
                          out = reader.read.chomp
         
     | 
| 
       174 
     | 
    
         
            -
                           
     | 
| 
      
 175 
     | 
    
         
            +
                          unless waith_thr.value.success?
         
     | 
| 
      
 176 
     | 
    
         
            +
                            raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
         
     | 
| 
      
 177 
     | 
    
         
            +
                          end
         
     | 
| 
      
 178 
     | 
    
         
            +
             
     | 
| 
       175 
179 
     | 
    
         
             
                          out
         
     | 
| 
       176 
180 
     | 
    
         
             
                        rescue Errno::EPIPE
         
     | 
| 
       177 
     | 
    
         
            -
                          out =  
     | 
| 
      
 181 
     | 
    
         
            +
                          out = begin
         
     | 
| 
      
 182 
     | 
    
         
            +
                                  reader.read.chomp
         
     | 
| 
      
 183 
     | 
    
         
            +
                                rescue StandardError
         
     | 
| 
      
 184 
     | 
    
         
            +
                                  nil
         
     | 
| 
      
 185 
     | 
    
         
            +
                                end
         
     | 
| 
       178 
186 
     | 
    
         
             
                          raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
         
     | 
| 
       179 
187 
     | 
    
         
             
                        end
         
     | 
| 
       180 
188 
     | 
    
         
             
                      end
         
     | 
| 
         @@ -189,13 +197,20 @@ module IOStreams 
     | 
|
| 
       189 
197 
     | 
    
         
             
                          # Give time for password to be processed and stdin to be passed to sftp process.
         
     | 
| 
       190 
198 
     | 
    
         
             
                          sleep self.class.sshpass_wait_seconds
         
     | 
| 
       191 
199 
     | 
    
         
             
                          writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
         
     | 
| 
       192 
     | 
    
         
            -
                          writer.puts  
     | 
| 
      
 200 
     | 
    
         
            +
                          writer.puts "bye"
         
     | 
| 
       193 
201 
     | 
    
         
             
                          writer.close
         
     | 
| 
       194 
202 
     | 
    
         
             
                          out = reader.read.chomp
         
     | 
| 
       195 
     | 
    
         
            -
                           
     | 
| 
      
 203 
     | 
    
         
            +
                          unless waith_thr.value.success?
         
     | 
| 
      
 204 
     | 
    
         
            +
                            raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
         
     | 
| 
      
 205 
     | 
    
         
            +
                          end
         
     | 
| 
      
 206 
     | 
    
         
            +
             
     | 
| 
       196 
207 
     | 
    
         
             
                          out
         
     | 
| 
       197 
208 
     | 
    
         
             
                        rescue Errno::EPIPE
         
     | 
| 
       198 
     | 
    
         
            -
                          out =  
     | 
| 
      
 209 
     | 
    
         
            +
                          out = begin
         
     | 
| 
      
 210 
     | 
    
         
            +
                                  reader.read.chomp
         
     | 
| 
      
 211 
     | 
    
         
            +
                                rescue StandardError
         
     | 
| 
      
 212 
     | 
    
         
            +
                                  nil
         
     | 
| 
      
 213 
     | 
    
         
            +
                                end
         
     | 
| 
       199 
214 
     | 
    
         
             
                          raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
         
     | 
| 
       200 
215 
     | 
    
         
             
                        end
         
     | 
| 
       201 
216 
     | 
    
         
             
                      end
         
     | 
| 
         @@ -203,15 +218,15 @@ module IOStreams 
     | 
|
| 
       203 
218 
     | 
    
         
             
                  end
         
     | 
| 
       204 
219 
     | 
    
         | 
| 
       205 
220 
     | 
    
         
             
                  def with_sftp_args
         
     | 
| 
       206 
     | 
    
         
            -
                    return yield sftp_args(ssh_options) unless ssh_options.key?( 
     | 
| 
      
 221 
     | 
    
         
            +
                    return yield sftp_args(ssh_options) unless ssh_options.key?("IdentityKey")
         
     | 
| 
       207 
222 
     | 
    
         | 
| 
       208 
     | 
    
         
            -
                    Utils.temp_file_name( 
     | 
| 
      
 223 
     | 
    
         
            +
                    Utils.temp_file_name("iostreams-sftp-args", "key") do |file_name|
         
     | 
| 
       209 
224 
     | 
    
         
             
                      options = ssh_options.dup
         
     | 
| 
       210 
     | 
    
         
            -
                      key     = options.delete( 
     | 
| 
      
 225 
     | 
    
         
            +
                      key     = options.delete("IdentityKey")
         
     | 
| 
       211 
226 
     | 
    
         
             
                      # sftp requires that private key is only readable by the current user
         
     | 
| 
       212 
     | 
    
         
            -
                      ::File.open(file_name,  
     | 
| 
      
 227 
     | 
    
         
            +
                      ::File.open(file_name, "wb", 0o600) { |io| io.write(key) }
         
     | 
| 
       213 
228 
     | 
    
         | 
| 
       214 
     | 
    
         
            -
                      options[ 
     | 
| 
      
 229 
     | 
    
         
            +
                      options["IdentityFile"] = file_name
         
     | 
| 
       215 
230 
     | 
    
         
             
                      yield sftp_args(options)
         
     | 
| 
       216 
231 
     | 
    
         
             
                    end
         
     | 
| 
       217 
232 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -228,22 +243,22 @@ module IOStreams 
     | 
|
| 
       228 
243 
     | 
    
         
             
                      args << "-oBatchMode=yes"
         
     | 
| 
       229 
244 
     | 
    
         
             
                      args << "-oPasswordAuthentication=no"
         
     | 
| 
       230 
245 
     | 
    
         
             
                    end
         
     | 
| 
       231 
     | 
    
         
            -
                    args << "-oIdentitiesOnly=yes" if ssh_options.key?( 
     | 
| 
      
 246 
     | 
    
         
            +
                    args << "-oIdentitiesOnly=yes" if ssh_options.key?("IdentityFile")
         
     | 
| 
       232 
247 
     | 
    
         
             
                    # Default is ask, but this is non-interactive so make the default fail without asking.
         
     | 
| 
       233 
     | 
    
         
            -
                    args << "-oStrictHostKeyChecking=yes" unless ssh_options.key?( 
     | 
| 
       234 
     | 
    
         
            -
                    args << "-oLogLevel=#{map_log_level}" unless ssh_options.key?( 
     | 
| 
      
 248 
     | 
    
         
            +
                    args << "-oStrictHostKeyChecking=yes" unless ssh_options.key?("StrictHostKeyChecking")
         
     | 
| 
      
 249 
     | 
    
         
            +
                    args << "-oLogLevel=#{map_log_level}" unless ssh_options.key?("LogLevel")
         
     | 
| 
       235 
250 
     | 
    
         
             
                    args << "-oPort=#{port}" unless port == 22
         
     | 
| 
       236 
251 
     | 
    
         
             
                    ssh_options.each_pair { |key, value| args << "-o#{key}=#{value}" }
         
     | 
| 
       237 
     | 
    
         
            -
                    args <<  
     | 
| 
       238 
     | 
    
         
            -
                    args <<  
     | 
| 
      
 252 
     | 
    
         
            +
                    args << "-b"
         
     | 
| 
      
 253 
     | 
    
         
            +
                    args << "-"
         
     | 
| 
       239 
254 
     | 
    
         
             
                    args << "#{username}@#{hostname}"
         
     | 
| 
       240 
255 
     | 
    
         
             
                    args
         
     | 
| 
       241 
256 
     | 
    
         
             
                  end
         
     | 
| 
       242 
257 
     | 
    
         | 
| 
       243 
258 
     | 
    
         
             
                  def build_ssh_options
         
     | 
| 
       244 
     | 
    
         
            -
                    options 
     | 
| 
       245 
     | 
    
         
            -
                    options[:logger] 
     | 
| 
       246 
     | 
    
         
            -
                    options[:port] 
     | 
| 
      
 259 
     | 
    
         
            +
                    options = ssh_options.dup
         
     | 
| 
      
 260 
     | 
    
         
            +
                    options[:logger] ||= logger if defined?(SemanticLogger)
         
     | 
| 
      
 261 
     | 
    
         
            +
                    options[:port] ||= port
         
     | 
| 
       247 
262 
     | 
    
         
             
                    options[:max_pkt_size] ||= 65_536
         
     | 
| 
       248 
263 
     | 
    
         
             
                    options[:password]     ||= @password
         
     | 
| 
       249 
264 
     | 
    
         
             
                    options
         
     |