iostreams 1.0.0.beta3 → 1.0.0.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ecd282f3e3fb3dbbb1cc912458ad80f1286a4921bc36c1797d84ef3863a9b57
-  data.tar.gz: 8f695a423c861f7899d0a20b0be370787486d83f2c2741c03a71fc801ae06e44
+  metadata.gz: 1f3b07c0b0634bbc94434c4fb9de6fe4a41de44c9a8a069e8e53922a80a1e560
+  data.tar.gz: d4d7cd4fbb3fedcdf8b6df763d8eace74a075c389895231171e5b41d5fefe975
 SHA512:
-  metadata.gz: 2f46b8cfccf5b45099465ed4d9169a124fe427d55c0168d03c22fedb73e092d3e41e1f481c955276f04b02377041a9e277bd40c9732e6521cfd368afa7f6b935
-  data.tar.gz: 18cc3db574ed6a61005f4743b74da70f787f740e039dda2127d62a02f150e1b90fc7555f6d64e0bb436258c8ef1d2bab64d7fb49615cb6262b1b5d41469e3211
+  metadata.gz: 2beb8a498df91a5b564858a1f7deb5bcf139a138ceb2373a8d42a0d7c0a60f913d7675a179e558136272eacb2b25d0194aee7cb482732440bda5da6502123fe7
+  data.tar.gz: a6891d61d2910ad1787213c5b0e8c1e5de3f9147c6ab43b267910878ecba8b97fbe753e643ceeaa369f7969b584780dc25a1137e0ba844e01d46fad1b6b97882

data/lib/io_streams/paths/s3.rb

@@ -134,6 +134,9 @@ module IOStreams
         key          = uri.path.sub(%r{\A/}, '')
         @client      = client || ::Aws::S3::Client.new
         @options     = args
+
+        URI.decode_www_form(uri.query).each { |key, value| @options[key] = value } if uri.query
+
         super(key)
       end
 
@@ -193,7 +196,7 @@ module IOStreams
         Utils.temp_file_name("iostreams_s3") do |file_name|
           read_file(file_name)
 
-          ::File.open(file_name, 'rb') { |io| io.read }
+          ::File.open(file_name, 'rb', &block)
         end
       end
 

data/lib/io_streams/paths/sftp.rb

@@ -13,7 +13,7 @@ module IOStreams
       @sshpass_bin          = 'sshpass'
       @sshpass_wait_seconds = 5
 
-      attr_reader :hostname, :username, :ssh_options, :url
+      attr_reader :hostname, :username, :ssh_options, :url, :port
 
       # Stream to a remote file over sftp.
       #
@@ -58,13 +58,17 @@ module IOStreams
         uri = URI.parse(url)
         raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
 
-        @hostname    = uri.hostname
-        @mkdir       = false
-        @username    = username || uri.user
-        @url         = url
-        @password    = password || uri.password
-        @port        = uri.port || 22
-        @ssh_options = ssh_options
+        @hostname = uri.hostname
+        @mkdir    = false
+        @username = username || uri.user
+        @url      = url
+        @password = password || uri.password
+        @port     = uri.port || 22
+        # Not Ruby 2.5 yet: transform_keys(&:to_s)
+        @ssh_options = {}
+        ssh_options.each_pair { |key, value| @ssh_options[key.to_s] = value }
+
+        URI.decode_www_form(uri.query).each { |key, value| @ssh_options[key] = value } if uri.query
 
         super(uri.path)
       end
@@ -152,37 +156,78 @@ module IOStreams
 
       # Use sftp and sshpass executables to download to a local file
       def sftp_download(remote_file_name, local_file_name)
-        Open3.popen2e(*sftp_args) do |writer, reader, waith_thr|
-          writer.puts password
-          # Give time for password to be processed and stdin to be passed to sftp process.
-          sleep self.class.sshpass_wait_seconds
-          writer.puts "get #{remote_file_name} #{local_file_name}"
-          writer.puts 'bye'
-          writer.close
-          out = reader.read.chomp
-          raise(Errors::CommunicationsFailure, "Failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
-          out
+        with_sftp_args do |args|
+          Open3.popen2e(*args) do |writer, reader, waith_thr|
+            begin
+              writer.puts password
+              # Give time for password to be processed and stdin to be passed to sftp process.
+              sleep self.class.sshpass_wait_seconds
+              writer.puts "get #{remote_file_name} #{local_file_name}"
+              writer.puts 'bye'
+              writer.close
+              out = reader.read.chomp
+              raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
+              out
+            rescue Errno::EPIPE
+              out = reader.read.chomp rescue nil
+              raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
+            end
+          end
         end
       end
 
       def sftp_upload(local_file_name, remote_file_name)
-        Open3.popen2e(*sftp_args) do |writer, reader, waith_thr|
-          writer.puts(password) if password
-          # Give time for password to be processed and stdin to be passed to sftp process.
-          sleep self.class.sshpass_wait_seconds
-          writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
-          writer.puts 'bye'
-          writer.close
-          out = reader.read.chomp
-          raise(Errors::CommunicationsFailure, "Failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
-          out
+        with_sftp_args do |args|
+          Open3.popen2e(*args) do |writer, reader, waith_thr|
+            begin
+              writer.puts(password) if password
+              # Give time for password to be processed and stdin to be passed to sftp process.
+              sleep self.class.sshpass_wait_seconds
+              writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
+              writer.puts 'bye'
+              writer.close
+              out = reader.read.chomp
+              raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
+              out
+            rescue Errno::EPIPE
+              out = reader.read.chomp rescue nil
+              raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
+            end
+          end
         end
       end
 
-      def sftp_args
-        args = [self.class.sshpass_bin, self.class.sftp_bin, '-oBatchMode=no']
-        # Force it to use the password when supplied.
-        args << "-oPubkeyAuthentication=no" if password
+      def with_sftp_args
+        return yield sftp_args(ssh_options) unless ssh_options.key?('IdentityKey')
+
+        Utils.temp_file_name('iostreams-sftp-args', 'key') do |file_name|
+          options = ssh_options.dup
+          key     = options.delete('IdentityKey')
+          # sftp requires that private key is only readable by the current user
+          File.open(file_name, 'wb', 0600) { |io| io.write(key) }
+
+          options['IdentityFile'] = file_name
+          yield sftp_args(ssh_options)
+        end
+      end
+
+      def sftp_args(ssh_options)
+        args = [self.class.sshpass_bin, self.class.sftp_bin]
+        # Force sftp to use the password when supplied,
+        # and stop sftp from prompting for a password when none was supplied.
+        if password
+          args << "-oBatchMode=no"
+          args << "-oNumberOfPasswordPrompts=1"
+          args << "-oPubkeyAuthentication=no"
+        else
+          args << "-oBatchMode=yes"
+          args << "-oPasswordAuthentication=no"
+        end
+        args << "-oIdentitiesOnly=yes" if ssh_options.key?('IdentityFile')
+        # Default is ask, but this is non-interactive so make the default fail without asking.
+        args << "-oStrictHostKeyChecking=yes" unless ssh_options.key?('StrictHostKeyChecking')
+        args << "-oLogLevel=#{map_log_level}" unless ssh_options.key?('LogLevel')
+        args << "-oPort=#{port}" unless port == 22
         ssh_options.each_pair { |key, value| args << "-o#{key}=#{value}" }
         args << '-b'
         args << '-'
@@ -193,11 +238,24 @@ module IOStreams
       def build_ssh_options
         options                = ssh_options.dup
         options[:logger]       ||= self.logger if defined?(SemanticLogger)
-        options[:port]         ||= @port
+        options[:port]         ||= port
         options[:max_pkt_size] ||= 65_536
         options[:password]     ||= @password
         options
       end
+
+      def map_log_level
+        return "INFO" unless defined?(SemanticLogger)
+
+        case logger.level
+        when :trace
+          "DEBUG3"
+        when :warn
+          "ERROR"
+        else
+          logger.level.to_s
+        end
+      end
     end
   end
 end

data/lib/io_streams/pgp.rb

@@ -275,7 +275,7 @@ module IOStreams
     # * Invalidated keys must be removed manually.
     def self.import(key:)
       version_check
-      command = "#{executable} --import"
+      command = "#{executable} --batch --import"
 
       out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
       logger&.debug { "IOStreams::Pgp.import: #{command}\n#{err}#{out}" }

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.0.0.beta3".freeze
+  VERSION = "1.0.0.beta4".freeze
 end

data/test/paths/s3_test.rb

@@ -97,6 +97,14 @@ module Paths
         end
       end
 
+      describe '#each_line' do
+        it 'reads line by line' do
+          lines = []
+          existing_path.each_line {|line| lines << line}
+          assert_equal raw.lines.collect(&:chomp), lines
+        end
+      end
+
       describe '#each_child' do
         # TODO: case_sensitive: false, directories: false, hidden: false
         let(:abd_file_names) { %w[abd/test1.txt abd/test5.file abd/extra/file.csv] }
@@ -116,7 +124,8 @@ module Paths
 
         it 'existing file returns just the file itself' do
           # Glorified exists call
-          assert_equal [each_root.join('readme').to_s], each_root.children("readme").collect(&:to_s)
+          existing_path
+          assert_equal root_path.join("test.txt").to_s, root_path.children("test.txt").first.to_s
         end
 
         it 'missing file does nothing' do

data/test/paths/sftp_test.rb

@@ -5,25 +5,21 @@ module Paths
     describe IOStreams::Paths::SFTP do
       before do
         unless ENV["SFTP_HOSTNAME"]
-          skip "Supply environment variables to test SFTP paths: SFTP_HOSTNAME, SFTP_USERNAME, SFTP_PASSWORD, and optional SFTP_DIR"
+          skip "Supply environment variables to test SFTP paths: SFTP_HOSTNAME, SFTP_USERNAME, SFTP_PASSWORD, and optional SFTP_DIR, SFTP_IDENTITY_FILE"
         end
       end
 
       let(:host_name) { ENV["SFTP_HOSTNAME"] }
       let(:username) { ENV["SFTP_USERNAME"] }
       let(:password) { ENV["SFTP_PASSWORD"] }
-      let(:ftp_dir) { ENV["SFTP_DIR"] || "iostreams_test"}
-      let(:url) { File.join("sftp://", host_name, ftp_dir) }
+      let(:ftp_dir) { ENV["SFTP_DIR"] || "iostreams_test" }
 
-      let :file_name do
-        File.join(File.dirname(__FILE__), '..', 'files', 'text.txt')
-      end
+      let(:url) { File.join("sftp://", host_name, ftp_dir) }
 
-      let :raw do
-        File.read(file_name)
-      end
+      let(:file_name) { File.join(File.dirname(__FILE__), '..', 'files', 'text.txt') }
+      let(:raw) { File.read(file_name) }
 
-      let(:root_path) { IOStreams::Paths::SFTP.new(url, username: username, password: password, ruby: false) }
+      let(:root_path) { IOStreams::Paths::SFTP.new(url, username: username, password: password) }
 
       let :existing_path do
         path = root_path.join('test.txt')
@@ -72,6 +68,31 @@ module Paths
             missing_path.write("Bad path")
           end
         end
+
+        describe 'use identity file instead of password' do
+          let :root_path do
+            IOStreams::Paths::SFTP.new(url, username: username, ssh_options: {'IdentityFile' => ENV["SFTP_IDENTITY_FILE"]} )
+          end
+
+          it 'writes' do
+            skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
+            assert_equal raw.size, write_path.writer { |io| io.write(raw) }
+            assert_equal raw, write_path.read
+          end
+        end
+
+        describe 'use identity key instead of password' do
+          let :root_path do
+            key = File.open(ENV["SFTP_IDENTITY_FILE"], 'rb', &:read)
+            IOStreams::Paths::SFTP.new(url, username: username, ssh_options: {'IdentityKey' => key})
+          end
+
+          it 'writes' do
+            skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
+            assert_equal raw.size, write_path.writer { |io| io.write(raw) }
+            assert_equal raw, write_path.read
+          end
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta3
+  version: 1.0.0.beta4
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-14 00:00:00.000000000 Z
+date: 2019-11-19 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: