iostreams 1.0.0.beta2 → 1.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4469de3d596bf7fd0485de78878ad8188c0a60c8a3b52cfbd2dcd207ec4e8e92
-  data.tar.gz: 25e2e16fa02f1d8b964cd8bdb521576a106148fd6446ebddf832735e5006d403
+  metadata.gz: 5ecd282f3e3fb3dbbb1cc912458ad80f1286a4921bc36c1797d84ef3863a9b57
+  data.tar.gz: 8f695a423c861f7899d0a20b0be370787486d83f2c2741c03a71fc801ae06e44
 SHA512:
-  metadata.gz: 37456322483004e740a678a46a32bae7dc1fd02d9ad1363a0ce02c95502cb0fb68d07295280193277b5b252b3ae39162a019ba3f85f1483c3e5503011ae68419
-  data.tar.gz: ea9cd84e99f50b7771d7429f78da254b15ffd1b08cb7cadfa4cb7c9033bb3cc6f59064addd3824e4bcf5f1072669268916c16cd2116d7e5b6e822bd34f6f4415
+  metadata.gz: 2f46b8cfccf5b45099465ed4d9169a124fe427d55c0168d03c22fedb73e092d3e41e1f481c955276f04b02377041a9e277bd40c9732e6521cfd368afa7f6b935
+  data.tar.gz: 18cc3db574ed6a61005f4743b74da70f787f740e039dda2127d62a02f150e1b90fc7555f6d64e0bb436258c8ef1d2bab64d7fb49615cb6262b1b5d41469e3211

data/lib/io_streams/bzip2/reader.rb

@@ -3,7 +3,7 @@ module IOStreams
     class Reader < IOStreams::Reader
       # Read from a Bzip2 stream, decompressing the contents as it is read
       def self.stream(input_stream, **_args)
-        Utils.load_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
+        Utils.load_soft_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
 
         begin
           io = RBzip2.default_adapter::Decompressor.new(input_stream)

data/lib/io_streams/bzip2/writer.rb

@@ -3,7 +3,7 @@ module IOStreams
     class Writer < IOStreams::Writer
       # Write to a stream, compressing with Bzip2
       def self.stream(input_stream, original_file_name: nil, **_args)
-        Utils.load_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
+        Utils.load_soft_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
 
         begin
           io = RBzip2.default_adapter::Compressor.new(input_stream)

data/lib/io_streams/io_streams.rb

@@ -99,10 +99,20 @@ module IOStreams
     root(root).join(*elements)
   end
 
-  # Returns a path to a local temporary file.
-  def self.temp_file(*args, &block)
-    # TODO: Possible enhancement: Add a :temp root so that temp files can be stored anywhere, or the location changed.
-    Paths::File.temp_file(*args, &block)
+  # Returns a path to a temporary file.
+  # Temporary file is deleted upon block completion if present.
+  #
+  # Parameters:
+  #   basename: [String]
+  #     Base file name to include in the temp file name.
+  #
+  #   extension: [String]
+  #     Optional extension to add to the tempfile.
+  #
+  # Example:
+  #   IOStreams.temp_file
+  def self.temp_file(basename, extension = "", &block)
+    Utils.temp_file_name(basename, extension) { |file_name| yield(Paths::File.new(file_name).stream(:none)) }
   end
 
   # Returns [IOStreams::Paths::File] current or named users home path

data/lib/io_streams/path.rb

@@ -85,7 +85,10 @@ module IOStreams
     def copy_from(source, **args)
       super(source, **args)
     rescue StandardError => exc
-      delete
+      begin
+        delete
+      rescue NotImplementedError
+      end
       raise(exc)
     end
 

data/lib/io_streams/paths/file.rb

@@ -3,12 +3,6 @@ require "fileutils"
 module IOStreams
   module Paths
     class File < IOStreams::Path
-      # Returns a path to a temporary file.
-      # Temporary file is deleted upon block completion if present.
-      def self.temp_file(basename, extension = "")
-        Utils.temp_file_name(basename, extension) { |file_name| yield(new(file_name).stream(:none)) }
-      end
-
       # Yields Paths within the current path.
       #
       # Examples:

data/lib/io_streams/paths/s3.rb

@@ -125,7 +125,7 @@ module IOStreams
       # @option params [String] :object_lock_legal_hold_status
       #   The Legal Hold status that you want to apply to the specified object.
       def initialize(url, client: nil, **args)
-        Utils.load_dependency('aws-sdk-s3', 'AWS S3') unless defined?(::Aws::S3::Client)
+        Utils.load_soft_dependency('aws-sdk-s3', 'AWS S3') unless defined?(::Aws::S3::Client)
 
         uri = URI.parse(url)
         raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == 's3'

data/lib/io_streams/paths/sftp.rb

@@ -1,9 +1,19 @@
+require 'open3'
+
 module IOStreams
   module Paths
     class SFTP < IOStreams::Path
       include SemanticLogger::Loggable if defined?(SemanticLogger)
 
-      attr_reader :hostname, :username, :create_path, :options, :url
+      class << self
+        attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds
+      end
+
+      @sftp_bin             = 'sftp'
+      @sshpass_bin          = 'sshpass'
+      @sshpass_wait_seconds = 5
+
+      attr_reader :hostname, :username, :ssh_options, :url
 
       # Stream to a remote file over sftp.
       #
@@ -16,40 +26,46 @@ module IOStreams
       # password: [String]
       #   Password for the user.
       #
-      # host: [String]
-      #   Name of the host to connect to.
-      #
-      # port: [Integer]
-      #   Port to connect to at the above host.
-      #
-      # **args
-      #   Any other options supported by Net::SSH.start
+      # **ssh_options
+      #   Any other options supported by ssh_config.
+      #   `man ssh_config` to see all available options.
       #
       # Examples:
       #
-      # # Sample URL
-      #   sftp://hostname/path/file_name
+      #   # Display the contents of a remote file
+      #   IOStreams.path("sftp://test.com/path/file_name.csv", username: "jack", password: "OpenSesame").reader do |io|
+      #     puts io.read
+      #   end
       #
-      # # Full url showing all the optional elements that can be set via the url:
+      #   # Full url showing all the optional elements that can be set via the url:
       #   sftp://username:password@hostname:22/path/file_name
-      def initialize(url, username:, password:, port: nil, max_pkt_size: 65_536, logger: nil, create_path: false, **args)
-        Utils.load_dependency('net-sftp', 'net/sftp') unless defined?(Net::SFTP)
-
+      #
+      #   # Display the contents of a remote file, supplying the username and password in the url:
+      #   IOStreams.path("sftp://jack:OpenSesame@test.com:22/path/file_name.csv").reader do |io|
+      #     puts io.read
+      #   end
+      #
+      #   # Display the contents of a remote file, supplying the username and password as arguments:
+      #   IOStreams.path("sftp://test.com/path/file_name.csv", username: "jack", password: "OpenSesame").reader do |io|
+      #     puts io.read
+      #   end
+      #
+      #   # When using the sftp executable use an identity file instead of a password to authenticate:
+      #   IOStreams.path("sftp://test.com/path/file_name.csv", username: "jack", IdentityFile: "~/.ssh/private_key").reader do |io|
+      #     puts io.read
+      #   end
+      def initialize(url, username: nil, password: nil, ruby: true, ssh_options: {})
         uri = URI.parse(url)
         raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
 
-        @hostname              = uri.hostname
-        @mkdir                 = false
-        @username              = username || uri.user
-        @create_path           = create_path
-
-        logger                 ||= self.logger if defined?(SemanticLogger)
-        options                = args.dup
-        options[:logger]       = logger
-        options[:port]         = port || uri.port || 22
-        options[:max_pkt_size] = max_pkt_size
-        options[:password]     = password || uri.password
-        @options               = options
+        @hostname    = uri.hostname
+        @mkdir       = false
+        @username    = username || uri.user
+        @url         = url
+        @password    = password || uri.password
+        @port        = uri.port || 22
+        @ssh_options = ssh_options
+
         super(uri.path)
       end
 
@@ -57,6 +73,7 @@ module IOStreams
         url
       end
 
+      # Note that mkdir is delayed and only executed when the file write is performed.
       def mkdir
         @mkdir = true
         self
@@ -74,11 +91,10 @@ module IOStreams
       # Note:
       # - raises Net::SFTP::StatusException when the file could not be read.
       def reader(&block)
-        result = nil
-        Net::SFTP.start(hostname, username, options) do |sftp|
-          result = sftp.file.open(path, 'rb', &block)
+        IOStreams.temp_file("iostreams-sftp-reader") do |temp_file|
+          sftp_download(path, temp_file.to_s)
+          temp_file.reader(&block)
         end
-        result
       end
 
       # Write to a file on a remote sftp server.
@@ -90,12 +106,97 @@ module IOStreams
       #       output.write('Hello World')
       #     end
       def writer(&block)
-        result = nil
-        Net::SFTP.start(hostname, username, options) do |sftp|
-          sftp.session.exec!("mkdir -p '#{::File.dirname(path)}'") if create_path
-          result = sftp.file.open(path, 'wb', &block)
+        IOStreams.temp_file("iostreams-sftp-writer") do |temp_file|
+          temp_file.writer(&block)
+          sftp_upload(temp_file.to_s, path)
+          temp_file.size
+        end
+      end
+
+      # TODO: Add #copy_from shortcut to detect when a file is supplied that does not require conversion.
+
+      # Search for files on the remote sftp server that match the provided pattern.
+      #
+      # The pattern matching works like Net::SFTP::Operations::Dir.glob and Dir.glob
+      # Each child also returns attributes that contain the file size, ownership, file dates and other details.
+      #
+      # Example Code:
+      # IOStreams.
+      #   path("sftp://sftp.example.org/my_files", username: username, password: password).
+      #   each_child('**/*.{csv,txt}') do |input, attributes|
+      #     puts "#{input.to_s} #{attributes}"
+      #   end
+      #
+      # Example Output:
+      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group", :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
+      def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
+        Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
+
+        flags = ::File::FNM_EXTGLOB
+        flags |= ::File::FNM_CASEFOLD unless case_sensitive
+        flags |= ::File::FNM_DOTMATCH if hidden
+
+        Net::SFTP.start(hostname, username, build_ssh_options) do |sftp|
+          sftp.dir.glob(".", pattern, flags) do |path|
+            next if !directories && !path.file?
+            new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password, ruby: ruby, **ssh_options)
+            yield(new_path, path.attributes.attributes)
+          end
+        end
+        nil
+      end
+
+      private
+
+      attr_reader :password
+
+      # Use sftp and sshpass executables to download to a local file
+      def sftp_download(remote_file_name, local_file_name)
+        Open3.popen2e(*sftp_args) do |writer, reader, waith_thr|
+          writer.puts password
+          # Give time for password to be processed and stdin to be passed to sftp process.
+          sleep self.class.sshpass_wait_seconds
+          writer.puts "get #{remote_file_name} #{local_file_name}"
+          writer.puts 'bye'
+          writer.close
+          out = reader.read.chomp
+          raise(Errors::CommunicationsFailure, "Failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
+          out
+        end
+      end
+
+      def sftp_upload(local_file_name, remote_file_name)
+        Open3.popen2e(*sftp_args) do |writer, reader, waith_thr|
+          writer.puts(password) if password
+          # Give time for password to be processed and stdin to be passed to sftp process.
+          sleep self.class.sshpass_wait_seconds
+          writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
+          writer.puts 'bye'
+          writer.close
+          out = reader.read.chomp
+          raise(Errors::CommunicationsFailure, "Failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}") unless waith_thr.value.success?
+          out
         end
-        result
+      end
+
+      def sftp_args
+        args = [self.class.sshpass_bin, self.class.sftp_bin, '-oBatchMode=no']
+        # Force it to use the password when supplied.
+        args << "-oPubkeyAuthentication=no" if password
+        ssh_options.each_pair { |key, value| args << "-o#{key}=#{value}" }
+        args << '-b'
+        args << '-'
+        args << "#{username}@#{hostname}"
+        args
+      end
+
+      def build_ssh_options
+        options                = ssh_options.dup
+        options[:logger]       ||= self.logger if defined?(SemanticLogger)
+        options[:port]         ||= @port
+        options[:max_pkt_size] ||= 65_536
+        options[:password]     ||= @password
+        options
       end
     end
   end

data/lib/io_streams/pgp.rb

@@ -239,12 +239,15 @@ module IOStreams
     # ascii: [true|false]
     #   Whether to export as ASCII text instead of binary format
     #   Default: true
-    def self.export(email:, ascii: true)
+    def self.export(email:, ascii: true, private: false, passphrase: nil)
       version_check
 
-      armor    = ascii ? '--armor' : nil
-      loopback = pgp_version.to_f >= 2.1 ? '--pinentry-mode loopback' : ''
-      command  = "#{executable} #{loopback} --no-tty --passphrase-fd 0 --batch #{armor} --export #{email}"
+      command = "#{executable} "
+      command << '--pinentry-mode loopback ' if pgp_version.to_f >= 2.1
+      command << '--armor ' if ascii
+      command << "--no-tty  --batch --passphrase"
+      command << (passphrase ? " #{passphrase} " : "-fd 0 ")
+      command << (private ? "--export-secret-keys #{email}" : "--export #{email}")
 
       out, err, status = Open3.capture3(command, binmode: true)
       logger.debug { "IOStreams::Pgp.export: #{command}\n#{err}" } if logger

data/lib/io_streams/streams.rb

@@ -117,7 +117,7 @@ module IOStreams
         class_for_stream(type, stream).stream(io_stream, opts, &block)
       else
         # Daisy chain multiple streams together
-        last = pipeline.keys.inject(block) { |inner, stream| ->(io) { class_for_stream(type, stream).stream(io, pipeline[stream], &inner) } }
+        last = pipeline.keys.inject(block) { |inner, stream_sym| ->(io) { class_for_stream(type, stream_sym).stream(io, pipeline[stream_sym], &inner) } }
         last.call(io_stream)
       end
     end

data/lib/io_streams/symmetric_encryption/reader.rb

@@ -4,7 +4,7 @@ module IOStreams
       # read from a file/stream using Symmetric Encryption
       def self.stream(input_stream, **args, &block)
         unless defined?(SymmetricEncryption)
-          Utils.load_dependency('symmetric-encryption', '.enc streaming')
+          Utils.load_soft_dependency('symmetric-encryption', '.enc streaming')
         end
 
         ::SymmetricEncryption::Reader.open(input_stream, **args, &block)

data/lib/io_streams/symmetric_encryption/writer.rb

@@ -6,7 +6,7 @@ module IOStreams
       # If the input_stream is already compressed consider setting compress: false.
       def self.stream(input_stream, compress: true, **args, &block)
         unless defined?(SymmetricEncryption)
-          Utils.load_dependency('symmetric-encryption', '.enc streaming')
+          Utils.load_soft_dependency('symmetric-encryption', '.enc streaming')
         end
 
         ::SymmetricEncryption::Writer.open(input_stream, compress: compress, **args, &block)
@@ -16,7 +16,7 @@ module IOStreams
       # By default the output stream is compressed unless the file_name extension indicates the file is already compressed.
       def self.file(file_name, compress: nil, **args, &block)
         unless defined?(SymmetricEncryption)
-          Utils.load_dependency('symmetric-encryption', '.enc streaming')
+          Utils.load_soft_dependency('symmetric-encryption', '.enc streaming')
         end
 
         ::SymmetricEncryption::Writer.open(file_name, compress: compress, **args, &block)

data/lib/io_streams/utils.rb

@@ -1,7 +1,7 @@
 module IOStreams
   module Utils
     # Lazy load dependent gem so that it remains a soft dependency.
-    def self.load_dependency(gem_name, stream_type, require_name = gem_name)
+    def self.load_soft_dependency(gem_name, stream_type, require_name = gem_name)
       require require_name
     rescue LoadError => e
       raise(LoadError, "Please install the gem '#{gem_name}' to support #{stream_type}. #{e.message}")

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.0.0.beta2".freeze
+  VERSION = "1.0.0.beta3".freeze
 end

data/lib/io_streams/zip/reader.rb

@@ -37,7 +37,7 @@ module IOStreams
         # The input stream from the first file found in the zip file is passed
         # to the supplied block
         def self.file(file_name, entry_file_name: nil)
-          Utils.load_dependency('rubyzip', 'Zip', 'zip') unless defined?(::Zip)
+          Utils.load_soft_dependency('rubyzip', 'Zip', 'zip') unless defined?(::Zip)
 
           ::Zip::InputStream.open(file_name) do |zin|
             get_entry(zin, entry_file_name) ||

data/lib/io_streams/zip/writer.rb

@@ -46,7 +46,7 @@ module IOStreams
         end
       else
         def self.write_file(file_name, entry_file_name)
-          Utils.load_dependency('rubyzip', 'Zip', 'zip') unless defined?(::Zip)
+          Utils.load_soft_dependency('rubyzip', 'Zip', 'zip') unless defined?(::Zip)
 
           zos = ::Zip::OutputStream.new(file_name)
           zos.put_next_entry(entry_file_name)

data/test/io_streams_test.rb

@@ -61,6 +61,23 @@ module IOStreams
           assert_equal :s3, path
         end
       end
+
+      describe '.temp_file' do
+        it 'returns value from block' do
+          result = IOStreams.temp_file('base', '.ext') { |_path| 257 }
+          assert_equal 257, result
+        end
+
+        it 'supplies new temp file_name' do
+          path1 = nil
+          path2 = nil
+          IOStreams.temp_file('base', '.ext') { |path| path1 = path }
+          IOStreams.temp_file('base', '.ext') { |path| path2 = path }
+          refute_equal path1.to_s, path2.to_s
+          assert path1.is_a?(IOStreams::Paths::File), path1
+          assert path2.is_a?(IOStreams::Paths::File), path2
+        end
+      end
     end
   end
 end

data/test/paths/file_test.rb

@@ -17,23 +17,6 @@ module Paths
         path
       end
 
-      describe '.temp_file' do
-        it 'returns value from block' do
-          result = IOStreams::Paths::File.temp_file('base', '.ext') { |_path| 257 }
-          assert_equal 257, result
-        end
-
-        it 'supplies new temp file_name' do
-          path1 = nil
-          path2 = nil
-          IOStreams::Paths::File.temp_file('base', '.ext') { |path| path1 = path }
-          IOStreams::Paths::File.temp_file('base', '.ext') { |path| path2 = path }
-          refute_equal path1.to_s, path2.to_s
-          assert path1.is_a?(IOStreams::Paths::File), path1
-          assert path2.is_a?(IOStreams::Paths::File), path2
-        end
-      end
-
       describe '#each_child' do
         it 'iterates an empty path' do
           none = nil

data/test/paths/sftp_test.rb

@@ -0,0 +1,78 @@
+require_relative '../test_helper'
+
+module Paths
+  class SFTPTest < Minitest::Test
+    describe IOStreams::Paths::SFTP do
+      before do
+        unless ENV["SFTP_HOSTNAME"]
+          skip "Supply environment variables to test SFTP paths: SFTP_HOSTNAME, SFTP_USERNAME, SFTP_PASSWORD, and optional SFTP_DIR"
+        end
+      end
+
+      let(:host_name) { ENV["SFTP_HOSTNAME"] }
+      let(:username) { ENV["SFTP_USERNAME"] }
+      let(:password) { ENV["SFTP_PASSWORD"] }
+      let(:ftp_dir) { ENV["SFTP_DIR"] || "iostreams_test"}
+      let(:url) { File.join("sftp://", host_name, ftp_dir) }
+
+      let :file_name do
+        File.join(File.dirname(__FILE__), '..', 'files', 'text.txt')
+      end
+
+      let :raw do
+        File.read(file_name)
+      end
+
+      let(:root_path) { IOStreams::Paths::SFTP.new(url, username: username, password: password, ruby: false) }
+
+      let :existing_path do
+        path = root_path.join('test.txt')
+        path.write(raw)
+        path
+      end
+
+      let :missing_path do
+        root_path.join("unknown_path", "test_file.txt")
+      end
+
+      let :missing_file_path do
+        root_path.join("test_file.txt")
+      end
+
+      let :write_path do
+        root_path.join("writer_test.txt")
+      end
+
+      describe '#reader' do
+        it 'reads' do
+          assert_equal raw, existing_path.reader { |io| io.read }
+        end
+
+        it 'fails when the file does not exist' do
+          assert_raises IOStreams::Errors::CommunicationsFailure do
+            missing_file_path.read
+          end
+        end
+
+        it 'fails when the directory does not exist' do
+          assert_raises IOStreams::Errors::CommunicationsFailure do
+            missing_path.read
+          end
+        end
+      end
+
+      describe '#writer' do
+        it 'writes' do
+          assert_equal raw.size, write_path.writer { |io| io.write(raw) }
+          assert_equal raw, write_path.read
+        end
+
+        it 'fails when the directory does not exist' do
+          assert_raises IOStreams::Errors::CommunicationsFailure do
+            missing_path.write("Bad path")
+          end
+        end
+      end
+    end
+  end
+end

data/test/stream_test.rb

@@ -47,7 +47,7 @@ class StreamTest < Minitest::Test
           result = IOStreams::Stream.new(io).
             file_name(multiple_zip_file_name).
             option(:zip, entry_file_name: 'test.json').
-            reader { |io| io.read }
+            read
           assert_equal contents_test_json, result
         end
       end
@@ -56,7 +56,7 @@ class StreamTest < Minitest::Test
         File.open(zip_gz_file_name, 'rb') do |io|
           result = IOStreams::Stream.new(io).
             file_name(zip_gz_file_name).
-            reader { |io| io.read }
+            read
           assert_equal contents_test_txt, result
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta2
+  version: 1.0.0.beta3
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-24 00:00:00.000000000 Z
+date: 2019-11-14 00:00:00.000000000 Z
 dependencies: []
 description: 
 email:
@@ -94,6 +94,7 @@ files:
 - test/paths/http_test.rb
 - test/paths/matcher_test.rb
 - test/paths/s3_test.rb
+- test/paths/sftp_test.rb
 - test/pgp_reader_test.rb
 - test/pgp_test.rb
 - test/pgp_writer_test.rb
@@ -147,6 +148,7 @@ test_files:
 - test/stream_test.rb
 - test/paths/matcher_test.rb
 - test/paths/s3_test.rb
+- test/paths/sftp_test.rb
 - test/paths/file_test.rb
 - test/paths/http_test.rb
 - test/record_reader_test.rb