iostreams 1.2.1 → 1.6.2

data/lib/io_streams/paths/sftp.rb

@@ -26,12 +26,13 @@ module IOStreams
       include SemanticLogger::Loggable if defined?(SemanticLogger)
 
       class << self
-        attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds
+        attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds, :before_password_wait_seconds
       end
 
-      @sftp_bin             = "sftp"
-      @sshpass_bin          = "sshpass"
-      @sshpass_wait_seconds = 5
+      @sftp_bin                     = "sftp"
+      @sshpass_bin                  = "sshpass"
+      @before_password_wait_seconds = 2
+      @sshpass_wait_seconds         = 5
 
       attr_reader :hostname, :username, :ssh_options, :url, :port
 
@@ -71,7 +72,9 @@ module IOStreams
       #   end
       #
       #   # When using the sftp executable use an identity file instead of a password to authenticate:
-      #   IOStreams.path("sftp://test.com/path/file_name.csv", username: "jack", ssh_options: {IdentityFile: "~/.ssh/private_key"}).reader do |io|
+      #   IOStreams.path("sftp://test.com/path/file_name.csv",
+      #                  username:    "jack",
+      #                  ssh_options: {IdentityFile: "~/.ssh/private_key"}).reader do |io|
       #     puts io.read
       #   end
       def initialize(url, username: nil, password: nil, ssh_options: {})
@@ -122,7 +125,8 @@ module IOStreams
       #   end
       #
       # Example Output:
-      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group", :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
+      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
+      #   :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
       def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
         Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
 
@@ -165,15 +169,23 @@ module IOStreams
         with_sftp_args do |args|
           Open3.popen2e(*args) do |writer, reader, waith_thr|
             begin
+              # Give time for remote sftp server to get ready to accept the password.
+              sleep self.class.before_password_wait_seconds
+
               writer.puts password
+
               # Give time for password to be processed and stdin to be passed to sftp process.
               sleep self.class.sshpass_wait_seconds
+
               writer.puts "get #{remote_file_name} #{local_file_name}"
               writer.puts "bye"
               writer.close
               out = reader.read.chomp
               unless waith_thr.value.success?
-                raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
+                raise(
+                  Errors::CommunicationsFailure,
+                  "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
+                )
               end
 
               out
@@ -183,7 +195,10 @@ module IOStreams
                     rescue StandardError
                       nil
                     end
-              raise(Errors::CommunicationsFailure, "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
+              raise(
+                Errors::CommunicationsFailure,
+                "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
+              )
             end
           end
         end
@@ -201,7 +216,10 @@ module IOStreams
               writer.close
               out = reader.read.chomp
               unless waith_thr.value.success?
-                raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
+                raise(
+                  Errors::CommunicationsFailure,
+                  "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
+                )
               end
 
               out
@@ -211,7 +229,10 @@ module IOStreams
                     rescue StandardError
                       nil
                     end
-              raise(Errors::CommunicationsFailure, "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}")
+              raise(
+                Errors::CommunicationsFailure,
+                "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
+              )
             end
           end
         end
@@ -256,9 +277,9 @@ module IOStreams
       end
 
       def build_ssh_options
-        options = ssh_options.dup
-        options[:logger] ||= logger if defined?(SemanticLogger)
-        options[:port] ||= port
+        options                = ssh_options.dup
+        options[:logger]       ||= logger if defined?(SemanticLogger)
+        options[:port]         ||= port
         options[:max_pkt_size] ||= 65_536
         options[:password]     ||= @password
         options

data/lib/io_streams/pgp.rb

@@ -46,7 +46,15 @@ module IOStreams
     #     `SecureRandom.urlsafe_base64(128)`
     #
     # See `man gpg` for the remaining options
-    def self.generate_key(name:, email:, comment: nil, passphrase:, key_type: "RSA", key_length: 4096, subkey_type: "RSA", subkey_length: key_length, expire_date: nil)
+    def self.generate_key(name:,
+                          email:,
+                          comment: nil,
+                          passphrase:,
+                          key_type: "RSA",
+                          key_length: 4096,
+                          subkey_type: "RSA",
+                          subkey_length: key_length,
+                          expire_date: nil)
       version_check
       params = ""
       params << "Key-Type: #{key_type}\n" if key_type
@@ -63,13 +71,13 @@ module IOStreams
 
       out, err, status = Open3.capture3(command, binmode: true, stdin_data: params)
       logger&.debug { "IOStreams::Pgp.generate_key: #{command}\n#{params}\n#{err}#{out}" }
-      if status.success?
-        if match = err.match(/gpg: key ([0-9A-F]+)\s+/)
-          match[1]
-        end
-      else
-        raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}")
-      end
+
+      raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}") unless status.success?
+
+      match = err.match(/gpg: key ([0-9A-F]+)\s+/)
+      return unless match
+
+      match[1]
     end
 
     # Delete all private and public keys for a particular email.
@@ -77,7 +85,8 @@ module IOStreams
     # Returns false if no key was found.
     # Raises an exception if it fails to delete the key.
     #
-    # email: [String] Email address for the key.
+    # email: [String] Optional email address for the key.
+    # key_id: [String] Optional id for the key.
     #
     # public: [true|false]
     #   Whether to delete the public key
@@ -86,12 +95,12 @@ module IOStreams
     # private: [true|false]
     #   Whether to delete the private key
     #   Default: false
-    def self.delete_keys(email:, public: true, private: false)
+    def self.delete_keys(email: nil, key_id: nil, public: true, private: false)
       version_check
       method_name = pgp_version.to_f >= 2.2 ? :delete_public_or_private_keys : :delete_public_or_private_keys_v1
       status      = false
-      status      = send(method_name, email: email, private: true) if private
-      status      = send(method_name, email: email, private: false) if public
+      status      = send(method_name, email: email, key_id: key_id, private: true) if private
+      status      = send(method_name, email: email, key_id: key_id, private: false) if public
       status
     end
 
@@ -102,11 +111,6 @@ module IOStreams
       !list_keys(email: email, key_id: key_id, private: private).empty?
     end
 
-    # Deprecated
-    def self.has_key?(**args)
-      key?(**args)
-    end
-
     # Returns [Array<Hash>] the list of keys.
     #   Each Hash consists of:
     #     key_length: [Integer]
@@ -150,16 +154,15 @@ module IOStreams
 
       out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
       logger&.debug { "IOStreams::Pgp.key_info: #{command}\n#{err}#{out}" }
-      if status.success? && out.length.positive?
-        # Sample Output:
-        #
-        #   pub  4096R/3A5456F5 2017-06-07
-        #   uid                            Joe Bloggs <j@bloggs.net>
-        #   sub  4096R/2C9B240B 2017-06-07
-        parse_list_output(out)
-      else
-        raise(Pgp::Failure, "GPG Failed extracting key details: #{err} #{out}")
-      end
+
+      raise(Pgp::Failure, "GPG Failed extracting key details: #{err} #{out}") unless status.success? && out.length.positive?
+
+      # Sample Output:
+      #
+      #   pub  4096R/3A5456F5 2017-06-07
+      #   uid                            Joe Bloggs <j@bloggs.net>
+      #   sub  4096R/2C9B240B 2017-06-07
+      parse_list_output(out)
     end
 
     # Returns [String] containing all the public keys for the supplied email address.
@@ -181,11 +184,10 @@ module IOStreams
 
       out, err, status = Open3.capture3(command, binmode: true)
       logger&.debug { "IOStreams::Pgp.export: #{command}\n#{err}" }
-      if status.success? && out.length.positive?
-        out
-      else
-        raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err}")
-      end
+
+      raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err}") unless status.success? && out.length.positive?
+
+      out
     end
 
     # Imports the supplied public/private key
@@ -226,7 +228,7 @@ module IOStreams
         err.each_line do |line|
           if line =~ /secret key imported/
             secret = true
-          elsif match = line.match(/key\s+(\w+):\s+(\w+).+\"(.*)<(.*)>\"/)
+          elsif (match = line.match(/key\s+(\w+):\s+(\w+).+\"(.*)<(.*)>\"/))
             results << {
               key_id:  match[1].to_s.strip,
               private: secret,
@@ -251,12 +253,15 @@ module IOStreams
     def self.import_and_trust(key:)
       raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == "")
 
-      email = key_info(key: key).last.fetch(:email)
-      raise(ArgumentError, "Recipient email cannot be extracted from supplied key") unless email
+      key_info = key_info(key: key).last
+
+      email = key_info.fetch(:email, nil)
+      key_id = key_info.fetch(:key_id, nil)
+      raise(ArgumentError, "Recipient email or key id cannot be extracted from supplied key") unless email || key_id
 
       import(key: key)
-      set_trust(email: email)
-      email
+      set_trust(email: email, key_id: key_id)
+      email || key_id
     end
 
     # Set the trust level for an existing key.
@@ -266,20 +271,19 @@ module IOStreams
     #
     # After importing keys, they are not trusted and the relevant trust level must be set.
     #   Default: 5 : Ultimate
-    def self.set_trust(email:, level: 5)
+    def self.set_trust(email: nil, key_id: nil, level: 5)
       version_check
-      fingerprint = fingerprint(email: email)
+      fingerprint = key_id || fingerprint(email: email)
       return unless fingerprint
 
       command          = "#{executable} --import-ownertrust"
       trust            = "#{fingerprint}:#{level + 1}:\n"
       out, err, status = Open3.capture3(command, stdin_data: trust)
       logger&.debug { "IOStreams::Pgp.set_trust: #{command}\n#{err}#{out}" }
-      if status.success?
-        err
-      else
-        raise(Pgp::Failure, "GPG Failed trusting key: #{err} #{out}")
-      end
+
+      raise(Pgp::Failure, "GPG Failed trusting key: #{err} #{out}") unless status.success?
+
+      err
     end
 
     # DEPRECATED - Use key_ids instead of fingerprints
@@ -287,18 +291,18 @@ module IOStreams
       version_check
       Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |_stdin, out, waith_thr|
         output = out.read.chomp
-        if waith_thr.value.success?
-          output.each_line do |line|
-            if match = line.match(/\Afpr.*::([^\:]*):\Z/)
-              return match[1]
-            end
+        unless waith_thr.value.success?
+          unless output =~ /(public key not found|No public key)/i
+            raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
           end
-          nil
-        else
-          return if output =~ /(public key not found|No public key)/i
+        end
 
-          raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
+        output.each_line do |line|
+          if (match = line.match(/\Afpr.*::([^\:]*):\Z/))
+            return match[1]
+          end
         end
+        nil
       end
     end
 
@@ -328,7 +332,7 @@ module IOStreams
           #           CAMELLIA128, CAMELLIA192, CAMELLIA256
           #   Hash: MD5, SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224
           #   Compression: Uncompressed, ZIP, ZLIB, BZIP2
-          if match = out.lines.first.match(/(\d+\.\d+.\d+)/)
+          if (match = out.lines.first.match(/(\d+\.\d+.\d+)/))
             match[1]
           end
         else
@@ -339,8 +343,6 @@ module IOStreams
       end
     end
 
-    private
-
     @logger = nil
 
     def self.logger
@@ -348,9 +350,12 @@ module IOStreams
     end
 
     def self.version_check
-      if pgp_version.to_f >= 2.3
-        raise(Pgp::UnsupportedVersion, "Version #{pgp_version} of #{executable} is not yet supported. You are welcome to submit a Pull Request.")
-      end
+      return unless pgp_version.to_f >= 2.4
+
+      raise(
+        Pgp::UnsupportedVersion,
+        "Version #{pgp_version} of #{executable} is not yet supported. Please submit a Pull Request to support it."
+      )
     end
 
     # v2.2.1 output:
@@ -370,7 +375,7 @@ module IOStreams
       results = []
       hash    = {}
       out.each_line do |line|
-        if match = line.match(/(pub|sec)\s+(\D+)(\d+)\s+(\d+-\d+-\d+)\s+(.*)/)
+        if (match = line.match(/(pub|sec)\s+(\D+)(\d+)\s+(\d+-\d+-\d+)\s+(.*)/))
           # v2.2:    pub   rsa1024 2017-10-24 [SCEA]
           hash = {
             private:    match[1] == "sec",
@@ -382,7 +387,7 @@ module IOStreams
                            match[4]
                          end)
           }
-        elsif match = line.match(%r{(pub|sec)\s+(\d+)(.*)/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?})
+        elsif (match = line.match(%r{(pub|sec)\s+(\d+)(.*)/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?}))
           # Matches: pub  2048R/C7F9D9CB 2016-10-26
           # Or:      pub  2048R/C7F9D9CB 2016-10-26 Receiver <receiver@example.org>
           hash = {
@@ -403,7 +408,7 @@ module IOStreams
             results << hash
             hash = {}
           end
-        elsif match = line.match(/uid\s+(\[(.+)\]\s+)?(.+)<(.+)>/)
+        elsif (match = line.match(/uid\s+(\[(.+)\]\s+)?(.+)<(.+)>/))
           # Matches:  uid       [ unknown] Joe Bloggs <j@bloggs.net>
           # Or:       uid                  Joe Bloggs <j@bloggs.net>
           # v2.2:     uid           [ultimate] Joe Bloggs <pgp_test@iostreams.net>
@@ -412,7 +417,15 @@ module IOStreams
           hash[:trust] = match[2].to_s.strip if match[1]
           results << hash
           hash = {}
-        elsif match = line.match(/([A-Z0-9]+)/)
+        elsif (match = line.match(/uid\s+(\[(.+)\]\s+)?(.+)/))
+          # Matches:  uid       [ unknown] Joe Bloggs
+          # Or:       uid                  Joe Bloggs
+          # v2.2:     uid           [ultimate] Joe Bloggs
+          hash[:name]  = match[3].to_s.strip
+          hash[:trust] = match[2].to_s.strip if match[1]
+          results << hash
+          hash = {}
+        elsif (match = line.match(/([A-Z0-9]+)/))
           # v2.2  18A0FC1C09C0D8AE34CE659257DC4AE323C7368C
           hash[:key_id] ||= match[1]
         end
@@ -420,10 +433,10 @@ module IOStreams
       results
     end
 
-    def self.delete_public_or_private_keys(email:, private: false)
+    def self.delete_public_or_private_keys(email: nil, key_id: nil, private: false)
       keys = private ? "secret-keys" : "keys"
 
-      list = list_keys(email: email, private: private)
+      list = email ? list_keys(email: email, private: private) : list_keys(key_id: key_id)
       return false if list.empty?
 
       list.each do |key_info|
@@ -435,17 +448,17 @@ module IOStreams
         logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" }
 
         unless status.success?
-          raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
+          raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
         end
-        raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}:#{out}") if out.include?("error")
+        raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}:#{out}") if out.include?("error")
       end
       true
     end
 
-    def self.delete_public_or_private_keys_v1(email:, private: false)
+    def self.delete_public_or_private_keys_v1(email: nil, key_id: nil, private: false)
       keys = private ? "secret-keys" : "keys"
 
-      command = "for i in `#{executable} --list-#{keys} --with-colons --fingerprint #{email} | grep \"^fpr\" | cut -d: -f10`; do\n"
+      command = "for i in `#{executable} --list-#{keys} --with-colons --fingerprint #{email || key_id} | grep \"^fpr\" | cut -d: -f10`; do\n"
       command << "#{executable} --batch --no-tty --yes --delete-#{keys} \"$i\" ;\n"
       command << "done"
 
@@ -454,9 +467,9 @@ module IOStreams
 
       return false if err =~ /(not found|no public key)/i
       unless status.success?
-        raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
+        raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
       end
-      raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}: #{out}") if out.include?("error")
+      raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
 
       true
     end

data/lib/io_streams/stream.rb

@@ -17,7 +17,7 @@ module IOStreams
     #
     # Example:
     #
-    # IOStreams.path('tempfile2527').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
+    # IOStreams.path("tempfile2527").stream(:zip).stream(:pgp, passphrase: "receiver_passphrase").read
     def stream(stream, **options)
       builder.stream(stream, **options)
       self
@@ -27,12 +27,12 @@ module IOStreams
     # If the relevant stream is not found for this file it is ignored.
     # For example, if the file does not have a pgp extension then the pgp option is not relevant.
     #
-    # IOStreams.path('keep_safe.pgp').option(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
+    # IOStreams.path("keep_safe.pgp").option(:pgp, passphrase: "receiver_passphrase").read
     #
     # # In this case the file is not pgp so the `passphrase` option is ignored.
-    # IOStreams.path('keep_safe.enc').option(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
+    # IOStreams.path("keep_safe.enc").option(:pgp, passphrase: "receiver_passphrase").read
     #
-    # IOStreams.path(output_file_name).option(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
+    # IOStreams.path(output_file_name).option(:pgp, passphrase: "receiver_passphrase").read
     def option(stream, **options)
       builder.option(stream, **options)
       self
@@ -177,7 +177,7 @@ module IOStreams
     end
 
     def copy_to(target, convert: true)
-      target = IOStreams.path(target) unless target.is_a?(Stream)
+      target = IOStreams.new(target)
       target.copy_from(self, convert: convert)
     end
 
@@ -191,11 +191,41 @@ module IOStreams
       end
     end
 
-    # Set/get the original file_name
+    # Set the original file_name
     def file_name=(file_name)
       builder.file_name = file_name
     end
 
+    # Set/get the tabular format_options
+    def format(format = :none)
+      if format == :none
+        builder.format
+      else
+        builder.format = format
+        self
+      end
+    end
+
+    # Set the tabular format
+    def format=(format)
+      builder.format = format
+    end
+
+    # Set/get the tabular format options
+    def format_options(format_options = :none)
+      if format_options == :none
+        builder.format_options
+      else
+        builder.format_options = format_options
+        self
+      end
+    end
+
+    # Set the tabular format_options
+    def format_options=(format_options)
+      builder.format_options = format_options
+    end
+
     # Returns [String] the last component of this path.
     # Returns `nil` if no `file_name` was set.
     #
@@ -282,20 +312,37 @@ module IOStreams
     def line_reader(embedded_within: nil, **args)
       embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?(".csv")
 
-      stream_reader { |io| yield IOStreams::Line::Reader.new(io, original_file_name: builder.file_name, embedded_within: embedded_within, **args) }
+      stream_reader do |io|
+        yield IOStreams::Line::Reader.new(io,
+                                          original_file_name: builder.file_name,
+                                          embedded_within:    embedded_within,
+                                          **args)
+      end
     end
 
     # Iterate over a file / stream returning each line as an array, one at a time.
     def row_reader(delimiter: nil, embedded_within: nil, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
-        yield IOStreams::Row::Reader.new(io, original_file_name: builder.file_name, **args)
+        yield IOStreams::Row::Reader.new(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args
+        )
       end
     end
 
     # Iterate over a file / stream returning each line as a hash, one at a time.
     def record_reader(delimiter: nil, embedded_within: nil, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
-        yield IOStreams::Record::Reader.new(io, original_file_name: builder.file_name, **args)
+        yield IOStreams::Record::Reader.new(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args
+        )
       end
     end
 
@@ -306,19 +353,38 @@ module IOStreams
     def line_writer(**args, &block)
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Line::Writer)
 
-      writer { |io| IOStreams::Line::Writer.stream(io, original_file_name: builder.file_name, **args, &block) }
+      writer do |io|
+        IOStreams::Line::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+      end
     end
 
     def row_writer(delimiter: $/, **args, &block)
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Row::Writer)
 
-      line_writer(delimiter: delimiter) { |io| IOStreams::Row::Writer.stream(io, original_file_name: builder.file_name, **args, &block) }
+      line_writer(delimiter: delimiter) do |io|
+        IOStreams::Row::Writer.stream(
+          io,
+          original_file_name: builder.file_name,
+          format: builder.format,
+          format_options: builder.format_options,
+          **args,
+          &block
+        )
+      end
     end
 
     def record_writer(delimiter: $/, **args, &block)
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Record::Writer)
 
-      line_writer(delimiter: delimiter) { |io| IOStreams::Record::Writer.stream(io, original_file_name: builder.file_name, **args, &block) }
+      line_writer(delimiter: delimiter) do |io|
+        IOStreams::Record::Writer.stream(
+          io,
+          original_file_name: builder.file_name,
+          format: builder.format,
+          format_options: builder.format_options,
+          **args,
+          &block)
+      end
     end
   end
 end