iostreams 0.20.3 → 1.0.0.beta

data/lib/io_streams/paths/matcher.rb

@@ -0,0 +1,61 @@
+module IOStreams
+  module Paths
+    # Implement fnmatch logic for any path iterator
+    class Matcher
+      # Characters indicating that pattern matching is required
+      MATCH_START_CHARS = /[*?\[{]/
+
+      attr_reader :path, :pattern, :flags
+
+      # If the supplied pattern contains sub-directories without wildcards, navigate down to that directory
+      # first before applying wildcard lookups from that point on.
+      #
+      # Examples: If the current path is "/path/work"
+      #   "a/b/c/**/*"  => "/path/work/a/b/c"
+      #   "a/b/c?/**/*" => "/path/work/a/b"
+      #   "**/*"        => "/path/work"
+      #
+      # Note: Absolute paths in the pattern are not supported.
+      def initialize(path, pattern, case_sensitive: false, hidden: false)
+        extract_optimized_path(path, pattern)
+
+        @flags = ::File::FNM_EXTGLOB
+        @flags |= ::File::FNM_CASEFOLD unless case_sensitive
+        @flags |= ::File::FNM_DOTMATCH if hidden
+      end
+
+      # Returns whether the relative `file_name` matches
+      def match?(file_name)
+        relative_file_name = file_name.sub(path.to_s, '').sub(%r{\A/}, '')
+        ::File.fnmatch?(pattern, relative_file_name, flags)
+      end
+
+      # Whether this pattern includes a recursive match.
+      # I.e. Includes `**` anywhere in the path
+      def recursive?
+        @recursive ||= pattern.nil? ? false : pattern.include?("**")
+      end
+
+      private
+
+      def extract_optimized_path(path, pattern)
+        elements = pattern.split('/')
+        index    = elements.find_index { |e| e.match(MATCH_START_CHARS) }
+        if index == 0
+          # Cannot optimize path since the very first entry contains a wildcard
+          @path    = path || IOStreams.path
+          @pattern = pattern
+        elsif index.nil?
+          # No index means it has no pattern.
+          @path    = path.nil? ? IOStreams.path(pattern) : path.join(pattern)
+          @pattern = nil
+        else
+          new_path = elements[0..index - 1].join('/')
+          @path    = path.nil? ? IOStreams.path(new_path) : path.join(new_path)
+          @pattern = elements[index..-1].join('/')
+        end
+      end
+
+    end
+  end
+end

data/lib/io_streams/paths/s3.rb

@@ -0,0 +1,269 @@
+require "uri"
+
+module IOStreams
+  module Paths
+    class S3 < IOStreams::Path
+      attr_reader :bucket_name, :key, :client
+
+      # Arguments:
+      #
+      # url: [String]
+      #   Prefix must be: `s3://`
+      #   followed by bucket name,
+      #   followed by path and file_name (key).
+      #   Examples:
+      #     s3://my-bucket-name/file_name.txt
+      #     s3://my-bucket-name/some_path/file_name.csv
+      #
+      # Writer specific options:
+      #
+      # @option params [String] :acl
+      #   The canned ACL to apply to the object.
+      #
+      # @option params [String] :cache_control
+      #   Specifies caching behavior along the request/reply chain.
+      #
+      # @option params [String] :content_disposition
+      #   Specifies presentational information for the object.
+      #
+      # @option params [String] :content_encoding
+      #   Specifies what content encodings have been applied to the object and
+      #   thus what decoding mechanisms must be applied to obtain the media-type
+      #   referenced by the Content-Type header field.
+      #
+      # @option params [String] :content_language
+      #   The language the content is in.
+      #
+      # @option params [Integer] :content_length
+      #   Size of the body in bytes. This parameter is useful when the size of
+      #   the body cannot be determined automatically.
+      #
+      # @option params [String] :content_md5
+      #   The base64-encoded 128-bit MD5 digest of the part data. This parameter
+      #   is auto-populated when using the command from the CLI. This parameted
+      #   is required if object lock parameters are specified.
+      #
+      # @option params [String] :content_type
+      #   A standard MIME type describing the format of the object data.
+      #
+      # @option params [Time,DateTime,Date,Integer,String] :expires
+      #   The date and time at which the object is no longer cacheable.
+      #
+      # @option params [String] :grant_full_control
+      #   Gives the grantee READ, READ\_ACP, and WRITE\_ACP permissions on the
+      #   object.
+      #
+      # @option params [String] :grant_read
+      #   Allows grantee to read the object data and its metadata.
+      #
+      # @option params [String] :grant_read_acp
+      #   Allows grantee to read the object ACL.
+      #
+      # @option params [String] :grant_write_acp
+      #   Allows grantee to write the ACL for the applicable object.
+      #
+      # @option params [required, String] :key
+      #   Object key for which the PUT operation was initiated.
+      #
+      # @option params [Hash<String,String>] :metadata
+      #   A map of metadata to store with the object in S3.
+      #
+      # @option params [String] :server_side_encryption
+      #   The Server-side encryption algorithm used when storing this object in
+      #   S3 (e.g., AES256, aws:kms).
+      #
+      # @option params [String] :storage_class
+      #   The type of storage to use for the object. Defaults to 'STANDARD'.
+      #
+      # @option params [String] :website_redirect_location
+      #   If the bucket is configured as a website, redirects requests for this
+      #   object to another object in the same bucket or to an external URL.
+      #   Amazon S3 stores the value of this header in the object metadata.
+      #
+      # @option params [String] :sse_customer_algorithm
+      #   Specifies the algorithm to use to when encrypting the object (e.g.,
+      #   AES256).
+      #
+      # @option params [String] :sse_customer_key
+      #   Specifies the customer-provided encryption key for Amazon S3 to use in
+      #   encrypting data. This value is used to store the object and then it is
+      #   discarded; Amazon does not store the encryption key. The key must be
+      #   appropriate for use with the algorithm specified in the
+      #   x-amz-server-side​-encryption​-customer-algorithm header.
+      #
+      # @option params [String] :sse_customer_key_md5
+      #   Specifies the 128-bit MD5 digest of the encryption key according to
+      #   RFC 1321. Amazon S3 uses this header for a message integrity check to
+      #   ensure the encryption key was transmitted without error.
+      #
+      # @option params [String] :ssekms_key_id
+      #   Specifies the AWS KMS key ID to use for object encryption. All GET and
+      #   PUT requests for an object protected by AWS KMS will fail if not made
+      #   via SSL or using SigV4. Documentation on configuring any of the
+      #   officially supported AWS SDKs and CLI can be found at
+      #   http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version
+      #
+      # @option params [String] :ssekms_encryption_context
+      #   Specifies the AWS KMS Encryption Context to use for object encryption.
+      #   The value of this header is a base64-encoded UTF-8 string holding JSON
+      #   with the encryption context key-value pairs.
+      #
+      # @option params [String] :request_payer
+      #   Confirms that the requester knows that she or he will be charged for
+      #   the request. Bucket owners need not specify this parameter in their
+      #   requests. Documentation on downloading objects from requester pays
+      #   buckets can be found at
+      #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+      #
+      # @option params [String] :tagging
+      #   The tag-set for the object. The tag-set must be encoded as URL Query
+      #   parameters. (For example, "Key1=Value1")
+      #
+      # @option params [String] :object_lock_mode
+      #   The object lock mode that you want to apply to this object.
+      #
+      # @option params [Time,DateTime,Date,Integer,String] :object_lock_retain_until_date
+      #   The date and time when you want this object's object lock to expire.
+      #
+      # @option params [String] :object_lock_legal_hold_status
+      #   The Legal Hold status that you want to apply to the specified object.
+      def initialize(url, client: nil, **args)
+        Utils.load_dependency('aws-sdk-s3', 'AWS S3') unless defined?(::Aws::S3::Client)
+
+        uri = URI.parse(url)
+        raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == 's3'
+
+        @bucket_name = uri.host
+        @key         = uri.path.sub(%r{\A/}, '')
+        @client      = client || ::Aws::S3::Client.new
+        @options     = args
+        super(url)
+      end
+
+      def delete
+        client.delete_object(bucket: bucket_name, key: key)
+        self
+      rescue Aws::S3::Errors::NotFound
+        self
+      end
+
+      def exist?
+        client.head_object(bucket: bucket_name, key: key)
+        true
+      rescue Aws::S3::Errors::NotFound
+        false
+      end
+
+      # Moves this file to the `target_path` by copying it to the new name and then deleting the current file.
+      #
+      # Notes:
+      # - Can copy across buckets.
+      def move_to(target_path)
+        target = IOStreams.new(target_path)
+        return super(target) unless target.is_a?(self.class)
+
+        source_name = ::File.join(bucket_name, key)
+        # TODO: Does/should it also copy metadata?
+        client.copy_object(bucket: target.bucket_name, key: target.key, copy_source: source_name)
+        delete
+        target
+      end
+
+      # S3 logically creates paths when a key is set.
+      def mkpath
+        self
+      end
+
+      def mkdir
+        self
+      end
+
+      def size
+        client.head_object(bucket: bucket_name, key: key).content_length
+      rescue Aws::S3::Errors::NotFound
+        nil
+      end
+
+      # TODO: delete_all
+
+      # Read from AWS S3 file.
+      def reader(&block)
+        # Since S3 download only supports a push stream, write it to a tempfile first.
+        Utils.temp_file_name("iostreams_s3") do |file_name|
+          read_file(file_name)
+
+          ::File.open(file_name, 'rb') { |io| io.read }
+        end
+      end
+
+      # Shortcut method if caller has a filename already with no other streams applied:
+      def read_file(file_name)
+        ::File.open(file_name, 'wb') do |file|
+          client.get_object(@options.merge(response_target: file, bucket: bucket_name, key: key))
+        end
+      end
+
+      # Write to AWS S3
+      #
+      # Raises [MultipartUploadError] If an object is being uploaded in
+      #   parts, and the upload can not be completed, then the upload is
+      #   aborted and this error is raised.  The raised error has a `#errors`
+      #   method that returns the failures that caused the upload to be
+      #   aborted.
+      def writer(&block)
+        # Since S3 upload only supports a pull stream, write it to a tempfile first.
+        Utils.temp_file_name("iostreams_s3") do |file_name|
+          result = ::File.open(file_name, "wb", &block)
+
+          # Upload file only once all data has been written to it
+          write_file(file_name)
+          result
+        end
+      end
+
+      # Shortcut method if caller has a filename already with no other streams applied:
+      def write_file(file_name)
+        if ::File.size(file_name) > 5 * 1024 * 1024
+          # Use multipart file upload
+          s3  = Aws::S3::Resource.new(client: client)
+          obj = s3.bucket(bucket_name).object(key)
+          obj.upload_file(file_name)
+        else
+          ::File.open(file_name, 'rb') do |file|
+            client.put_object(@options.merge(bucket: bucket_name, key: key, body: file))
+          end
+        end
+      end
+
+      # Notes:
+      # - Currently all S3 lookups are recursive as of the pattern regardless of whether the pattern includes `**`.
+      def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
+        raise(NotImplementedError, "AWS S3 #each_child does not yet return directories") if directories
+
+        matcher = Matcher.new(self, pattern, case_sensitive: case_sensitive, hidden: hidden)
+
+        # When the pattern includes an exact file name without any pattern characters
+        if matcher.pattern.nil?
+          yield(matcher.path) if matcher.path.exist?
+          return
+        end
+
+        prefix = URI.parse(matcher.path.to_s).path.sub(%r{\A/}, '')
+        token  = nil
+        loop do
+          # Fetches upto 1,000 entries at a time
+          resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
+          resp.contents.each do |object|
+            file_name = ::File.join("s3://", resp.name, object.key)
+            next unless matcher.match?(file_name)
+
+            yield self.class.new(file_name)
+          end
+          token = resp.next_continuation_token
+          break if token.nil?
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/io_streams/paths/sftp.rb

@@ -0,0 +1,99 @@
+module IOStreams
+  module Paths
+    class SFTP < IOStreams::Path
+      include SemanticLogger::Loggable if defined?(SemanticLogger)
+
+      attr_reader :hostname, :username, :file_name, :create_path, :options
+
+      # Stream to a remote file over sftp.
+      #
+      # file_name: [String]
+      #   Name of file to write to.
+      #
+      # username: [String]
+      #   Name of user to login with.
+      #
+      # password: [String]
+      #   Password for the user.
+      #
+      # host: [String]
+      #   Name of the host to connect to.
+      #
+      # port: [Integer]
+      #   Port to connect to at the above host.
+      #
+      # **args
+      #   Any other options supported by Net::SSH.start
+      #
+      # Examples:
+      #
+      # # Sample URL
+      #   sftp://hostname/path/file_name
+      #
+      # # Full url showing all the optional elements that can be set via the url:
+      #   sftp://username:password@hostname:22/path/file_name
+      def initialize(url, username:, password:, port: nil, max_pkt_size: 65_536, logger: nil, create_path: false, **args)
+        Utils.load_dependency('net-sftp', 'net/sftp') unless defined?(Net::SFTP)
+
+        uri = URI.parse(url)
+        raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
+
+        @hostname              = uri.hostname
+        @file_name             = uri.path
+        @mkdir                 = false
+        @username              = username || uri.user
+        @create_path           = create_path
+
+        logger                 ||= self.logger if defined?(SemanticLogger)
+        options                = args.dup
+        options[:logger]       = logger
+        options[:port]         = port || uri.port || 22
+        options[:max_pkt_size] = max_pkt_size
+        options[:password]     = password || uri.password
+        @options               = options
+        super(file_name)
+      end
+
+      def mkdir
+        @mkdir = true
+        self
+      end
+
+      # Read a file from a remote sftp server.
+      #
+      # Example:
+      #   IOStreams.
+      #     path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
+      #     reader do |input|
+      #       puts input.read
+      #     end
+      #
+      # Note:
+      # - raises Net::SFTP::StatusException when the file could not be read.
+      def reader(&block)
+        result = nil
+        Net::SFTP.start(hostname, username, options) do |sftp|
+          result = sftp.file.open(file_name, 'rb', &block)
+        end
+        result
+      end
+
+      # Write to a file on a remote sftp server.
+      #
+      # Example:
+      #   IOStreams.
+      #     path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
+      #     writer do |output|
+      #       output.write('Hello World')
+      #     end
+      def writer(&block)
+        result = nil
+        Net::SFTP.start(hostname, username, options) do |sftp|
+          sftp.session.exec!("mkdir -p '#{::File.dirname(file_name)}'") if create_path
+          result = sftp.file.open(file_name, 'wb', &block)
+        end
+        result
+      end
+    end
+  end
+end

data/lib/io_streams/pgp.rb

@@ -216,7 +216,7 @@ module IOStreams
     #     email:      [String]
     def self.key_info(key:)
       version_check
-      command = "#{executable}"
+      command = executable.to_s
 
       out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
       logger.debug { "IOStreams::Pgp.key_info: #{command}\n#{err}#{out}" } if logger
@@ -275,8 +275,8 @@ module IOStreams
       command = "#{executable} --import"
 
       out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
-      logger.debug { "IOStreams::Pgp.import: #{command}\n#{err}#{out}" } if logger
-      if status.success? && err.length > 0
+      logger&.debug { "IOStreams::Pgp.import: #{command}\n#{err}#{out}" }
+      if status.success? && !err.empty?
         # Sample output
         #
         #   gpg: key C16500E3: secret key imported\n"
@@ -306,10 +306,26 @@ module IOStreams
         results
       else
         return [] if err =~ /already in secret keyring/
+
         raise(Pgp::Failure, "GPG Failed importing key: #{err}#{out}")
       end
     end
 
+    # Returns [String] email for the supplied after importing and trusting the key
+    #
+    # Notes:
+    # - If the same email address has multiple keys then only the first is currently trusted.
+    def self.import_and_trust(key:)
+      raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == '')
+
+      email = key_info(key: key).first.fetch(:email)
+      raise(ArgumentError, "Recipient email cannot be extracted from supplied key") unless email
+
+      import(key: key)
+      set_trust(email: email)
+      email
+    end
+
     # Set the trust level for an existing key.
     #
     # Returns [String] output if the trust was successfully updated
@@ -325,7 +341,7 @@ module IOStreams
       command          = "#{executable} --import-ownertrust"
       trust            = "#{fingerprint}:#{level + 1}:\n"
       out, err, status = Open3.capture3(command, stdin_data: trust)
-      logger.debug { "IOStreams::Pgp.set_trust: #{command}\n#{err}#{out}" } if logger
+      logger&.debug { "IOStreams::Pgp.set_trust: #{command}\n#{err}#{out}" }
       if status.success?
         err
       else
@@ -336,7 +352,7 @@ module IOStreams
     # DEPRECATED - Use key_ids instead of fingerprints
     def self.fingerprint(email:)
       version_check
-      Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |stdin, out, waith_thr|
+      Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |_stdin, out, waith_thr|
         output = out.read.chomp
         if waith_thr.value.success?
           output.each_line do |line|
@@ -347,6 +363,7 @@ module IOStreams
           nil
         else
           return if output =~ /(public key not found|No public key)/i
+
           raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
         end
       end
@@ -361,7 +378,7 @@ module IOStreams
       @pgp_version ||= begin
         command          = "#{executable} --version"
         out, err, status = Open3.capture3(command)
-        logger.debug { "IOStreams::Pgp.version: #{command}\n#{err}#{out}" } if logger
+        logger&.debug { "IOStreams::Pgp.version: #{command}\n#{err}#{out}" }
         if status.success?
           # Sample output
           #   #{executable} (GnuPG) 2.0.30
@@ -383,6 +400,7 @@ module IOStreams
           end
         else
           return [] if err =~ /(key not found|No (public|secret) key)/i
+
           raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email || key_id}: #{err}#{out}")
         end
       end
@@ -397,7 +415,9 @@ module IOStreams
     end
 
     def self.version_check
-      raise(Pgp::UnsupportedVersion, "Version #{pgp_version} of #{executable} is not yet supported. You are welcome to submit a Pull Request.") if pgp_version.to_f >= 2.3
+      if pgp_version.to_f >= 2.3
+        raise(Pgp::UnsupportedVersion, "Version #{pgp_version} of #{executable} is not yet supported. You are welcome to submit a Pull Request.")
+      end
     end
 
     # v2.2.1 output:
@@ -425,7 +445,7 @@ module IOStreams
             key_type:   match[2],
             date:       (Date.parse(match[4].to_s) rescue match[4])
           }
-        elsif match = line.match(/(pub|sec)\s+(\d+)(.*)\/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?/)
+        elsif match = line.match(%r{(pub|sec)\s+(\d+)(.*)/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?})
           # Matches: pub  2048R/C7F9D9CB 2016-10-26
           # Or:      pub  2048R/C7F9D9CB 2016-10-26 Receiver <receiver@example.org>
           hash = {
@@ -455,7 +475,6 @@ module IOStreams
           # v2.2  18A0FC1C09C0D8AE34CE659257DC4AE323C7368C
           hash[:key_id] ||= match[1]
         end
-
       end
       results
     end
@@ -467,13 +486,18 @@ module IOStreams
       return false if list.empty?
 
       list.each do |key_info|
-        if key_id = key_info[:key_id]
-          command          = "#{executable} --batch --no-tty --yes --delete-#{keys} #{key_id}"
-          out, err, status = Open3.capture3(command, binmode: true)
-          logger.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" } if logger
+        key_id = key_info[:key_id]
+        next unless key_id
 
-          raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}") unless status.success?
-          raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}:#{out}") if out.include?('error')
+        command          = "#{executable} --batch --no-tty --yes --delete-#{keys} #{key_id}"
+        out, err, status = Open3.capture3(command, binmode: true)
+        logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" }
+
+        unless status.success?
+          raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
+        end
+        if out.include?('error')
+          raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}:#{out}")
         end
       end
       true
@@ -487,13 +511,17 @@ module IOStreams
       command << 'done'
 
       out, err, status = Open3.capture3(command, binmode: true)
-      logger.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}: #{out}" } if logger
+      logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}: #{out}" }
 
       return false if err =~ /(not found|no public key)/i
-      raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}") unless status.success?
-      raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}: #{out}") if out.include?('error')
+      unless status.success?
+        raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
+      end
+      if out.include?('error')
+        raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}: #{out}")
+      end
+
       true
     end
-
   end
 end