iostreams 1.11.0 → 2.1.0

data/lib/io_streams/paths/s3.rb

@@ -21,6 +21,15 @@ module IOStreams
       #     s3://my-bucket-name/file_name.txt
       #     s3://my-bucket-name/some_path/file_name.csv
       #
+      #   Any query string in the url is added to the S3 request parameters, for example:
+      #     s3://my-bucket-name/file_name.csv?acl=bucket-owner-full-control
+      #
+      #   SECURITY WARNING:
+      #     Do not interpolate untrusted file names into the url, since a name such as
+      #     `file.csv?acl=public-read` would set request parameters.
+      #     Instead join untrusted names onto the path, which does not parse them as a query:
+      #       IOStreams.path("s3://my-bucket-name/uploads").join(untrusted_name)
+      #
       # access_key_id: [String]
       #   AWS Access Key Id to use to access this bucket.
       #
@@ -37,10 +46,10 @@ module IOStreams
       #
       #   Example:
       #     client = Aws::S3::Client.new(endpoint: "https://s3.test.com")
-      #     IOStreams::Paths::S3.new(client: client)
+      #     IOStreams::Paths::S3.new("s3://bucket/path/file_name.txt", client: client)
       #
       #   Example:
-      #     IOStreams::Paths::S3.new(client: { endpoint: "https://s3.test.com" })
+      #     IOStreams::Paths::S3.new("s3://bucket/path/file_name.txt", client: { endpoint: "https://s3.test.com" })
       #
       # Writer specific options:
       #
@@ -185,6 +194,7 @@ module IOStreams
       end
 
       def delete
+        authorize!
         client.delete_object(bucket: bucket_name, key: path)
         self
       rescue Aws::S3::Errors::NotFound
@@ -192,6 +202,7 @@ module IOStreams
       end
 
       def exist?
+        authorize!
         client.head_object(bucket: bucket_name, key: path)
         true
       rescue Aws::S3::Errors::NotFound
@@ -211,11 +222,13 @@ module IOStreams
 
       # Make S3 perform direct copies within S3 itself.
       def copy_to(target_path, convert: true, **args)
-        return super(target_path, convert: convert, **args) if convert || (size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
+        return super if convert || (size.to_i >= S3_COPY_OBJECT_SIZE_LIMIT)
 
         target = IOStreams.new(target_path)
         return super(target, convert: convert, **args) unless target.is_a?(self.class)
 
+        authorize!
+        target.authorize!
         source_name = ::File.join(bucket_name, path)
         client.copy_object(options.merge(bucket: target.bucket_name, key: target.path, copy_source: source_name))
         target
@@ -230,6 +243,8 @@ module IOStreams
           return super(source, convert: convert, **args)
         end
 
+        authorize!
+        source.authorize!
         source_name = ::File.join(source.bucket_name, source.path)
         client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))
       end
@@ -244,6 +259,7 @@ module IOStreams
       end
 
       def size
+        authorize!
         client.head_object(bucket: bucket_name, key: path).content_length
       rescue Aws::S3::Errors::NotFound
         nil
@@ -254,7 +270,7 @@ module IOStreams
       # Read from AWS S3 file.
       def stream_reader(&block)
         # Since S3 download only supports a push stream, write it to a tempfile first.
-        Utils.temp_file_name("iostreams_s3") do |file_name|
+        Utils.private_temp_file("iostreams_s3") do |file_name|
           read_file(file_name)
 
           ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
@@ -263,6 +279,7 @@ module IOStreams
 
       # Shortcut method if caller has a filename already with no other streams applied:
       def read_file(file_name)
+        authorize!
         ::File.open(file_name, "wb") do |file|
           client.get_object(options.merge(response_target: file, bucket: bucket_name, key: path))
         end
@@ -277,7 +294,7 @@ module IOStreams
       #   aborted.
       def stream_writer(&block)
         # Since S3 upload only supports a pull stream, write it to a tempfile first.
-        Utils.temp_file_name("iostreams_s3") do |file_name|
+        Utils.private_temp_file("iostreams_s3") do |file_name|
           result = ::File.open(file_name, "wb") { |io| builder.writer(io, &block) }
 
           # Upload file only once all data has been written to it
@@ -288,6 +305,7 @@ module IOStreams
 
       # Shortcut method if caller has a filename already with no other streams applied:
       def write_file(file_name)
+        authorize!
         if ::File.size(file_name) > MULTIPART_UPLOAD_SIZE
           # Use multipart file upload
           s3  = Aws::S3::Resource.new(client: client)
@@ -308,29 +326,26 @@ module IOStreams
                          case_sensitive: case_sensitive, directories: directories, hidden: hidden)
         end
 
+        authorize!
         matcher = Matcher.new(self, pattern, case_sensitive: case_sensitive, hidden: hidden)
 
         # When the pattern includes an exact file name without any pattern characters
         if matcher.pattern.nil?
-          yield(matcher.path) if matcher.path.exist?
+          yield(matcher.path) if allowed_child?(matcher.path) && matcher.path.exist?
           return
         end
 
         prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/}, "")
-        token  = nil
-        loop do
-          # Fetches upto 1,000 entries at a time
-          resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
-          resp.contents.each do |object|
-            next if !directories && object.key.end_with?("/")
+        each_object(prefix) do |name, object|
+          next if !directories && object.key.end_with?("/")
 
-            file_name = ::File.join("s3://", resp.name, object.key)
-            next unless matcher.match?(file_name)
+          file_name = ::File.join("s3://", name, object.key)
+          next unless matcher.match?(file_name)
 
-            yield(self.class.new(file_name), object.to_h)
-          end
-          token = resp.next_continuation_token
-          break if token.nil?
+          child = child_path(name, object.key)
+          next unless allowed_child?(child)
+
+          yield(child, object.to_h)
         end
         nil
       end
@@ -344,6 +359,42 @@ module IOStreams
       def client
         @client ||= ::Aws::S3::Client.new(@client_options)
       end
+
+      private
+
+      # Yields the bucket name and each object in the bucket whose key starts with the supplied prefix.
+      def each_object(prefix)
+        token = nil
+        loop do
+          # Fetches upto 1,000 entries at a time
+          resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
+          resp.contents.each { |object| yield(resp.name, object) }
+          token = resp.next_continuation_token
+          break if token.nil?
+        end
+      end
+
+      # Returns [String] the bucket and key, which is compared against the allowed paths.
+      #
+      # S3 treats `.` and `..` in a key as ordinary characters, but other services that implement the
+      # S3 API may resolve them, so keys containing them are denied.
+      def allowed_location
+        if path.split("/").intersect?([".", ".."])
+          raise(Errors::AccessDenied, "Access denied to #{self}: '.' and '..' are not allowed in S3 keys")
+        end
+
+        to_s.sub(%r{/+\z}, "")
+      end
+
+      # Set the key directly rather than parsing it as part of a URL, since a key can contain
+      # characters such as `?`, `+` or `%` that a URL parser would treat as a query or as escapes.
+      #
+      # The child uses this path's client, so that it has the same credentials and region.
+      def child_path(bucket_name, key)
+        child      = self.class.new("s3://#{bucket_name}", client: client)
+        child.path = key.dup.freeze
+        child
+      end
     end
   end
 end

data/lib/io_streams/paths/sftp/net_ssh.rb

@@ -0,0 +1,104 @@
+module IOStreams
+  module Paths
+    class SFTP < IOStreams::Path
+      # Translates the ssh options supplied for the sftp executable into options for net-ssh,
+      # which `SFTP#each_child` uses to list files instead of the sftp executable.
+      #
+      # The net-ssh options match those that SFTP passes to the sftp executable.
+      module NetSSH
+        # The ssh options that net-ssh supports.
+        OPTIONS = %w[
+          HostKey IdentityKey IdentityFile UserKnownHostsFile StrictHostKeyChecking
+          ConnectTimeout ServerAliveInterval ServerAliveCountMax LogLevel
+        ].freeze
+
+        # Yields [Hash] the net-ssh options.
+        #
+        # Raises [ArgumentError] when `ssh_options` includes an option that is not in `OPTIONS`.
+        def self.options(ssh_options, port:, password:)
+          options = {port: port, max_pkt_size: 65_536, non_interactive: true}
+          options[:logger] = IOStreams.logger if IOStreams.logger
+          # Like the sftp executable, only use the password when one is supplied, and otherwise only public keys.
+          if password
+            options[:password]     = password
+            options[:auth_methods] = %w[password keyboard-interactive]
+          else
+            options[:auth_methods] = %w[publickey]
+          end
+          # Like the sftp executable, which uses `StrictHostKeyChecking=yes`, instead of the
+          # net-ssh default of trusting a host key the first time it is seen.
+          options[:verify_host_key] = :always
+
+          ssh_options.each_pair { |key, value| add(options, key, value) }
+          return yield(options) unless ssh_options.key?("HostKey")
+
+          # Like the sftp executable, the host key replaces the user's known_hosts file.
+          Utils.private_temp_file("iostreams-sftp-known-hosts") do |file_name|
+            ::File.binwrite(file_name, ssh_options["HostKey"])
+            options[:user_known_hosts_file] = [file_name]
+            yield(options)
+          end
+        end
+
+        def self.add(options, key, value)
+          case key
+          when "HostKey"
+            # Written to a temp file by `.options`.
+          when "IdentityKey"
+            (options[:key_data] ||= []) << value
+            options[:keys_only] = true
+          when "IdentityFile"
+            (options[:keys] ||= []) << value
+            options[:keys_only] = true
+          when "UserKnownHostsFile"
+            options[:user_known_hosts_file] = value.to_s.split
+          when "StrictHostKeyChecking"
+            options[:verify_host_key] = verify_host_key(value)
+          when "ConnectTimeout"
+            options[:timeout] = Integer(value)
+          when "ServerAliveInterval"
+            options[:keepalive]          = Integer(value).positive?
+            options[:keepalive_interval] = Integer(value)
+          when "ServerAliveCountMax"
+            options[:keepalive_maxcount] = Integer(value)
+          when "LogLevel"
+            options[:verbose] = log_level(value)
+          else
+            raise(ArgumentError,
+                  "SFTP #each_child does not support the ssh option #{key.inspect}. It supports: #{OPTIONS.join(', ')}")
+          end
+        end
+
+        def self.verify_host_key(value)
+          case value.to_s.downcase
+          when "yes", "ask"
+            :always
+          when "accept-new"
+            :accept_new
+          when "no", "off"
+            :never
+          else
+            raise(ArgumentError, "Invalid StrictHostKeyChecking value: #{value.inspect}")
+          end
+        end
+
+        def self.log_level(value)
+          case value.to_s.upcase
+          when "QUIET", "FATAL"
+            :fatal
+          when "ERROR"
+            :error
+          when "INFO", "VERBOSE"
+            :info
+          when "DEBUG", "DEBUG1", "DEBUG2", "DEBUG3"
+            :debug
+          else
+            raise(ArgumentError, "Invalid LogLevel value: #{value.inspect}")
+          end
+        end
+
+        private_class_method :add, :verify_host_key, :log_level
+      end
+    end
+  end
+end

data/lib/io_streams/paths/sftp.rb

@@ -23,8 +23,6 @@ module IOStreams
     #       output.write('Hello World')
     #     end
     class SFTP < IOStreams::Path
-      include SemanticLogger::Loggable if defined?(SemanticLogger)
-
       class << self
         attr_accessor :sshpass_bin, :sftp_bin, :sshpass_wait_seconds, :before_password_wait_seconds
       end
@@ -34,6 +32,8 @@ module IOStreams
       @before_password_wait_seconds = 2
       @sshpass_wait_seconds         = 5
 
+      autoload :NetSSH, "io_streams/paths/sftp/net_ssh"
+
       attr_reader :hostname, :username, :ssh_options, :url, :port
 
       # Stream to a remote file over sftp.
@@ -41,6 +41,11 @@ module IOStreams
       # url: [String]
       #   "sftp://<host_name>/<file_name>"
       #
+      #   SECURITY WARNING:
+      #     A username and password supplied in the url remain part of it, so `#to_s` and `#url`
+      #     return them, as does any log or error message that includes the path.
+      #     Supply them with the `username:` and `password:` arguments instead.
+      #
       # username: [String]
       #   Name of user to login with.
       #
@@ -65,6 +70,11 @@ module IOStreams
       #   - Any other options supported by ssh_config.
       #     `man ssh_config` to see all available options.
       #
+      #   `#each_child` lists files with the net-sftp gem instead of the sftp executable, so it only supports
+      #   these ssh options: HostKey, IdentityKey, IdentityFile, UserKnownHostsFile, StrictHostKeyChecking,
+      #   ConnectTimeout, ServerAliveInterval, ServerAliveCountMax and LogLevel. Any other option raises
+      #   ArgumentError.
+      #
       # Examples:
       #
       #   # Display the contents of a remote file
@@ -104,7 +114,7 @@ module IOStreams
         # Not Ruby 2.5 yet: transform_keys(&:to_s)
         @ssh_options = {}
         ssh_options.each_pair { |key, value| @ssh_options[key.to_s] = value }
-        @ssh_options.merge(uri.query) if uri.query
+        validate_username!
 
         super(uri.path)
       end
@@ -126,7 +136,9 @@ module IOStreams
 
       # TODO: Add #copy_from shortcut to detect when a file is supplied that does not require conversion.
 
-      # Search for files on the remote sftp server that match the provided pattern.
+      # Search for files on the remote sftp server that match the provided pattern, within this path.
+      # When the url does not include a path, for example `sftp://sftp.example.org`, it searches
+      # the login directory.
       #
       # The pattern matching works like Net::SFTP::Operations::Dir.glob and Dir.glob
       # Each child also returns attributes that contain the file size, ownership, file dates and other details.
@@ -139,7 +151,7 @@ module IOStreams
       #   end
       #
       # Example Output:
-      # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
+      # sftp://sftp.example.org/my_files/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
       #   :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
       def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
         unless block_given?
@@ -147,64 +159,95 @@ module IOStreams
                          case_sensitive: case_sensitive, directories: directories, hidden: hidden)
         end
 
+        authorize!
         Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
 
         flags = ::File::FNM_EXTGLOB
         flags |= ::File::FNM_CASEFOLD unless case_sensitive
         flags |= ::File::FNM_DOTMATCH if hidden
 
-        Net::SFTP.start(hostname, username, build_ssh_options) do |sftp|
-          sftp.dir.glob(".", pattern, flags) do |path|
-            next if !directories && !path.file?
+        NetSSH.options(ssh_options, port: port, password: password) do |options|
+          Net::SFTP.start(hostname, username, options) do |sftp|
+            # Without a path in the url, list the login directory.
+            sftp.dir.glob(path.empty? ? "." : path, pattern, flags) do |entry|
+              next if !directories && !entry.file?
 
-            new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password, **ssh_options)
-            yield(new_path, path.attributes.attributes)
+              child = child_path(entry.name)
+              yield(child, entry.attributes.attributes) if allowed_child?(child)
+            end
           end
         end
         nil
       end
 
+      protected
+
+      attr_writer :url
+
       private
 
       attr_reader :password
 
+      # Returns [String] the host, port and path, which is compared against the allowed paths.
+      # `.` and `..` are resolved the way the sftp server resolves them.
+      def allowed_location
+        "sftp://#{hostname.to_s.downcase}:#{port}#{normalize_path(path)}".chomp("/")
+      end
+
+      # Usernames are passed to the `sftp` executable, so reject values that it could treat as options.
+      def validate_username!
+        return if username.nil?
+        return unless username.to_s.start_with?("-") || username.to_s.match?(/[[:cntrl:]]/)
+
+        raise(ArgumentError, "Invalid SFTP username: it cannot start with '-' or contain control characters")
+      end
+
+      # Set the path directly rather than parsing it as part of a URL, since a file name can contain
+      # characters such as `?`, `#`, `+` or `%` that a URL parser would treat as a query or as escapes.
+      #
+      # The supplied name is relative to this path, or to the login directory when this url has no path.
+      def child_path(name)
+        server     = port == 22 ? "sftp://#{hostname}" : "sftp://#{hostname}:#{port}"
+        child      = self.class.new(server, username: username, password: password, ssh_options: ssh_options)
+        child.path = (path.empty? ? "/#{name}" : ::File.join(path, name)).freeze
+        child.url  = "#{server}#{child.path}"
+        child
+      end
+
       def stream_reader(&block)
-        IOStreams.temp_file("iostreams-sftp-reader") do |temp_file|
-          sftp_download(path, temp_file.to_s)
-          ::File.open(temp_file.to_s, "rb") { |io| builder.reader(io, &block) }
+        Utils.private_temp_file("iostreams-sftp-reader") do |file_name|
+          sftp_download(path, file_name)
+          ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
         end
       end
 
       def stream_writer(&block)
-        IOStreams.temp_file("iostreams-sftp-writer") do |temp_file|
-          ::File.open(temp_file.to_s, "wb") { |io| builder.writer(io, &block) }
-          sftp_upload(temp_file.to_s, path)
-          temp_file.size
+        Utils.private_temp_file("iostreams-sftp-writer") do |file_name|
+          ::File.open(file_name, "wb") { |io| builder.writer(io, &block) }
+          sftp_upload(file_name, path)
+          ::File.size(file_name)
         end
       end
 
-      # Use sftp and sshpass executables to download to a local file
+      # Use the sftp executable to download to a local file, via sshpass when a password is supplied
       def sftp_download(remote_file_name, local_file_name)
         with_sftp_args do |args|
           Open3.popen2e(*args) do |writer, reader, waith_thr|
-            # Give time for remote sftp server to get ready to accept the password.
-            sleep self.class.before_password_wait_seconds
+            if password
+              # Give time for remote sftp server to get ready to accept the password.
+              sleep self.class.before_password_wait_seconds
 
-            writer.puts password
+              writer.puts password
 
-            # Give time for password to be processed and stdin to be passed to sftp process.
-            sleep self.class.sshpass_wait_seconds
+              # Give time for password to be processed and stdin to be passed to sftp process.
+              sleep self.class.sshpass_wait_seconds
+            end
 
-            writer.puts "get #{remote_file_name} #{local_file_name}"
+            writer.puts "get #{remote_file_name.inspect} #{local_file_name.inspect}"
             writer.puts "bye"
             writer.close
             out = reader.read.chomp
-            unless waith_thr.value.success?
-              raise(
-                Errors::CommunicationsFailure,
-                "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
-              )
-            end
+            raise_failure("Download", out) unless waith_thr.value.success?
 
             out
           rescue Errno::EPIPE
@@ -213,10 +256,7 @@ module IOStreams
             rescue StandardError
               nil
             end
-            raise(
-              Errors::CommunicationsFailure,
-              "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
-            )
+            raise_failure("Download", out)
           end
         end
       end
@@ -224,19 +264,16 @@ module IOStreams
       def sftp_upload(local_file_name, remote_file_name)
         with_sftp_args do |args|
           Open3.popen2e(*args) do |writer, reader, waith_thr|
-            writer.puts(password) if password
-            # Give time for password to be processed and stdin to be passed to sftp process.
-            sleep self.class.sshpass_wait_seconds
+            if password
+              writer.puts(password)
+              # Give time for password to be processed and stdin to be passed to sftp process.
+              sleep self.class.sshpass_wait_seconds
+            end
             writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
             writer.puts "bye"
             writer.close
             out = reader.read.chomp
-            unless waith_thr.value.success?
-              raise(
-                Errors::CommunicationsFailure,
-                "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
-              )
-            end
+            raise_failure("Upload", out) unless waith_thr.value.success?
 
             out
           rescue Errno::EPIPE
@@ -245,14 +282,21 @@ module IOStreams
             rescue StandardError
               nil
             end
-            raise(
-              Errors::CommunicationsFailure,
-              "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
-            )
+            raise_failure("Upload", out)
           end
         end
       end
 
+      # When the server does not prompt for a password, sftp reads the password line as a command
+      # and echoes it in its output, so remove it before the output is included in the error.
+      def raise_failure(action, out)
+        out = out.gsub(password.to_s, "[FILTERED]") if out && !password.to_s.empty?
+        raise(
+          Errors::CommunicationsFailure,
+          "#{action} failed calling #{self.class.sftp_bin}#{" via #{self.class.sshpass_bin}" if password}: #{out}"
+        )
+      end
+
       def with_sftp_args
         return yield sftp_args(ssh_options) if !ssh_options.key?("IdentityKey") && !ssh_options.key?("HostKey")
 
@@ -276,9 +320,9 @@ module IOStreams
       end
 
       def with_temp_file(options, option, value)
-        Utils.temp_file_name("iostreams-sftp-args", "key") do |file_name|
-          # sftp requires that private key is only readable by the current user
-          ::File.open(file_name, "wb", 0o600) { |io| io.write(value) }
+        # sftp requires that private key is only readable by the current user
+        Utils.private_temp_file("iostreams-sftp-args", "key") do |file_name|
+          ::File.binwrite(file_name, value)
 
           options[option] = file_name
           yield options
@@ -286,7 +330,8 @@ module IOStreams
       end
 
       def sftp_args(ssh_options)
-        args = [self.class.sshpass_bin, self.class.sftp_bin]
+        # sshpass is only needed to supply the password to sftp.
+        args = password ? [self.class.sshpass_bin, self.class.sftp_bin] : [self.class.sftp_bin]
         # Force sftp to use the password when supplied,
         # and stop sftp from prompting for a password when none was supplied.
         if password
@@ -305,29 +350,23 @@ module IOStreams
         ssh_options.each_pair { |key, value| args << "-o#{key}=#{value}" }
         args << "-b"
         args << "-"
+        # Stop sftp from treating the destination as an option.
+        args << "--"
         args << "#{username}@#{hostname}"
         args
       end
 
-      def build_ssh_options
-        options = ssh_options.dup
-        options[:logger]       ||= logger if defined?(SemanticLogger)
-        options[:port]         ||= port
-        options[:max_pkt_size] ||= 65_536
-        options[:password]     ||= @password
-        options
-      end
-
       def map_log_level
-        return "INFO" unless defined?(SemanticLogger)
-
-        case logger.level
+        level = IOStreams.logger&.level
+        case level
         when :trace
           "DEBUG3"
         when :warn
           "ERROR"
+        when Symbol
+          level.to_s
         else
-          logger.level.to_s
+          "INFO"
         end
       end
     end