iostreams 1.11.0 → 2.1.0

data/lib/io_streams/path.rb

@@ -1,5 +1,9 @@
 module IOStreams
   class Path < IOStreams::Stream
+    # Stream option names whose values must not be displayed, such as `passphrase` or `signer_passphrase`.
+    SENSITIVE_OPTION = /pass(phrase|word)|secret/i
+    private_constant :SENSITIVE_OPTION
+
     attr_accessor :path
 
     def initialize(path)
@@ -21,7 +25,7 @@ module IOStreams
 
       new_path         = dup
       new_path.builder = nil
-      new_path.path    = relative.start_with?(path) ? relative : ::File.join(path, relative)
+      new_path.path    = contains?(relative) ? relative : ::File.join(path, relative)
       new_path
     end
 
@@ -41,7 +45,7 @@ module IOStreams
     # Runs the pattern from the current path, returning the complete path for located files.
     #
     # See IOStreams::Paths::File.each for arguments.
-    def each_child(pattern = "*", **args, &block)
+    def each_child(pattern = "*", **args, &)
       raise NotImplementedError
     end
 
@@ -57,6 +61,24 @@ module IOStreams
       path
     end
 
+    # See Stream#reader.
+    #
+    # Raises [IOStreams::Errors::AccessDenied] when this path is not within any of the allowed paths,
+    # see `IOStreams.add_allowed_path`.
+    def reader(...)
+      authorize!
+      super
+    end
+
+    # See Stream#writer.
+    #
+    # Raises [IOStreams::Errors::AccessDenied] when this path is not within any of the allowed paths,
+    # see `IOStreams.add_allowed_path`.
+    def writer(...)
+      authorize!
+      super
+    end
+
     # Removes the last element of the path, the file name, before creating the entire path.
     # Returns self
     def mkpath
@@ -82,9 +104,9 @@ module IOStreams
     end
 
     # Cleanup an incomplete write to the target "file" if the copy fails.
-    # rubocop:disable Lint/SuppressedException
+    # rubocop:disable-next Lint/SuppressedException
     def copy_from(source, **args)
-      super(source, **args)
+      super
     rescue StandardError => e
       begin
         delete
@@ -92,7 +114,6 @@ module IOStreams
       end
       raise(e)
     end
-    # rubocop:enable Lint/SuppressedException
 
     # Moves the file by copying it to the new path and then deleting the current path.
     # Returns [IOStreams::Path] the target path.
@@ -190,15 +211,103 @@ module IOStreams
 
     def inspect
       str = "#<#{self.class.name}:#{path}"
-      str << " @builder=#{builder.streams.inspect}" if builder.streams
-      str << " @options=#{builder.options.inspect}" if builder.options
-      str << " pipeline=#{pipeline.inspect}>"
+      str << " @builder=#{redact(builder.streams).inspect}" if builder.streams
+      str << " @options=#{redact(builder.options).inspect}" if builder.options
+      str << " pipeline=#{redact(pipeline).inspect}>"
+    end
+
+    protected
+
+    # Raises [IOStreams::Errors::AccessDenied] when allowed paths have been added, see `IOStreams.add_allowed_path`,
+    # and this path is not within any of them.
+    def authorize!
+      return if IOStreams.allowed_paths.empty? || (@permitted_path && @permitted_path == path)
+
+      authorize_location!(allowed_location)
+    end
+
+    # Returns [true|false] whether this path is within the allowed paths, see `#authorize!`.
+    def allowed?
+      authorize!
+      true
+    rescue Errors::AccessDenied
+      false
     end
 
     private
 
+    # Returns [String] the normalized location of this path, which is compared against the allowed paths.
+    #
+    # Each path class that can be used with allowed paths overrides this method. Without it every path
+    # of that class is denied once allowed paths have been added.
+    #
+    # Raises [IOStreams::Errors::AccessDenied] when the location cannot be determined.
+    def allowed_location
+      raise(Errors::AccessDenied, "Access denied: #{self.class.name} does not support allowed paths")
+    end
+
+    # Raises [IOStreams::Errors::AccessDenied] when the supplied location, see `#allowed_location`,
+    # is not within any of the allowed paths.
+    def authorize_location!(location)
+      return if IOStreams.allowed_paths.any? { |allowed_path| within?(location, allowed_path) }
+
+      raise(Errors::AccessDenied, "Access denied to #{location}: it is not within any of the allowed paths")
+    end
+
+    # Returns [String] the supplied path with `.`, `..` and repeated `/` resolved, the way a remote server
+    # resolves them, without accessing it. The path always starts with `/`, and `..` cannot go above it.
+    def normalize_path(name)
+      segments = []
+      name.split("/").each do |segment|
+        case segment
+        when "", "."
+          next
+        when ".."
+          segments.pop
+        else
+          segments << segment
+        end
+      end
+      "/#{segments.join('/')}"
+    end
+
+    # Returns [true|false] whether a child found by `#each_child` is within the allowed paths, logging it when it is not.
+    def allowed_child?(child)
+      return true if child.allowed?
+
+      IOStreams.logger&.warn("Skipping #{child} since it is not within any of the allowed paths")
+      false
+    end
+
+    # Allows this exact path regardless of the allowed paths, for paths created by IOStreams itself such as temp files.
+    # Returns self
+    def permit!
+      @permitted_path = path
+      self
+    end
+
+    # Returns [Hash<Symbol:Hash>] the streams with the values of sensitive options replaced.
+    def redact(streams)
+      streams.transform_values do |options|
+        next options unless options.is_a?(Hash)
+
+        options.to_h { |name, value| [name, name.to_s.match?(SENSITIVE_OPTION) ? "[FILTERED]" : value] }
+      end
+    end
+
     def builder
       @builder ||= IOStreams::Builder.new(path)
     end
+
+    # Returns [true|false] whether the supplied path is this path, or is within this path.
+    # For example "a/b" contains "a/b/c.csv", but not "a/bc.csv".
+    def contains?(other)
+      path.empty? || within?(other, path)
+    end
+
+    # Returns [true|false] whether the path `child` is `parent`, or is within `parent`.
+    def within?(child, parent)
+      child == parent || child.start_with?(parent.end_with?("/") ? parent : "#{parent}/")
+    end
   end
 end

data/lib/io_streams/paths/file.rb

@@ -15,16 +15,16 @@ module IOStreams
       # Examples:
       #
       # # Case Insensitive file name lookup:
-      # IOStreams.path("ruby").glob("r*.md") { |name| puts name }
+      # IOStreams.path("ruby").each_child("r*.md") { |path| puts path }
       #
       # # Case Sensitive file name lookup:
-      # IOStreams.path("ruby").each("R*.md", case_sensitive: true) { |name| puts name }
+      # IOStreams.path("ruby").each_child("R*.md", case_sensitive: true) { |path| puts path }
       #
       # # Also return the names of directories found during the search:
-      # IOStreams.path("ruby").each("R*.md", directories: true) { |name| puts name }
+      # IOStreams.path("ruby").each_child("R*.md", directories: true) { |path| puts path }
       #
       # # Case Insensitive recursive file name lookup:
-      # IOStreams.path("ruby").glob("**/*.md") { |name| puts name }
+      # IOStreams.path("ruby").each_child("**/*.md") { |path| puts path }
       #
       # Parameters:
       #   pattern [String]
@@ -77,9 +77,9 @@ module IOStreams
       # "\a"        "a"              true     # escaped ordinary remains ordinary
       # "[\?]"      "?"              true     # can escape inside bracket expression
       #
-      # "*"         ".profile"       false    # wildcard doesn't match leading
-      # "*"         ".profile"       true     # period by default.
-      # ".*"        ".profile"       true                                   {hidden: true}
+      # "*"         ".profile"       false    # wildcard doesn't match leading period by default
+      # "*"         ".profile"       true     # unless hidden is enabled    {hidden: true}
+      # ".*"        ".profile"       true     # leading period is explicit
       #
       # "**/*.rb"   "main.rb"        false
       # "**/*.rb"   "./main.rb"      false
@@ -94,6 +94,8 @@ module IOStreams
                          case_sensitive: case_sensitive, directories: directories, hidden: hidden)
         end
 
+        authorize!
+
         flags = 0
         flags |= ::File::FNM_CASEFOLD unless case_sensitive
         flags |= ::File::FNM_DOTMATCH if hidden
@@ -116,7 +118,8 @@ module IOStreams
         results.each do |full_path|
           next if !directories && ::File.directory?(full_path)
 
-          yield(self.class.new(full_path))
+          child = self.class.new(full_path)
+          yield(child) if allowed_child?(child)
         end
       end
 
@@ -128,6 +131,8 @@ module IOStreams
         target = IOStreams.new(target_path)
         return super(target) unless target.is_a?(self.class)
 
+        authorize!
+        target.authorize!
         target.mkpath
         # In case the file is being moved across partitions
         FileUtils.move(path, target.to_s)
@@ -135,25 +140,30 @@ module IOStreams
       end
 
       def mkpath
+        authorize!
         dir = ::File.dirname(path)
-        FileUtils.mkdir_p(dir) unless ::File.exist?(dir)
+        FileUtils.mkdir_p(dir)
         self
       end
 
       def mkdir
-        FileUtils.mkdir_p(path) unless ::File.exist?(path)
+        authorize!
+        FileUtils.mkdir_p(path)
         self
       end
 
       def exist?
+        authorize!
         ::File.exist?(path)
       end
 
       def size
+        authorize!
         ::File.size(path)
       end
 
       def delete
+        authorize!
         return self unless exist?
 
         ::File.directory?(path) ? Dir.delete(path) : ::File.unlink(path)
@@ -161,6 +171,7 @@ module IOStreams
       end
 
       def delete_all
+        authorize!
         return self unless exist?
 
         ::File.directory?(path) ? FileUtils.remove_dir(path) : ::File.unlink(path)
@@ -169,11 +180,46 @@ module IOStreams
 
       # Returns the real path by stripping `.`, `..` and expands any symlinks.
       def realpath
+        authorize!
         self.class.new(::File.realpath(path))
       end
 
       private
 
+      # Returns [String] the real path of this file, following any symbolic links, which is compared
+      # against the allowed paths.
+      #
+      # The part of the path that does not exist yet, for example a file that is about to be written,
+      # is appended to the real path of the part that does exist. It cannot contain `.` or `..`, since
+      # what they refer to depends on directories that have not been created yet.
+      def allowed_location
+        existing = ::File.absolute_path?(path) ? path : ::File.join(Dir.pwd, path)
+        missing  = []
+        until present?(existing)
+          parent = ::File.dirname(existing)
+          break if parent == existing
+
+          missing.unshift(::File.basename(existing))
+          existing = parent
+        end
+
+        if missing.intersect?([".", ".."])
+          raise(Errors::AccessDenied, "Access denied to #{path}: '.' and '..' are not allowed after a missing directory")
+        end
+
+        ::File.join(::File.realpath(existing), *missing)
+      rescue SystemCallError => e
+        raise(Errors::AccessDenied, "Access denied to #{path}: #{e.message}")
+      end
+
+      # Returns [true|false] whether the file, directory or symbolic link exists, without following the link.
+      def present?(file_name)
+        ::File.lstat(file_name)
+        true
+      rescue SystemCallError
+        false
+      end
+
       # Read from file
       def stream_reader(&block)
         ::File.open(path, "rb") { |io| builder.reader(io, &block) }
@@ -189,7 +235,7 @@ module IOStreams
         begin
           ::File.open(path, "wb") { |io| builder.writer(io, &block) }
         rescue StandardError => e
-          ::File.unlink(path) if ::File.exist?(path)
+          ::FileUtils.rm_f(path)
           raise(e)
         end
       end

data/lib/io_streams/paths/http.rb

@@ -18,6 +18,11 @@ module IOStreams
       #     Full url showing all the optional elements that can be set via the url:
       #       https://username:password@hostname/path/file_name
       #
+      #     SECURITY WARNING:
+      #       A username and password supplied in the url remain part of it, so `#to_s` and `#url`
+      #       return them, as does any log or error message that includes the path.
+      #       Supply them with the `username:` and `password:` arguments instead.
+      #
       #   username: [String]
       #     When supplied, basic authentication is used with the username and password.
       #
@@ -26,7 +31,32 @@ module IOStreams
       #
       #   http_redirect_count: [Integer]
       #     Maximum number of http redirects to follow.
-      def initialize(url, username: nil, password: nil, http_redirect_count: 10, parameters: nil)
+      #     Set to 0 to disable following redirects entirely.
+      #     Default: 10
+      #
+      #   allow_hosts: [String | Array<String>]
+      #     Optional allow-list of host names that may be contacted, applied to the
+      #     supplied url and to every redirect that is followed.
+      #     When supplied, a request to any other host raises CommunicationsFailure.
+      #     Use this to limit Server Side Request Forgery (SSRF) exposure when the url
+      #     can be influenced by untrusted input.
+      #     Default: nil (any host is allowed).
+      #
+      #   maximum_file_size: [Integer]
+      #     Optional maximum number of bytes to download.
+      #     When the response body exceeds this size the download is aborted with a
+      #     CommunicationsFailure, protecting against unbounded (denial of service) responses.
+      #     Default: nil (no limit).
+      #
+      # Security notes:
+      # - Redirect targets are supplied by the remote server. Validating only the url that is
+      #   passed in is therefore not sufficient to prevent SSRF: use `allow_hosts` (or disable
+      #   redirects with `http_redirect_count: 0`) when the url is not fully trusted.
+      # - Basic authentication credentials are only sent to the original host. They are not
+      #   resent when a redirect points at a different scheme, host, or port, so that a
+      #   redirect cannot leak the credentials to another server.
+      def initialize(url, username: nil, password: nil, http_redirect_count: 10, parameters: nil,
+                     allow_hosts: nil, maximum_file_size: nil)
         uri = URI.parse(url)
         unless %w[http https].include?(uri.scheme)
           raise(
@@ -38,6 +68,8 @@ module IOStreams
         @username            = username || uri.user
         @password            = password || uri.password
         @http_redirect_count = http_redirect_count
+        @allow_hosts         = allow_hosts.nil? ? nil : Array(allow_hosts)
+        @maximum_file_size   = maximum_file_size
         @url                 = parameters ? "#{url}?#{URI.encode_www_form(parameters)}" : url
         super(uri.path)
       end
@@ -53,6 +85,8 @@ module IOStreams
 
       private
 
+      attr_reader :allow_hosts, :maximum_file_size
+
       # Read a file using an http get.
       #
       # For example:
@@ -63,27 +97,40 @@ module IOStreams
       #
       # Notes:
       # * Since Net::HTTP download only supports a push stream, the data is streamed into a tempfile first.
-      def stream_reader(&block)
-        handle_redirects(url, http_redirect_count, &block)
+      def stream_reader(&)
+        handle_redirects(url, http_redirect_count, &)
       end
 
       def handle_redirects(uri, http_redirect_count, &block)
         uri    = URI.parse(uri) unless uri.is_a?(URI)
         result = nil
-        raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
+
+        validate_uri!(uri)
 
         Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
           request = Net::HTTP::Get.new(uri)
-          request.basic_auth(username, password) if username
+          # Only send credentials to the original host to avoid leaking them via a redirect.
+          request.basic_auth(username, password) if username && same_origin?(uri)
 
           http.request(request) do |response|
-            raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}") if response.is_a?(Net::HTTPNotFound)
+            if response.is_a?(Net::HTTPNotFound)
+              raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{without_credentials(uri)}")
+            end
             if response.is_a?(Net::HTTPUnauthorized)
               raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
             end
 
             if response.is_a?(Net::HTTPRedirection)
-              new_uri = response["location"]
+              raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
+
+              location = response["location"]
+              unless location
+                raise(IOStreams::Errors::CommunicationsFailure,
+                      "Redirect missing location header: #{without_credentials(uri)}")
+              end
+
+              # Resolve relative redirects against the current uri.
+              new_uri = uri.merge(location)
               return handle_redirects(new_uri, http_redirect_count - 1, &block)
             end
 
@@ -92,8 +139,8 @@ module IOStreams
             end
 
             # Since Net::HTTP download only supports a push stream, write it to a tempfile first.
-            Utils.temp_file_name("iostreams_http") do |file_name|
-              ::File.open(file_name, "wb") { |io| response.read_body { |chunk| io.write(chunk) } }
+            Utils.private_temp_file("iostreams_http") do |file_name|
+              download_to_file(response, file_name)
               # Return a read stream
               result = ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
             end
@@ -101,6 +148,73 @@ module IOStreams
         end
         result
       end
+
+      # Validate that the host may be contacted, and that the scheme is still http(s)
+      # after following a redirect.
+      #
+      # A redirect must also be within the allowed paths, see `IOStreams.add_allowed_path`.
+      def validate_uri!(uri)
+        unless %w[http https].include?(uri.scheme)
+          raise(IOStreams::Errors::CommunicationsFailure,
+                "Invalid redirect, only http and https are supported: #{without_credentials(uri)}")
+        end
+        authorize_location!(http_location(uri)) unless IOStreams.allowed_paths.empty?
+        return if allow_hosts.nil? || allow_hosts.include?(uri.hostname)
+
+        raise(IOStreams::Errors::CommunicationsFailure, "Host not in the allowed list of hosts: #{uri.hostname}")
+      end
+
+      # Returns [String] the scheme, host, port and path of the url, which is compared against the allowed paths.
+      def allowed_location
+        http_location(original_uri)
+      end
+
+      # Returns [String] the scheme, host, port and path of the supplied uri, without any credentials or query.
+      #
+      # The path is decoded and `.` and `..` resolved, the way most web servers resolve them, so that for
+      # example `%2e%2e` cannot be used to leave an allowed path. A backslash is treated as a `/`.
+      def http_location(uri)
+        raise(Errors::AccessDenied, "Access denied: #{without_credentials(uri)} has no host") if uri.host.to_s.empty?
+
+        path = URI.decode_uri_component(uri.path).tr("\\", "/")
+        "#{uri.scheme}://#{uri.host.downcase}:#{uri.port}#{normalize_path(path)}".chomp("/")
+      rescue ArgumentError => e
+        raise(Errors::AccessDenied, "Access denied to #{without_credentials(uri)}: #{e.message}")
+      end
+
+      # Returns [String] the uri without any user name or password, for use in error messages.
+      def without_credentials(uri)
+        return uri.to_s unless uri.user
+
+        uri      = uri.dup
+        uri.user = nil
+        uri.to_s
+      end
+
+      def same_origin?(uri)
+        original = original_uri
+        uri.scheme == original.scheme && uri.hostname == original.hostname && uri.port == original.port
+      end
+
+      def original_uri
+        @original_uri ||= URI.parse(url)
+      end
+
+      def download_to_file(response, file_name)
+        size = 0
+        ::File.open(file_name, "wb") do |io|
+          response.read_body do |chunk|
+            size += chunk.bytesize
+            if maximum_file_size && (size > maximum_file_size)
+              raise(
+                IOStreams::Errors::CommunicationsFailure,
+                "Exceeded maximum allowed download size of #{maximum_file_size} bytes"
+              )
+            end
+            io.write(chunk)
+          end
+        end
+      end
     end
   end
 end

data/lib/io_streams/paths/matcher.rb

@@ -3,7 +3,7 @@ module IOStreams
     # Implement fnmatch logic for any path iterator
     class Matcher
       # Characters indicating that pattern matching is required
-      MATCH_START_CHARS = /[*?\[{]/.freeze
+      MATCH_START_CHARS = /[*?\[{]/
 
       attr_reader :path, :pattern, :flags
 
@@ -50,9 +50,9 @@ module IOStreams
           @path    = path || IOStreams.path
           @pattern = pattern
         else
-          new_path = elements[0..index - 1].join("/")
+          new_path = elements[0..(index - 1)].join("/")
           @path    = path.nil? ? IOStreams.path(new_path) : path.join(new_path)
-          @pattern = elements[index..-1].join("/")
+          @pattern = elements[index..].join("/")
         end
       end
     end