iostreams 2.1.0 → 2.2.0

data/lib/io_streams/path.rb

@@ -1,10 +1,6 @@
 module IOStreams
   class Path < IOStreams::Stream
-    # Stream option names whose values must not be displayed, such as `passphrase` or `signer_passphrase`.
-    SENSITIVE_OPTION = /pass(phrase|word)|secret/i
-    private_constant :SENSITIVE_OPTION
-
-    attr_accessor :path
+    attr_reader :path
 
     def initialize(path)
       raise(ArgumentError, "Path cannot be nil") if path.nil?
@@ -13,20 +9,30 @@ module IOStreams
       @path      = path.frozen? ? path : path.dup.freeze
       @io_stream = nil
       @builder   = nil
+      @root_name = nil
     end
 
+    # A copy of a root path is not a root path, see #root!, so that it can be changed.
+    def initialize_copy(source)
+      super
+      @root_name = nil
+    end
+
+    # Returns [IOStreams::Path] a new path with the elements joined to this path, created with the same arguments as
+    # this path, such as the credentials of an SFTP path, but without the streams, options, format and schema set on
+    # this one. Without any elements it returns this path itself.
+    #
     # If elements already contains the current path then it is used as is without
     # adding the current path for a second time
     def join(*elements)
       return self if elements.empty?
 
       elements = elements.collect(&:to_s)
-      relative = ::File.join(*elements)
+      relative = Utils::PathName.join(*elements)
 
-      new_path         = dup
-      new_path.builder = nil
-      new_path.path    = contains?(relative) ? relative : ::File.join(path, relative)
-      new_path
+      new_path = dup
+      new_path.reset_builder
+      new_path.replace_path(contains?(relative) ? relative : Utils::PathName.join(path, relative))
     end
 
     def relative?
@@ -103,18 +109,6 @@ module IOStreams
       raise NotImplementedError
     end
 
-    # Cleanup an incomplete write to the target "file" if the copy fails.
-    # rubocop:disable-next Lint/SuppressedException
-    def copy_from(source, **args)
-      super
-    rescue StandardError => e
-      begin
-        delete
-      rescue NotImplementedError
-      end
-      raise(e)
-    end
-
     # Moves the file by copying it to the new path and then deleting the current path.
     # Returns [IOStreams::Path] the target path.
     #
@@ -128,7 +122,8 @@ module IOStreams
       target
     end
 
-    # Returns [IOStreams::Path] the directory for this file.
+    # Returns [IOStreams::Path] the directory for this file, a new path created with the same arguments as this path,
+    # but without the streams, options, format and schema set on this one, see #join.
     #
     # If `path` does not include a directory name then "." is returned.
     #
@@ -139,10 +134,9 @@ module IOStreams
     #   IOStreams.path("test").directory            #=> "."
     #   IOStreams.path(".profile").directory        #=> "."
     def directory
-      new_path         = dup
-      new_path.builder = nil
-      new_path.path    = ::File.dirname(path)
-      new_path
+      new_path = dup
+      new_path.reset_builder
+      new_path.replace_path(Utils::PathName.dirname(path))
     end
 
     # When path is a file, deletes this file.
@@ -209,17 +203,85 @@ module IOStreams
       path == other.path
     end
 
+    # Returns [Array<Symbol>] the options for creating a path of this class whose values must not be displayed,
+    # such as a password, see .redact_options. A path class with sensitive options overrides it.
+    def self.sensitive_option_names
+      []
+    end
+
+    # Returns [Hash] the supplied options for creating a path of this class, such as `password:`, with the value of
+    # each sensitive option replaced with "[FILTERED]", so that they can be displayed, for example by an application
+    # that stores them to create the path later. See .sensitive_option_names and `IOStreams::Utils.redact_options`.
+    def self.redact_options(options)
+      Utils.redact_options(options, sensitive_option_names)
+    end
+
+    # Returns [String|nil] the deprecation warning for the supplied name or url of a path of this class, as the
+    # application supplies it to `IOStreams.path`, when IOStreams v3.0 reads it as another file or url, which says what
+    # to supply instead, so that it names the same one on both, or nil when v3.0 reads it the same way. A path class
+    # whose names v3.0 reads differently overrides it.
+    def self.name_deprecation(_name)
+      nil
+    end
+
+    # Returns [String|nil] the deprecation warning for the supplied name or url, see .name_deprecation, only when
+    # IOStreams v3.0 reads it as another location, which the allowed paths compare, see `IOStreams.add_allowed_path`,
+    # rather than only spelling its url another way. A path class whose names v3.0 reads as another location overrides
+    # it.
+    def self.location_deprecation(_name)
+      nil
+    end
+
+    # Returns [true|false] whether the url has no path, such as `sftp://host`, or `https://host?a=1`, which IOStreams
+    # v3.0 reads with the path `/`, see `IOStreams::Paths::SFTP.name_deprecation`. Its text is checked rather than
+    # parsed, so that a pattern with characters that a url cannot hold, such as braces, can be checked too.
+    def self.url_without_path?(url)
+      url.match?(%r{\A[a-z][a-z0-9+.-]*://[^/?#]*(?:[?#]|\z)}i)
+    end
+    private_class_method :url_without_path?
+
+    # Shows the full name of the path, see #display_name, without any credentials, and the options of its streams
+    # without the values of sensitive options, such as a passphrase, see `IOStreams.redact_stream_options`.
     def inspect
-      str = "#<#{self.class.name}:#{path}"
+      str = "#<#{self.class.name}:#{display_name}"
       str << " @builder=#{redact(builder.streams).inspect}" if builder.streams
       str << " @options=#{redact(builder.options).inspect}" if builder.options
       str << " pipeline=#{redact(pipeline).inspect}>"
     end
 
+    # Returns [String] the full name of this path, see #to_s, without any credentials, so that it can be logged
+    # or displayed. A user name, password or query in an SFTP or HTTP url is removed, since it can hold
+    # credentials, so unlike #to_s it cannot be used to create the path again. Each byte of a name that is not
+    # valid UTF-8 is shown as `\xHH`, see `IOStreams::Utils.display_text`.
+    #
+    # Example:
+    #   IOStreams.path("sftp://jack:secret@example.org/data/a.csv").display_name
+    #   # => "sftp://example.org/data/a.csv"
+    def display_name
+      Utils.display_text(to_s)
+    end
+
+    # Deprecated: a path can be used by many objects at once, such as a root path that file names are joined to, so
+    # changing it in place changes it for all of them. Use #join or #directory, which return a new path, without the
+    # streams, options, format and schema set on this one, or `IOStreams.path` with the new name. Reports a
+    # deprecation warning, see `IOStreams::Deprecation`.
+    def path=(path)
+      Deprecation.warn("#path= is deprecated, use #join or #directory, which return a new path, or IOStreams.path " \
+                       "with the new name")
+      @path = path
+    end
+
     protected
 
+    # Returns [self] after setting its path in place, for IOStreams itself, see #path=.
+    def replace_path(path)
+      @path = path
+      self
+    end
+
     # Raises [IOStreams::Errors::AccessDenied] when allowed paths have been added, see `IOStreams.add_allowed_path`,
-    # and this path is not within any of them.
+    # and this path is not within any of them, before #reader, #writer, `Stream#internal_reader` and
+    # `Stream#internal_writer` open it.
     def authorize!
       return if IOStreams.allowed_paths.empty? || (@permitted_path && @permitted_path == path)
 
@@ -275,7 +337,7 @@ module IOStreams
     def allowed_child?(child)
       return true if child.allowed?
 
-      IOStreams.logger&.warn("Skipping #{child} since it is not within any of the allowed paths")
+      IOStreams.logger&.warn("Skipping #{child.display_name} since it is not within any of the allowed paths")
       false
     end
 
@@ -286,13 +348,41 @@ module IOStreams
       self
     end
 
+    # Makes this path the root path of the supplied name, which `IOStreams.add_root` adds, and the whole process shares,
+    # so that changing it in place reports a deprecation warning, see #warn_change. A copy of it is not a root path, see
+    # #initialize_copy.
+    # Returns self
+    def root!(name)
+      @root_name = name
+      self
+    end
+
+    # Returns [true|false] whether the streams of this path read a regular local file in v3.0: the file itself, or the
+    # temp file that it is downloaded into, as v2.2 reads, unless v3.0 reads it from another IO, see #v3_source, such as
+    # an S3 object, see `IOStreams::Stream#v3_reads_file?`.
+    def v3_reads_file?
+      v3_source.nil?
+    end
+
+    # Changing a root path in place, see #root!, such as with #stream, is deprecated, since it changes the root for the
+    # whole process. v3.0 keeps its own frozen copy of the root instead, so that changing a path no longer changes the
+    # root: the path that `IOStreams.add_root` returns is that copy, so changing it raises FrozenError, while a path
+    # supplied to `IOStreams.add_root` is not, and `IOStreams.root`, `IOStreams.roots` and `IOStreams.join` without
+    # elements return copies of it, so changing them changes only the copy. A copy of the root, or a path joined to it,
+    # can be changed in either version, which the warning names for the root that this path is.
+    def warn_change(method_name)
+      return unless @root_name
+
+      root, join = @root_name == :default ? ["", ""] : ["(#{@root_name.inspect})", ", root: #{@root_name.inspect}"]
+      Deprecation.warn("#{method_name} is deprecated on a root path, which the whole process shares, since v3.0 " \
+                       "keeps its own frozen copy of the root, and returns copies of it, so changing this path no " \
+                       "longer changes the root, change a copy of it instead, such as IOStreams.root#{root}.dup, or a " \
+                       "path joined to it, such as IOStreams.join(\"a.csv\"#{join})")
+    end
+
     # Returns [Hash<Symbol:Hash>] the streams with the values of sensitive options replaced.
     def redact(streams)
-      streams.transform_values do |options|
-        next options unless options.is_a?(Hash)
-
-        options.to_h { |name, value| [name, name.to_s.match?(SENSITIVE_OPTION) ? "[FILTERED]" : value] }
-      end
+      IOStreams.redact_stream_options(streams)
     end
 
     def builder

data/lib/io_streams/paths/file.rb

@@ -3,13 +3,64 @@ require "fileutils"
 module IOStreams
   module Paths
     class File < IOStreams::Path
-      attr_accessor :create_path
+      # A pattern supplied to #each_child that is not a String, such as a Pathname, is deprecated.
+      PATTERN_DEPRECATION = "a pattern that is not a String is deprecated for #each_child and #children, since v3.0 " \
+                            "raises ArgumentError for it, supply a String instead, such as pattern.to_s".freeze
+      private_constant :PATTERN_DEPRECATION
+
+      # Moving a file onto an existing directory is deprecated, see #move_to.
+      MOVE_DEPRECATION = "#move_to onto an existing directory is deprecated, since v3.0 raises ArgumentError for it, " \
+                         "supply the file within the directory instead, such as " \
+                         "move_to(IOStreams.path(directory).join(path.basename))".freeze
+      private_constant :MOVE_DEPRECATION
+
+      attr_reader :create_path
 
       def initialize(file_name, create_path: true)
         @create_path = create_path
         super(file_name)
       end
 
+      # Sets whether writing the file creates its directory first, see #mkpath.
+      def create_path=(create_path)
+        warn_change("#create_path=")
+        @create_path = create_path
+      end
+
+      # Returns [String|nil] the deprecation warning for a name that IOStreams v3.0 reads as another file, see
+      # `IOStreams::Path.name_deprecation`, or nil for one that it reads the same way:
+      # - A name of `~`, or starting with `~/`, which is within a directory called `~` in the current directory, while
+      #   v3.0 reads it within the home directory. A name such as `~user/a.csv` or `~$Book1.xlsx` is the same on both.
+      # - A `file://` url, such as `file:///data/a.csv` or `file://~/a.csv`, which is a file name relative to the current
+      #   directory, within a directory called `file:`, while v3.0 reads it as the local file that it names, such as
+      #   `/data/a.csv`, or within the home directory.
+      def self.name_deprecation(name)
+        if file_url?(name)
+          "a file:// url is deprecated, since v3.0 reads it as the local file that it names, such as /data/a.csv for " \
+            "file:///data/a.csv, or a file within the home directory for file://~/a.csv, rather than as a file name " \
+            "relative to the current directory: supply the name of the file instead, such as /data/a.csv, or " \
+            "File.expand_path(\"~/a.csv\") for the home directory"
+        elsif name == "~" || name.start_with?("~/")
+          "a local file name of ~, or starting with ~/, is deprecated, since v3.0 reads it within the home directory, " \
+            "rather than within a directory called ~ in the current directory: expand it with File.expand_path for " \
+            "the home directory, or start it with ./~/ for the directory called ~"
+        end
+      end
+
+      # Returns [String|nil] the deprecation warning for a name that IOStreams v3.0 reads as another file, see
+      # .name_deprecation, which is in another directory, and so another allowed path.
+      def self.location_deprecation(name)
+        name_deprecation(name)
+      end
+
+      # Returns [true|false] whether the name is a `file://` url, in any case, as IOStreams v3.0 takes it, see
+      # .name_deprecation. Its first bytes are compared, so that a name that is not valid in its encoding, such as a
+      # Latin-1 name, can be checked too.
+      def self.file_url?(name)
+        name.include?("://") && name.byteslice(0, 7).b.casecmp?("file://")
+      end
+      private_class_method :file_url?
+
       # Yields Paths within the current path.
       #
       # Examples:
@@ -88,20 +139,28 @@ module IOStreams
       # "**.rb"     "./main.rb"      false
       # "**.rb"     "lib/song.rb"    true
       # "*"         "dave/.profile"  true
+      #
+      # A pattern that is not a String, such as a Pathname, is deprecated, and reports a deprecation warning, see
+      # `IOStreams::Deprecation`.
       def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
         unless block_given?
           return to_enum(__method__, pattern,
                          case_sensitive: case_sensitive, directories: directories, hidden: hidden)
         end
 
+        Deprecation.warn(PATTERN_DEPRECATION) unless pattern.is_a?(String)
         authorize!
 
+        # A pattern that is a Pathname is taken by its #to_path, as `::File.join` takes it, which v2.1 joined the pattern
+        # with: `IOStreams::Utils::PathName.join` does not take it on JRuby, where it joins the names by their bytes, and
+        # the case-insensitive retry below matches and downcases the pattern as a String.
+        pattern = pattern.to_path if pattern.respond_to?(:to_path)
+
         flags = 0
         flags |= ::File::FNM_CASEFOLD unless case_sensitive
         flags |= ::File::FNM_DOTMATCH if hidden
 
-        # Dir.each_child("testdir") {|x| puts "Got #{x}" }
-        full_pattern = ::File.join(path, pattern)
+        full_pattern = Utils::PathName.join(path, pattern)
 
         results = Dir.glob(full_pattern, flags)
 
@@ -111,7 +170,7 @@ module IOStreams
         if results.empty? && !case_sensitive && pattern.match?(/[A-Z]/)
           # Try converting the pattern to lowercase and re-matching
           lowercase_pattern = pattern.downcase
-          lowercase_full_pattern = ::File.join(path, lowercase_pattern)
+          lowercase_full_pattern = Utils::PathName.join(path, lowercase_pattern)
           results = Dir.glob(lowercase_full_pattern, flags)
         end
 
@@ -127,12 +186,16 @@ module IOStreams
       #
       # Notes:
       # - Can copy across buckets.
+      # - A target that is an existing directory, which the file is moved into, as `mv` does, returning the directory, is
+      #   deprecated, and reports a deprecation warning, see `IOStreams::Deprecation`: supply the file within it instead,
+      #   such as `move_to(IOStreams.path("/archive").join(path.basename))`.
       def move_to(target_path)
         target = IOStreams.new(target_path)
         return super(target) unless target.is_a?(self.class)
 
         authorize!
         target.authorize!
+        Deprecation.warn(MOVE_DEPRECATION) if ::File.directory?(target.path)
         target.mkpath
         # In case the file is being moved across partitions
         FileUtils.move(path, target.to_s)
@@ -141,7 +204,7 @@ module IOStreams
 
       def mkpath
         authorize!
-        dir = ::File.dirname(path)
+        dir = Utils::PathName.dirname(path)
         FileUtils.mkdir_p(dir)
         self
       end
@@ -193,13 +256,13 @@ module IOStreams
       # is appended to the real path of the part that does exist. It cannot contain `.` or `..`, since
       # what they refer to depends on directories that have not been created yet.
       def allowed_location
-        existing = ::File.absolute_path?(path) ? path : ::File.join(Dir.pwd, path)
+        existing = ::File.absolute_path?(path) ? path : Utils::PathName.join(Dir.pwd, path)
         missing  = []
         until present?(existing)
-          parent = ::File.dirname(existing)
+          parent = Utils::PathName.dirname(existing)
           break if parent == existing
 
-          missing.unshift(::File.basename(existing))
+          missing.unshift(Utils::PathName.basename(existing))
           existing = parent
         end
 
@@ -207,7 +270,7 @@ module IOStreams
           raise(Errors::AccessDenied, "Access denied to #{path}: '.' and '..' are not allowed after a missing directory")
         end
 
-        ::File.join(::File.realpath(existing), *missing)
+        Utils::PathName.join(::File.realpath(existing), *missing)
       rescue SystemCallError => e
         raise(Errors::AccessDenied, "Access denied to #{path}: #{e.message}")
       end

data/lib/io_streams/paths/http.rb

@@ -6,6 +6,19 @@ module IOStreams
     class HTTP < IOStreams::Path
       attr_reader :username, :password, :http_redirect_count, :url
 
+      # The password, and the query `parameters`, which can hold an access key, see IOStreams::Path.redact_options.
+      def self.sensitive_option_names = %i[password parameters]
+
+      # Returns [String|nil] the deprecation warning for an HTTP(S) url without a path, such as `https://host`, or
+      # `https://host?a=1`, which IOStreams v3.0 reads with the path `/`, as `https://host/?a=1`, see
+      # `IOStreams::Path.name_deprecation`.
+      def self.name_deprecation(url)
+        return unless url_without_path?(url)
+
+        "an HTTP url without a path, such as https://host or https://host?a=1, is deprecated, since v3.0 adds a / " \
+          "after the host, https://host/?a=1, which changes its #to_s and #path: supply the url with the /"
+      end
+
       # Stream to/from a remote file over http(s).
       #
       # Parameters:
@@ -21,7 +34,8 @@ module IOStreams
       #     SECURITY WARNING:
       #       A username and password supplied in the url remain part of it, so `#to_s` and `#url`
       #       return them, as does any log or error message that includes the path.
-      #       Supply them with the `username:` and `password:` arguments instead.
+      #       Supply them with the `username:` and `password:` arguments instead,
+      #       and log `#display_name`, which never includes them.
       #
       #   username: [String]
       #     When supplied, basic authentication is used with the username and password.
@@ -72,6 +86,7 @@ module IOStreams
         @maximum_file_size   = maximum_file_size
         @url                 = parameters ? "#{url}?#{URI.encode_www_form(parameters)}" : url
         super(uri.path)
+        warn_url_without_host(uri)
       end
 
       # Does not support relative file names since there is no concept of current working directory
@@ -83,17 +98,48 @@ module IOStreams
         url
       end
 
+      # Returns [String] the url without the user name, password, query or fragment, which can hold credentials,
+      # such as the signature of a pre-signed url, or the `parameters:` supplied to this path.
+      def display_name
+        uri          = URI.parse(url)
+        uri.user     = nil
+        uri.query    = nil
+        uri.fragment = nil
+        uri.to_s
+      end
+
       private
 
       attr_reader :allow_hosts, :maximum_file_size
 
+      # Reports a deprecation warning for a url without a host, such as `http:///a.csv`, which IOStreams 3.0 raises
+      # ArgumentError for when the path is created, so that a mistake in a configured url is reported wherever the
+      # code runs.
+      def warn_url_without_host(uri)
+        return unless uri.host.to_s.empty?
+
+        Deprecation.warn("the HTTP url #{display_name.inspect} is deprecated, since it names no host, name one, such " \
+                         "as 'https://example.org/a.csv'")
+      end
+
+      # The methods of the IO that v3.0 reads the response from, see #v3_source: those of an IO that read, and #close.
+      V3_SOURCE_METHODS = [*IOStreams::Reader::V3_READ_METHODS, :close].freeze
+      private_constant :V3_SOURCE_METHODS
+
+      # Returns [Array(Array<Symbol>, String)] the methods of the IO that v3.0 reads the response from, in place of the
+      # temp File that v2.2 downloads it into, and its name, see `IOStreams::Stream#v3_source`: v3.0 reads the response as
+      # the application reads it, with an IO that only reads.
+      def v3_source
+        [V3_SOURCE_METHODS, "the IO of an HTTP path"]
+      end
+
       # Read a file using an http get.
       #
       # For example:
       #   IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').reader {|file| puts file.read}
       #
       # Read the file without unzipping and streaming the first file in the zip:
-      #   IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').stream(:none).reader {|file| puts file.read}
+      #   IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').raw.reader {|file| puts file.read}
       #
       # Notes:
       # * Since Net::HTTP download only supports a push stream, the data is streamed into a tempfile first.

data/lib/io_streams/paths/matcher.rb

@@ -38,20 +38,23 @@ module IOStreams
 
       private
 
+      # The directory is created without reporting a deprecation warning for its name, see `IOStreams::Paths.build`,
+      # since it is derived from the pattern, whose name `IOStreams.each_child` reports instead, and since other gems
+      # create a matcher themselves, such as Rocket Job, whose users cannot change its lines that a warning would name.
       def extract_optimized_path(path, pattern)
         elements = pattern.split("/")
         index    = elements.find_index { |e| e.match(MATCH_START_CHARS) }
         if index.nil?
           # No index means it has no pattern.
-          @path    = path.nil? ? IOStreams.path(pattern) : path.join(pattern)
+          @path    = path.nil? ? Paths.build(pattern) : path.join(pattern)
           @pattern = nil
         elsif index.zero?
           # Cannot optimize path since the very first entry contains a wildcard
-          @path    = path || IOStreams.path
+          @path    = path || Paths.build("")
           @pattern = pattern
         else
           new_path = elements[0..(index - 1)].join("/")
-          @path    = path.nil? ? IOStreams.path(new_path) : path.join(new_path)
+          @path    = path.nil? ? Paths.build(new_path) : path.join(new_path)
           @pattern = elements[index..].join("/")
         end
       end