dependabot-python 0.79.0

data/lib/dependabot/python/update_checker/poetry_version_resolver.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+
+require "excon"
+require "toml-rb"
+
+require "dependabot/python/file_parser"
+require "dependabot/python/file_updater/pyproject_preparer"
+require "dependabot/python/update_checker"
+require "dependabot/shared_helpers"
+require "dependabot/python/version"
+require "dependabot/python/requirement"
+require "dependabot/errors"
+require "dependabot/python/native_helpers"
+require "dependabot/python/python_versions"
+
+module Dependabot
+  module Python
+    class UpdateChecker
+      # This class does version resolution for pyproject.toml files.
+      class PoetryVersionResolver
+        VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/.freeze
+
+        attr_reader :dependency, :dependency_files, :credentials
+
+        def initialize(dependency:, dependency_files:, credentials:,
+                       unlock_requirement:, latest_allowable_version:)
+          @dependency               = dependency
+          @dependency_files         = dependency_files
+          @credentials              = credentials
+          @latest_allowable_version = latest_allowable_version
+          @unlock_requirement       = unlock_requirement
+
+          check_private_sources_are_reachable
+        end
+
+        def latest_resolvable_version
+          return @latest_resolvable_version if @resolution_already_attempted
+
+          @resolution_already_attempted = true
+          @latest_resolvable_version ||= fetch_latest_resolvable_version
+        end
+
+        private
+
+        attr_reader :latest_allowable_version
+
+        def unlock_requirement?
+          @unlock_requirement
+        end
+
+        def fetch_latest_resolvable_version
+          @latest_resolvable_version_string ||=
+            SharedHelpers.in_a_temporary_directory do
+              write_temporary_dependency_files
+
+              if python_version && !pre_installed_python?(python_version)
+                run_poetry_command("pyenv install -s")
+                run_poetry_command(
+                  "pyenv exec pip install -r #{python_requirements_path}"
+                )
+              end
+
+              # Shell out to Poetry, which handles everything for us.
+              # Calling `lock` avoids doing an install.
+              run_poetry_command("pyenv exec poetry lock")
+
+              updated_lockfile =
+                if File.exist?("poetry.lock") then File.read("poetry.lock")
+                else File.read("pyproject.lock")
+                end
+              updated_lockfile = TomlRB.parse(updated_lockfile)
+
+              fetch_version_from_parsed_lockfile(updated_lockfile)
+            end
+          return unless @latest_resolvable_version_string
+
+          Python::Version.new(@latest_resolvable_version_string)
+        end
+
+        def fetch_version_from_parsed_lockfile(updated_lockfile)
+          updated_lockfile.fetch("package", []).
+            find { |d| d["name"] == dependency.name }.
+            fetch("version")
+        end
+
+        def write_temporary_dependency_files(update_pyproject: true)
+          dependency_files.each do |file|
+            path = file.name
+            FileUtils.mkdir_p(Pathname.new(path).dirname)
+            File.write(path, file.content)
+          end
+
+          # Overwrite the .python-version with updated content
+          File.write(".python-version", python_version) if python_version
+
+          # Overwrite the pyproject with updated content
+          if update_pyproject
+            File.write("pyproject.toml", updated_pyproject_content)
+          else
+            File.write("pyproject.toml", sanitized_pyproject_content)
+          end
+        end
+
+        def python_version
+          pyproject_object = TomlRB.parse(pyproject.content)
+          poetry_object = pyproject_object.dig("tool", "poetry")
+
+          requirement =
+            poetry_object&.dig("dependencies", "python") ||
+            poetry_object&.dig("dev-dependencies", "python")
+
+          return python_version_file&.content unless requirement
+
+          requirements =
+            Python::Requirement.requirements_array(requirement)
+
+          PythonVersions::PYTHON_VERSIONS.find do |version|
+            requirements.any? do |req|
+              req.satisfied_by?(Python::Version.new(version))
+            end
+          end
+        end
+
+        def pre_installed_python?(version)
+          PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version)
+        end
+
+        def updated_pyproject_content
+          content = pyproject.content
+          content = sanitize_pyproject_content(content)
+          content = freeze_other_dependencies(content)
+          content = unlock_target_dependency(content) if unlock_requirement?
+          content
+        end
+
+        def sanitized_pyproject_content
+          content = pyproject.content
+          content = sanitize_pyproject_content(content)
+          content
+        end
+
+        def sanitize_pyproject_content(pyproject_content)
+          Python::FileUpdater::PyprojectPreparer.
+            new(pyproject_content: pyproject_content).
+            sanitize
+        end
+
+        def freeze_other_dependencies(pyproject_content)
+          Python::FileUpdater::PyprojectPreparer.
+            new(pyproject_content: pyproject_content).
+            freeze_top_level_dependencies_except([dependency], lockfile)
+        end
+
+        def unlock_target_dependency(pyproject_content)
+          pyproject_object = TomlRB.parse(pyproject_content)
+          poetry_object = pyproject_object.dig("tool", "poetry")
+
+          %w(dependencies dev-dependencies).each do |type|
+            names = poetry_object[type]&.keys || []
+            pkg_name = names.find { |nm| normalise(nm) == dependency.name }
+            next unless pkg_name
+
+            if poetry_object.dig(type, pkg_name).is_a?(Hash)
+              poetry_object[type][pkg_name]["version"] =
+                updated_version_requirement_string
+            else
+              poetry_object[type][pkg_name] =
+                updated_version_requirement_string
+            end
+          end
+
+          TomlRB.dump(pyproject_object)
+        end
+
+        def check_private_sources_are_reachable
+          sources_to_check =
+            pyproject_sources +
+            config_variable_sources
+
+          sources_to_check.
+            map { |details| details["url"] }.
+            reject { |url| MAIN_PYPI_INDEXES.include?(url) }.
+            each do |url|
+              sanitized_url = url.gsub(%r{(?<=//).*(?=@)}, "redacted")
+
+              response = Excon.get(
+                url,
+                idempotent: true,
+                **SharedHelpers.excon_defaults
+              )
+
+              if response.status == 401 || response.status == 403
+                raise PrivateSourceAuthenticationFailure, sanitized_url
+              end
+            rescue Excon::Error::Timeout, Excon::Error::Socket
+              raise PrivateSourceTimedOut, sanitized_url
+            end
+        end
+
+        def updated_version_requirement_string
+          lower_bound_req = updated_version_req_lower_bound
+
+          # Add the latest_allowable_version as an upper bound. This means
+          # ignore conditions are considered when checking for the latest
+          # resolvable version.
+          #
+          # NOTE: This isn't perfect. If v2.x is ignored and v3 is out but
+          # unresolvable then the `latest_allowable_version` will be v3, and
+          # we won't be ignoring v2.x releases like we should be.
+          return lower_bound_req if latest_allowable_version.nil?
+          unless Python::Version.correct?(latest_allowable_version)
+            return lower_bound_req
+          end
+
+          lower_bound_req + ", <= #{latest_allowable_version}"
+        end
+
+        def updated_version_req_lower_bound
+          if dependency.version
+            ">= #{dependency.version}"
+          else
+            version_for_requirement =
+              dependency.requirements.map { |r| r[:requirement] }.compact.
+              reject { |req_string| req_string.start_with?("<") }.
+              select { |req_string| req_string.match?(VERSION_REGEX) }.
+              map { |req_string| req_string.match(VERSION_REGEX) }.
+              select { |version| Gem::Version.correct?(version) }.
+              max_by { |version| Gem::Version.new(version) }
+
+            ">= #{version_for_requirement || 0}"
+          end
+        end
+
+        def pyproject
+          dependency_files.find { |f| f.name == "pyproject.toml" }
+        end
+
+        def pyproject_lock
+          dependency_files.find { |f| f.name == "pyproject.lock" }
+        end
+
+        def poetry_lock
+          dependency_files.find { |f| f.name == "poetry.lock" }
+        end
+
+        def lockfile
+          poetry_lock || pyproject_lock
+        end
+
+        def python_version_file
+          dependency_files.find { |f| f.name == ".python-version" }
+        end
+
+        def run_poetry_command(command)
+          raw_response = nil
+          IO.popen(command, err: %i(child out)) do |process|
+            raw_response = process.read
+          end
+
+          # Raise an error with the output from the shell session if Pipenv
+          # returns a non-zero status
+          return if $CHILD_STATUS.success?
+
+          raise SharedHelpers::HelperSubprocessFailed.new(
+            raw_response,
+            command
+          )
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalise(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def config_variable_sources
+          @config_variable_sources ||=
+            credentials.
+            select { |cred| cred["type"] == "python_index" }.
+            map { |h| { "url" => h["index-url"].gsub(%r{/*$}, "") + "/" } }
+        end
+
+        def pyproject_sources
+          sources =
+            TomlRB.parse(pyproject.content).dig("tool", "poetry", "source") ||
+            []
+
+          @pyproject_sources ||=
+            sources.
+            map { |h| h.dup.merge("url" => h["url"].gsub(%r{/*$}, "") + "/") }
+        end
+
+        def python_requirements_path
+          File.join(NativeHelpers.python_helper_path, "requirements.txt")
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/update_checker/requirements_updater.rb

@@ -0,0 +1,365 @@
+# frozen_string_literal: true
+
+require "dependabot/python/requirement_parser"
+require "dependabot/python/update_checker"
+require "dependabot/python/version"
+require "dependabot/python/requirement"
+
+# rubocop:disable Metrics/ClassLength
+module Dependabot
+  module Python
+    class UpdateChecker
+      class RequirementsUpdater
+        PYPROJECT_OR_SEPARATOR = /(?<=[a-zA-Z0-9*])\s*\|+/.freeze
+        PYPROJECT_SEPARATOR = /#{PYPROJECT_OR_SEPARATOR}|,/.freeze
+
+        class UnfixableRequirement < StandardError; end
+
+        attr_reader :requirements, :update_strategy, :has_lockfile,
+                    :latest_version, :latest_resolvable_version
+
+        def initialize(requirements:, update_strategy:, has_lockfile:,
+                       latest_version:, latest_resolvable_version:)
+          @requirements = requirements
+          @update_strategy = update_strategy
+          @has_lockfile = has_lockfile
+
+          if latest_version
+            @latest_version = Python::Version.new(latest_version)
+          end
+
+          return unless latest_resolvable_version
+
+          @latest_resolvable_version =
+            Python::Version.new(latest_resolvable_version)
+        end
+
+        def updated_requirements
+          requirements.map do |req|
+            case req[:file]
+            when "setup.py" then updated_setup_requirement(req)
+            when "pyproject.toml" then updated_pyproject_requirement(req)
+            when "Pipfile" then updated_pipfile_requirement(req)
+            when /\.txt$|\.in$/ then updated_requirement(req)
+            else raise "Unexpected filename: #{req[:file]}"
+            end
+          end
+        end
+
+        private
+
+        # rubocop:disable Metrics/PerceivedComplexity
+        def updated_setup_requirement(req)
+          return req unless latest_resolvable_version
+          return req unless req.fetch(:requirement)
+          return req if new_version_satisfies?(req)
+
+          req_strings = req[:requirement].split(",").map(&:strip)
+
+          new_requirement =
+            if req_strings.any? { |r| requirement_class.new(r).exact? }
+              find_and_update_equality_match(req_strings)
+            elsif req_strings.any? { |r| r.start_with?("~=", "==") }
+              tw_req = req_strings.find { |r| r.start_with?("~=", "==") }
+              convert_to_range(tw_req, latest_resolvable_version)
+            else
+              update_requirements_range(req_strings)
+            end
+
+          req.merge(requirement: new_requirement)
+        rescue UnfixableRequirement
+          req.merge(requirement: :unfixable)
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def updated_pipfile_requirement(req)
+          # For now, we just proxy to updated_requirement. In future this
+          # method may treat Pipfile requirements differently.
+          updated_requirement(req)
+        end
+
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/PerceivedComplexity
+        def updated_pyproject_requirement(req)
+          return req unless latest_resolvable_version
+          return req unless req.fetch(:requirement)
+          return req if new_version_satisfies?(req) && !has_lockfile
+
+          # If the requirement uses || syntax then we always want to widen it
+          if req.fetch(:requirement).match?(PYPROJECT_OR_SEPARATOR)
+            return widen_pyproject_requirement(req)
+          end
+
+          # If the requirement is a development dependency we always want to
+          # bump it
+          if req.fetch(:groups).include?("dev-dependencies")
+            return update_pyproject_version(req)
+          end
+
+          case update_strategy
+          when :widen_ranges then widen_pyproject_requirement(req)
+          when :bump_versions then update_pyproject_version(req)
+          else raise "Unexpected update strategy: #{update_strategy}"
+          end
+        rescue UnfixableRequirement
+          req.merge(requirement: :unfixable)
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def update_pyproject_version(req)
+          requirement_strings = req[:requirement].split(",").map(&:strip)
+
+          new_requirement =
+            if requirement_strings.any? { |r| r.match?(/^=|^\d/) }
+              # If there is an equality operator, just update that. It must
+              # be binding and any other requirements will be being ignored
+              find_and_update_equality_match(requirement_strings)
+            elsif requirement_strings.any? { |r| r.start_with?("~", "^") }
+              # If a compatibility operator is being used, just bump its
+              # version (and remove any other requirements)
+              v_req = requirement_strings.find { |r| r.start_with?("~", "^") }
+              bump_version(v_req, latest_resolvable_version.to_s)
+            elsif new_version_satisfies?(req)
+              # Otherwise we're looking at a range operator. No change
+              # required if it's already satisfied
+              req.fetch(:requirement)
+            else
+              # But if it's not, update it
+              update_requirements_range(requirement_strings)
+            end
+
+          req.merge(requirement: new_requirement)
+        end
+
+        def widen_pyproject_requirement(req)
+          return req if new_version_satisfies?(req)
+
+          new_requirement =
+            if req[:requirement].match?(PYPROJECT_OR_SEPARATOR)
+              add_new_requirement_option(req[:requirement])
+            else
+              widen_requirement_range(req[:requirement])
+            end
+
+          req.merge(requirement: new_requirement)
+        end
+
+        def add_new_requirement_option(req_string)
+          option_to_copy = req_string.split(PYPROJECT_OR_SEPARATOR).last.
+                           split(PYPROJECT_SEPARATOR).first.strip
+          operator       = option_to_copy.gsub(/\d.*/, "").strip
+
+          new_option =
+            case operator
+            when "", "==", "==="
+              find_and_update_equality_match([option_to_copy])
+            when "~=", "~", "^"
+              bump_version(option_to_copy, latest_resolvable_version.to_s)
+            else
+              # We don't expect to see OR conditions used with range
+              # operators. If / when we see it, we should handle it.
+              raise "Unexpected operator: #{operator}"
+            end
+
+          # TODO: Match source spacing
+          "#{req_string.strip} || #{new_option.strip}"
+        end
+
+        def widen_requirement_range(req_string)
+          requirement_strings = req_string.split(",").map(&:strip)
+
+          if requirement_strings.any? { |r| r.match?(/(^=|^\d)[^*]*$/) }
+            # If there is an equality operator, just update that.
+            # (i.e., assume it's being used deliberately)
+            find_and_update_equality_match(requirement_strings)
+          elsif requirement_strings.any? { |r| r.start_with?("~", "^") } ||
+                requirement_strings.any? { |r| r.include?("*") }
+            # If a compatibility operator is being used, widen its
+            # range to include the new version
+            v_req = requirement_strings.
+                    find { |r| r.start_with?("~", "^") || r.include?("*") }
+            convert_to_range(v_req, latest_resolvable_version)
+          else
+            # Otherwise we have a range, and need to update the upper bound
+            update_requirements_range(requirement_strings)
+          end
+        end
+
+        # rubocop:disable Metrics/PerceivedComplexity
+        def updated_requirement(req)
+          return req unless latest_resolvable_version
+          return req unless req.fetch(:requirement)
+
+          requirement_strings = req[:requirement].split(",").map(&:strip)
+
+          new_requirement =
+            if requirement_strings.any? { |r| r.match?(/^[=\d]/) }
+              find_and_update_equality_match(requirement_strings)
+            elsif requirement_strings.any? { |r| r.start_with?("~=") }
+              tw_req = requirement_strings.find { |r| r.start_with?("~=") }
+              bump_version(tw_req, latest_resolvable_version.to_s)
+            elsif new_version_satisfies?(req)
+              req.fetch(:requirement)
+            else
+              update_requirements_range(requirement_strings)
+            end
+          req.merge(requirement: new_requirement)
+        rescue UnfixableRequirement
+          req.merge(requirement: :unfixable)
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def new_version_satisfies?(req)
+          requirement_class.
+            requirements_array(req.fetch(:requirement)).
+            any? { |r| r.satisfied_by?(latest_resolvable_version) }
+        end
+
+        def find_and_update_equality_match(requirement_strings)
+          if requirement_strings.any? { |r| requirement_class.new(r).exact? }
+            # True equality match
+            requirement_strings.find { |r| requirement_class.new(r).exact? }.
+              sub(
+                RequirementParser::VERSION,
+                latest_resolvable_version.to_s
+              )
+          else
+            # Prefix match
+            requirement_strings.find { |r| r.match?(/^(=+|\d)/) }.
+              sub(RequirementParser::VERSION) do |v|
+                at_same_precision(latest_resolvable_version.to_s, v)
+              end
+          end
+        end
+
+        def at_same_precision(new_version, old_version)
+          # return new_version unless old_version.include?("*")
+
+          count = old_version.split(".").count
+          precision = old_version.split(".").index("*") || count
+
+          new_version.
+            split(".").
+            first(count).
+            map.with_index { |s, i| i < precision ? s : "*" }.
+            join(".")
+        end
+
+        def update_requirements_range(requirement_strings)
+          ruby_requirements =
+            requirement_strings.map { |r| requirement_class.new(r) }
+
+          updated_requirement_strings = ruby_requirements.flat_map do |r|
+            next r.to_s if r.satisfied_by?(latest_resolvable_version)
+
+            case op = r.requirements.first.first
+            when "<", "<="
+              "<" + update_greatest_version(r.to_s, latest_resolvable_version)
+            when "!="
+              nil
+            when ">", ">="
+              raise UnfixableRequirement
+            else
+              raise "Unexpected op for unsatisfied requirement: #{op}"
+            end
+          end.compact
+
+          updated_requirement_strings.
+            sort_by { |r| requirement_class.new(r).requirements.first.last }.
+            map(&:to_s).join(",").delete(" ")
+        end
+
+        # Updates the version in a constraint to be the given version
+        def bump_version(req_string, version_to_be_permitted)
+          old_version = req_string.
+                        match(/(#{RequirementParser::VERSION})/).
+                        captures.first
+
+          req_string.sub(
+            old_version,
+            at_same_precision(version_to_be_permitted, old_version)
+          )
+        end
+
+        def convert_to_range(req_string, version_to_be_permitted)
+          # Construct an upper bound at the same precision that the original
+          # requirement was at (taking into account ~ dynamics)
+          index_to_update = index_to_update_for(req_string)
+          ub_segments = version_to_be_permitted.segments
+          ub_segments << 0 while ub_segments.count <= index_to_update
+          ub_segments = ub_segments[0..index_to_update]
+          ub_segments[index_to_update] += 1
+
+          lb_segments = lower_bound_segments_for_req(req_string)
+
+          # Ensure versions have the same length as each other (cosmetic)
+          length = [lb_segments.count, ub_segments.count].max
+          lb_segments.fill(0, lb_segments.count...length)
+          ub_segments.fill(0, ub_segments.count...length)
+
+          ">=#{lb_segments.join('.')},<#{ub_segments.join('.')}"
+        end
+
+        def lower_bound_segments_for_req(req_string)
+          requirement = requirement_class.new(req_string)
+          version = requirement.requirements.first.last
+          version = version.release if version.prerelease?
+
+          lb_segments = version.segments
+          lb_segments.pop while lb_segments.last.zero?
+
+          lb_segments
+        end
+
+        def index_to_update_for(req_string)
+          req = requirement_class.new(req_string.split(/[.\-]\*/).first)
+          version = req.requirements.first.last.release
+
+          if req_string.strip.start_with?("^")
+            version.segments.index { |i| i != 0 }
+          elsif req_string.include?("*")
+            version.segments.count - 1
+          elsif req_string.strip.start_with?("~=", "==")
+            version.segments.count - 2
+          elsif req_string.strip.start_with?("~")
+            req_string.split(".").count == 1 ? 0 : 1
+          else raise "Don't know how to convert #{req_string} to range"
+          end
+        end
+
+        # Updates the version in a "<" or "<=" constraint to allow the given
+        # version
+        def update_greatest_version(req_string, version_to_be_permitted)
+          if version_to_be_permitted.is_a?(String)
+            version_to_be_permitted =
+              Python::Version.new(version_to_be_permitted)
+          end
+          version = Python::Version.new(req_string.gsub(/<=?/, ""))
+          version = version.release if version.prerelease?
+
+          index_to_update = [
+            version.segments.map.with_index { |n, i| n.zero? ? 0 : i }.max,
+            version_to_be_permitted.segments.count - 1
+          ].min
+
+          new_segments = version.segments.map.with_index do |_, index|
+            if index < index_to_update
+              version_to_be_permitted.segments[index]
+            elsif index == index_to_update
+              version_to_be_permitted.segments[index] + 1
+            else 0
+            end
+          end
+
+          new_segments.join(".")
+        end
+
+        def requirement_class
+          Python::Requirement
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength