dependabot-core 0.76.1

data/lib/dependabot/update_checkers/python/pip/poetry_version_resolver.rb

@@ -0,0 +1,300 @@
+# frozen_string_literal: true
+
+require "excon"
+require "toml-rb"
+
+require "dependabot/file_parsers/python/pip"
+require "dependabot/file_updaters/python/pip/pyproject_preparer"
+require "dependabot/update_checkers/python/pip"
+require "dependabot/shared_helpers"
+require "dependabot/utils/python/version"
+require "dependabot/utils/python/requirement"
+require "dependabot/errors"
+require "python_versions"
+
+module Dependabot
+  module UpdateCheckers
+    module Python
+      class Pip
+        # This class does version resolution for pyproject.toml files.
+        class PoetryVersionResolver
+          VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/.freeze
+
+          attr_reader :dependency, :dependency_files, :credentials
+
+          def initialize(dependency:, dependency_files:, credentials:,
+                         unlock_requirement:, latest_allowable_version:)
+            @dependency               = dependency
+            @dependency_files         = dependency_files
+            @credentials              = credentials
+            @latest_allowable_version = latest_allowable_version
+            @unlock_requirement       = unlock_requirement
+
+            check_private_sources_are_reachable
+          end
+
+          def latest_resolvable_version
+            return @latest_resolvable_version if @resolution_already_attempted
+
+            @resolution_already_attempted = true
+            @latest_resolvable_version ||= fetch_latest_resolvable_version
+          end
+
+          private
+
+          attr_reader :latest_allowable_version
+
+          def unlock_requirement?
+            @unlock_requirement
+          end
+
+          def fetch_latest_resolvable_version
+            @latest_resolvable_version_string ||=
+              SharedHelpers.in_a_temporary_directory do
+                write_temporary_dependency_files
+
+                if python_version && !pre_installed_python?(python_version)
+                  run_poetry_command("pyenv install -s")
+                  run_poetry_command(
+                    "pyenv exec pip install -r #{python_requirements_path}"
+                  )
+                end
+
+                # Shell out to Poetry, which handles everything for us.
+                # Calling `lock` avoids doing an install.
+                run_poetry_command("pyenv exec poetry lock")
+
+                updated_lockfile =
+                  if File.exist?("poetry.lock") then File.read("poetry.lock")
+                  else File.read("pyproject.lock")
+                  end
+                updated_lockfile = TomlRB.parse(updated_lockfile)
+
+                fetch_version_from_parsed_lockfile(updated_lockfile)
+              end
+            return unless @latest_resolvable_version_string
+
+            Utils::Python::Version.new(@latest_resolvable_version_string)
+          end
+
+          def fetch_version_from_parsed_lockfile(updated_lockfile)
+            updated_lockfile.fetch("package", []).
+              find { |d| d["name"] == dependency.name }.
+              fetch("version")
+          end
+
+          def write_temporary_dependency_files(update_pyproject: true)
+            dependency_files.each do |file|
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, file.content)
+            end
+
+            # Overwrite the .python-version with updated content
+            File.write(".python-version", python_version) if python_version
+
+            # Overwrite the pyproject with updated content
+            if update_pyproject
+              File.write("pyproject.toml", updated_pyproject_content)
+            else
+              File.write("pyproject.toml", sanitized_pyproject_content)
+            end
+          end
+
+          def python_version
+            pyproject_object = TomlRB.parse(pyproject.content)
+            poetry_object = pyproject_object.dig("tool", "poetry")
+
+            requirement =
+              poetry_object&.dig("dependencies", "python") ||
+              poetry_object&.dig("dev-dependencies", "python")
+
+            return python_version_file&.content unless requirement
+
+            requirements =
+              Utils::Python::Requirement.requirements_array(requirement)
+
+            PythonVersions::PYTHON_VERSIONS.find do |version|
+              requirements.any? do |req|
+                req.satisfied_by?(Utils::Python::Version.new(version))
+              end
+            end
+          end
+
+          def pre_installed_python?(version)
+            PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version)
+          end
+
+          def updated_pyproject_content
+            content = pyproject.content
+            content = sanitize_pyproject_content(content)
+            content = freeze_other_dependencies(content)
+            content = unlock_target_dependency(content) if unlock_requirement?
+            content
+          end
+
+          def sanitized_pyproject_content
+            content = pyproject.content
+            content = sanitize_pyproject_content(content)
+            content
+          end
+
+          def sanitize_pyproject_content(pyproject_content)
+            FileUpdaters::Python::Pip::PyprojectPreparer.
+              new(pyproject_content: pyproject_content).
+              sanitize
+          end
+
+          def freeze_other_dependencies(pyproject_content)
+            FileUpdaters::Python::Pip::PyprojectPreparer.
+              new(pyproject_content: pyproject_content).
+              freeze_top_level_dependencies_except([dependency], lockfile)
+          end
+
+          def unlock_target_dependency(pyproject_content)
+            pyproject_object = TomlRB.parse(pyproject_content)
+            poetry_object = pyproject_object.dig("tool", "poetry")
+
+            %w(dependencies dev-dependencies).each do |type|
+              names = poetry_object[type]&.keys || []
+              pkg_name = names.find { |nm| normalise(nm) == dependency.name }
+              next unless pkg_name
+
+              if poetry_object.dig(type, pkg_name).is_a?(Hash)
+                poetry_object[type][pkg_name]["version"] =
+                  updated_version_requirement_string
+              else
+                poetry_object[type][pkg_name] =
+                  updated_version_requirement_string
+              end
+            end
+
+            TomlRB.dump(pyproject_object)
+          end
+
+          def check_private_sources_are_reachable
+            sources_to_check =
+              pyproject_sources +
+              config_variable_sources
+
+            sources_to_check.
+              map { |details| details["url"] }.
+              reject { |url| MAIN_PYPI_INDEXES.include?(url) }.
+              each do |url|
+                sanitized_url = url.gsub(%r{(?<=//).*(?=@)}, "redacted")
+
+                response = Excon.get(
+                  url,
+                  idempotent: true,
+                  **SharedHelpers.excon_defaults
+                )
+
+                if response.status == 401 || response.status == 403
+                  raise PrivateSourceAuthenticationFailure, sanitized_url
+                end
+              rescue Excon::Error::Timeout, Excon::Error::Socket
+                raise PrivateSourceTimedOut, sanitized_url
+              end
+          end
+
+          def updated_version_requirement_string
+            lower_bound_req = updated_version_req_lower_bound
+
+            # Add the latest_allowable_version as an upper bound. This means
+            # ignore conditions are considered when checking for the latest
+            # resolvable version.
+            #
+            # NOTE: This isn't perfect. If v2.x is ignored and v3 is out but
+            # unresolvable then the `latest_allowable_version` will be v3, and
+            # we won't be ignoring v2.x releases like we should be.
+            return lower_bound_req if latest_allowable_version.nil?
+            unless Utils::Python::Version.correct?(latest_allowable_version)
+              return lower_bound_req
+            end
+
+            lower_bound_req + ", <= #{latest_allowable_version}"
+          end
+
+          def updated_version_req_lower_bound
+            if dependency.version
+              ">= #{dependency.version}"
+            else
+              version_for_requirement =
+                dependency.requirements.map { |r| r[:requirement] }.compact.
+                reject { |req_string| req_string.start_with?("<") }.
+                select { |req_string| req_string.match?(VERSION_REGEX) }.
+                map { |req_string| req_string.match(VERSION_REGEX) }.
+                select { |version| Gem::Version.correct?(version) }.
+                max_by { |version| Gem::Version.new(version) }
+
+              ">= #{version_for_requirement || 0}"
+            end
+          end
+
+          def pyproject
+            dependency_files.find { |f| f.name == "pyproject.toml" }
+          end
+
+          def pyproject_lock
+            dependency_files.find { |f| f.name == "pyproject.lock" }
+          end
+
+          def poetry_lock
+            dependency_files.find { |f| f.name == "poetry.lock" }
+          end
+
+          def lockfile
+            poetry_lock || pyproject_lock
+          end
+
+          def python_version_file
+            dependency_files.find { |f| f.name == ".python-version" }
+          end
+
+          def run_poetry_command(command)
+            raw_response = nil
+            IO.popen(command, err: %i(child out)) do |process|
+              raw_response = process.read
+            end
+
+            # Raise an error with the output from the shell session if Pipenv
+            # returns a non-zero status
+            return if $CHILD_STATUS.success?
+
+            raise SharedHelpers::HelperSubprocessFailed.new(
+              raw_response,
+              command
+            )
+          end
+
+          # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+          def normalise(name)
+            name.downcase.gsub(/[-_.]+/, "-")
+          end
+
+          def config_variable_sources
+            @config_variable_sources ||=
+              credentials.
+              select { |cred| cred["type"] == "python_index" }.
+              map { |h| { "url" => h["index-url"].gsub(%r{/*$}, "") + "/" } }
+          end
+
+          def pyproject_sources
+            sources =
+              TomlRB.parse(pyproject.content).dig("tool", "poetry", "source") ||
+              []
+
+            @pyproject_sources ||=
+              sources.
+              map { |h| h.dup.merge("url" => h["url"].gsub(%r{/*$}, "") + "/") }
+          end
+
+          def python_requirements_path
+            project_root = File.join(File.dirname(__FILE__), "../../../../..")
+            File.join(project_root, "helpers/python/requirements.txt")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/update_checkers/python/pip/requirements_updater.rb

@@ -0,0 +1,367 @@
+# frozen_string_literal: true
+
+require "dependabot/update_checkers/python/pip"
+require "dependabot/utils/python/version"
+require "dependabot/utils/python/requirement"
+
+# rubocop:disable Metrics/ClassLength
+module Dependabot
+  module UpdateCheckers
+    module Python
+      class Pip
+        class RequirementsUpdater
+          PYPROJECT_OR_SEPARATOR = /(?<=[a-zA-Z0-9*])\s*\|+/.freeze
+          PYPROJECT_SEPARATOR = /#{PYPROJECT_OR_SEPARATOR}|,/.freeze
+
+          class UnfixableRequirement < StandardError; end
+
+          attr_reader :requirements, :update_strategy, :has_lockfile,
+                      :latest_version, :latest_resolvable_version
+
+          def initialize(requirements:, update_strategy:, has_lockfile:,
+                         latest_version:, latest_resolvable_version:)
+            @requirements = requirements
+            @update_strategy = update_strategy
+            @has_lockfile = has_lockfile
+
+            if latest_version
+              @latest_version = Utils::Python::Version.new(latest_version)
+            end
+
+            return unless latest_resolvable_version
+
+            @latest_resolvable_version =
+              Utils::Python::Version.new(latest_resolvable_version)
+          end
+
+          def updated_requirements
+            requirements.map do |req|
+              case req[:file]
+              when "setup.py" then updated_setup_requirement(req)
+              when "pyproject.toml" then updated_pyproject_requirement(req)
+              when "Pipfile" then updated_pipfile_requirement(req)
+              when /\.txt$|\.in$/ then updated_requirement(req)
+              else raise "Unexpected filename: #{req[:file]}"
+              end
+            end
+          end
+
+          private
+
+          # rubocop:disable Metrics/PerceivedComplexity
+          def updated_setup_requirement(req)
+            return req unless latest_resolvable_version
+            return req unless req.fetch(:requirement)
+            return req if new_version_satisfies?(req)
+
+            req_strings = req[:requirement].split(",").map(&:strip)
+
+            new_requirement =
+              if req_strings.any? { |r| requirement_class.new(r).exact? }
+                find_and_update_equality_match(req_strings)
+              elsif req_strings.any? { |r| r.start_with?("~=", "==") }
+                tw_req = req_strings.find { |r| r.start_with?("~=", "==") }
+                convert_to_range(tw_req, latest_resolvable_version)
+              else
+                update_requirements_range(req_strings)
+              end
+
+            req.merge(requirement: new_requirement)
+          rescue UnfixableRequirement
+            req.merge(requirement: :unfixable)
+          end
+          # rubocop:enable Metrics/PerceivedComplexity
+
+          def updated_pipfile_requirement(req)
+            # For now, we just proxy to updated_requirement. In future this
+            # method may treat Pipfile requirements differently.
+            updated_requirement(req)
+          end
+
+          # rubocop:disable Metrics/CyclomaticComplexity
+          # rubocop:disable Metrics/PerceivedComplexity
+          def updated_pyproject_requirement(req)
+            return req unless latest_resolvable_version
+            return req unless req.fetch(:requirement)
+            return req if new_version_satisfies?(req) && !has_lockfile
+
+            # If the requirement uses || syntax then we always want to widen it
+            if req.fetch(:requirement).match?(PYPROJECT_OR_SEPARATOR)
+              return widen_pyproject_requirement(req)
+            end
+
+            # If the requirement is a development dependency we always want to
+            # bump it
+            if req.fetch(:groups).include?("dev-dependencies")
+              return update_pyproject_version(req)
+            end
+
+            case update_strategy
+            when :widen_ranges then widen_pyproject_requirement(req)
+            when :bump_versions then update_pyproject_version(req)
+            else raise "Unexpected update strategy: #{update_strategy}"
+            end
+          rescue UnfixableRequirement
+            req.merge(requirement: :unfixable)
+          end
+          # rubocop:enable Metrics/CyclomaticComplexity
+          # rubocop:enable Metrics/PerceivedComplexity
+
+          def update_pyproject_version(req)
+            requirement_strings = req[:requirement].split(",").map(&:strip)
+
+            new_requirement =
+              if requirement_strings.any? { |r| r.match?(/^=|^\d/) }
+                # If there is an equality operator, just update that. It must
+                # be binding and any other requirements will be being ignored
+                find_and_update_equality_match(requirement_strings)
+              elsif requirement_strings.any? { |r| r.start_with?("~", "^") }
+                # If a compatibility operator is being used, just bump its
+                # version (and remove any other requirements)
+                v_req = requirement_strings.find { |r| r.start_with?("~", "^") }
+                bump_version(v_req, latest_resolvable_version.to_s)
+              elsif new_version_satisfies?(req)
+                # Otherwise we're looking at a range operator. No change
+                # required if it's already satisfied
+                req.fetch(:requirement)
+              else
+                # But if it's not, update it
+                update_requirements_range(requirement_strings)
+              end
+
+            req.merge(requirement: new_requirement)
+          end
+
+          def widen_pyproject_requirement(req)
+            return req if new_version_satisfies?(req)
+
+            new_requirement =
+              if req[:requirement].match?(PYPROJECT_OR_SEPARATOR)
+                add_new_requirement_option(req[:requirement])
+              else
+                widen_requirement_range(req[:requirement])
+              end
+
+            req.merge(requirement: new_requirement)
+          end
+
+          def add_new_requirement_option(req_string)
+            option_to_copy = req_string.split(PYPROJECT_OR_SEPARATOR).last.
+                             split(PYPROJECT_SEPARATOR).first.strip
+            operator       = option_to_copy.gsub(/\d.*/, "").strip
+
+            new_option =
+              case operator
+              when "", "==", "==="
+                find_and_update_equality_match([option_to_copy])
+              when "~=", "~", "^"
+                bump_version(option_to_copy, latest_resolvable_version.to_s)
+              else
+                # We don't expect to see OR conditions used with range
+                # operators. If / when we see it, we should handle it.
+                raise "Unexpected operator: #{operator}"
+              end
+
+            # TODO: Match source spacing
+            "#{req_string.strip} || #{new_option.strip}"
+          end
+
+          def widen_requirement_range(req_string)
+            requirement_strings = req_string.split(",").map(&:strip)
+
+            if requirement_strings.any? { |r| r.match?(/(^=|^\d)[^*]*$/) }
+              # If there is an equality operator, just update that.
+              # (i.e., assume it's being used deliberately)
+              find_and_update_equality_match(requirement_strings)
+            elsif requirement_strings.any? { |r| r.start_with?("~", "^") } ||
+                  requirement_strings.any? { |r| r.include?("*") }
+              # If a compatibility operator is being used, widen its
+              # range to include the new version
+              v_req = requirement_strings.
+                      find { |r| r.start_with?("~", "^") || r.include?("*") }
+              convert_to_range(v_req, latest_resolvable_version)
+            else
+              # Otherwise we have a range, and need to update the upper bound
+              update_requirements_range(requirement_strings)
+            end
+          end
+
+          # rubocop:disable Metrics/PerceivedComplexity
+          def updated_requirement(req)
+            return req unless latest_resolvable_version
+            return req unless req.fetch(:requirement)
+
+            requirement_strings = req[:requirement].split(",").map(&:strip)
+
+            new_requirement =
+              if requirement_strings.any? { |r| r.start_with?("=") }
+                find_and_update_equality_match(requirement_strings)
+              elsif requirement_strings.any? { |r| r.start_with?("~=") }
+                tw_req = requirement_strings.find { |r| r.start_with?("~=") }
+                bump_version(tw_req, latest_resolvable_version.to_s)
+              elsif new_version_satisfies?(req)
+                req.fetch(:requirement)
+              else
+                update_requirements_range(requirement_strings)
+              end
+
+            req.merge(requirement: new_requirement)
+          rescue UnfixableRequirement
+            req.merge(requirement: :unfixable)
+          end
+          # rubocop:enable Metrics/PerceivedComplexity
+
+          def new_version_satisfies?(req)
+            requirement_class.
+              requirements_array(req.fetch(:requirement)).
+              any? { |r| r.satisfied_by?(latest_resolvable_version) }
+          end
+
+          def find_and_update_equality_match(requirement_strings)
+            if requirement_strings.any? { |r| requirement_class.new(r).exact? }
+              # True equality match
+              requirement_strings.find { |r| requirement_class.new(r).exact? }.
+                sub(
+                  PythonRequirementParser::VERSION,
+                  latest_resolvable_version.to_s
+                )
+            else
+              # Prefix match
+              requirement_strings.find { |r| r.match?(/^(=+|\d)/) }.
+                sub(PythonRequirementParser::VERSION) do |v|
+                  at_same_precision(latest_resolvable_version.to_s, v)
+                end
+            end
+          end
+
+          def at_same_precision(new_version, old_version)
+            # return new_version unless old_version.include?("*")
+
+            count = old_version.split(".").count
+            precision = old_version.split(".").index("*") || count
+
+            new_version.
+              split(".").
+              first(count).
+              map.with_index { |s, i| i < precision ? s : "*" }.
+              join(".")
+          end
+
+          def update_requirements_range(requirement_strings)
+            ruby_requirements =
+              requirement_strings.map { |r| requirement_class.new(r) }
+
+            updated_requirement_strings = ruby_requirements.flat_map do |r|
+              next r.to_s if r.satisfied_by?(latest_resolvable_version)
+
+              case op = r.requirements.first.first
+              when "<", "<="
+                "<" + update_greatest_version(r.to_s, latest_resolvable_version)
+              when "!="
+                nil
+              when ">", ">="
+                raise UnfixableRequirement
+              else
+                raise "Unexpected op for unsatisfied requirement: #{op}"
+              end
+            end.compact
+
+            updated_requirement_strings.
+              sort_by { |r| requirement_class.new(r).requirements.first.last }.
+              map(&:to_s).join(",").delete(" ")
+          end
+
+          # Updates the version in a constraint to be the given version
+          def bump_version(req_string, version_to_be_permitted)
+            old_version = req_string.
+                          match(/(#{PythonRequirementParser::VERSION})/).
+                          captures.first
+
+            req_string.sub(
+              old_version,
+              at_same_precision(version_to_be_permitted, old_version)
+            )
+          end
+
+          def convert_to_range(req_string, version_to_be_permitted)
+            # Construct an upper bound at the same precision that the original
+            # requirement was at (taking into account ~ dynamics)
+            index_to_update = index_to_update_for(req_string)
+            ub_segments = version_to_be_permitted.segments
+            ub_segments << 0 while ub_segments.count <= index_to_update
+            ub_segments = ub_segments[0..index_to_update]
+            ub_segments[index_to_update] += 1
+
+            lb_segments = lower_bound_segments_for_req(req_string)
+
+            # Ensure versions have the same length as each other (cosmetic)
+            length = [lb_segments.count, ub_segments.count].max
+            lb_segments.fill(0, lb_segments.count...length)
+            ub_segments.fill(0, ub_segments.count...length)
+
+            ">=#{lb_segments.join('.')},<#{ub_segments.join('.')}"
+          end
+
+          def lower_bound_segments_for_req(req_string)
+            requirement = requirement_class.new(req_string)
+            version = requirement.requirements.first.last
+            version = version.release if version.prerelease?
+
+            lb_segments = version.segments
+            lb_segments.pop while lb_segments.last.zero?
+
+            lb_segments
+          end
+
+          def index_to_update_for(req_string)
+            req = requirement_class.new(req_string.split(/[.\-]\*/).first)
+            version = req.requirements.first.last.release
+
+            if req_string.strip.start_with?("^")
+              version.segments.index { |i| i != 0 }
+            elsif req_string.include?("*")
+              version.segments.count - 1
+            elsif req_string.strip.start_with?("~=", "==")
+              version.segments.count - 2
+            elsif req_string.strip.start_with?("~")
+              req_string.split(".").count == 1 ? 0 : 1
+            else raise "Don't know how to convert #{req_string} to range"
+            end
+          end
+
+          # Updates the version in a "<" or "<=" constraint to allow the given
+          # version
+          def update_greatest_version(req_string, version_to_be_permitted)
+            if version_to_be_permitted.is_a?(String)
+              version_to_be_permitted =
+                Utils::Python::Version.new(version_to_be_permitted)
+            end
+            version = Utils::Python::Version.new(req_string.gsub(/<=?/, ""))
+            version = version.release if version.prerelease?
+
+            index_to_update = [
+              version.segments.map.with_index { |n, i| n.zero? ? 0 : i }.max,
+              version_to_be_permitted.segments.count - 1
+            ].min
+
+            new_segments = version.segments.map.with_index do |_, index|
+              if index < index_to_update
+                version_to_be_permitted.segments[index]
+              elsif index == index_to_update
+                version_to_be_permitted.segments[index] + 1
+              else 0
+              end
+            end
+
+            new_segments.join(".")
+          end
+
+          def requirement_class
+            Utils::Python::Requirement
+          end
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength