dependabot-uv 0.299.1

data/lib/dependabot/uv/update_checker/index_finder.rb

@@ -0,0 +1,227 @@
+# typed: true
+# frozen_string_literal: true
+
+require "dependabot/uv/update_checker"
+require "dependabot/uv/authed_url_builder"
+require "dependabot/errors"
+
+module Dependabot
+  module Uv
+    class UpdateChecker
+      class IndexFinder
+        PYPI_BASE_URL = "https://pypi.org/simple/"
+        ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/
+
+        def initialize(dependency_files:, credentials:, dependency:)
+          @dependency_files = dependency_files
+          @credentials      = credentials
+          @dependency       = dependency
+        end
+
+        def index_urls
+          extra_index_urls =
+            config_variable_index_urls[:extra] +
+            pipfile_index_urls[:extra] +
+            requirement_file_index_urls[:extra] +
+            pip_conf_index_urls[:extra] +
+            pyproject_index_urls[:extra]
+
+          extra_index_urls = extra_index_urls.map do |url|
+            clean_check_and_remove_environment_variables(url)
+          end
+
+          # URL encode any `@` characters within registry URL creds.
+          # TODO: The test that fails if the `map` here is removed is likely a
+          # bug in Ruby's URI parser, and should be fixed there.
+          [main_index_url, *extra_index_urls].map do |url|
+            url.rpartition("@").tap { |a| a.first.gsub!("@", "%40") }.join
+          end.uniq
+        end
+
+        private
+
+        attr_reader :dependency_files
+        attr_reader :credentials
+
+        def main_index_url
+          url =
+            config_variable_index_urls[:main] ||
+            pipfile_index_urls[:main] ||
+            requirement_file_index_urls[:main] ||
+            pip_conf_index_urls[:main] ||
+            pyproject_index_urls[:main] ||
+            PYPI_BASE_URL
+
+          clean_check_and_remove_environment_variables(url)
+        end
+
+        def requirement_file_index_urls
+          urls = { main: nil, extra: [] }
+
+          requirements_files.each do |file|
+            if file.content.match?(/^--index-url\s+['"]?([^\s'"]+)['"]?/)
+              urls[:main] =
+                file.content.match(/^--index-url\s+['"]?([^\s'"]+)['"]?/)
+                    .captures.first&.strip
+            end
+            urls[:extra] +=
+              file.content
+                  .scan(/^--extra-index-url\s+['"]?([^\s'"]+)['"]?/)
+                  .flatten
+                  .map(&:strip)
+          end
+
+          urls
+        end
+
+        def pip_conf_index_urls
+          urls = { main: nil, extra: [] }
+
+          return urls unless pip_conf
+
+          content = pip_conf.content
+
+          if content.match?(/^index-url\s*=/x)
+            urls[:main] = content.match(/^index-url\s*=\s*(.+)/)
+                                 .captures.first
+          end
+          urls[:extra] += content.scan(/^extra-index-url\s*=(.+)/).flatten
+
+          urls
+        end
+
+        def pipfile_index_urls
+          urls = { main: nil, extra: [] }
+
+          return urls unless pipfile
+
+          pipfile_object = TomlRB.parse(pipfile.content)
+
+          urls[:main] = pipfile_object["source"]&.first&.fetch("url", nil)
+
+          pipfile_object["source"]&.each do |source|
+            urls[:extra] << source.fetch("url") if source["url"]
+          end
+          urls[:extra] = urls[:extra].uniq
+
+          urls
+        rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
+          urls
+        end
+
+        def pyproject_index_urls
+          urls = { main: nil, extra: [] }
+
+          return urls unless pyproject
+
+          sources =
+            TomlRB.parse(pyproject.content).dig("tool", "poetry", "source") ||
+            []
+
+          sources.each do |source|
+            # If source is PyPI, skip it, and let it pick the default URI
+            next if source["name"].casecmp?("PyPI")
+
+            if @dependency.all_sources.include?(source["name"])
+              # If dependency has specified this source, use it
+              return { main: source["url"], extra: [] }
+            elsif source["default"]
+              urls[:main] = source["url"]
+            elsif source["priority"] != "explicit"
+              # if source is not explicit, add it to extra
+              urls[:extra] << source["url"]
+            end
+          end
+          urls[:extra] = urls[:extra].uniq
+
+          urls
+        rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
+          urls
+        end
+
+        def config_variable_index_urls
+          urls = { main: nil, extra: [] }
+
+          index_url_creds = credentials
+                            .select { |cred| cred["type"] == "python_index" }
+
+          if (main_cred = index_url_creds.find(&:replaces_base?))
+            urls[:main] = AuthedUrlBuilder.authed_url(credential: main_cred)
+          end
+
+          urls[:extra] =
+            index_url_creds
+            .reject(&:replaces_base?)
+            .map { |cred| AuthedUrlBuilder.authed_url(credential: cred) }
+
+          urls
+        end
+
+        def clean_check_and_remove_environment_variables(url)
+          url = url.strip.gsub(%r{/*$}, "") + "/"
+
+          return authed_base_url(url) unless url.match?(ENVIRONMENT_VARIABLE_REGEX)
+
+          config_variable_urls =
+            [
+              config_variable_index_urls[:main],
+              *config_variable_index_urls[:extra]
+            ]
+            .compact
+            .map { |u| u.strip.gsub(%r{/*$}, "") + "/" }
+
+          regexp = url
+                   .sub(%r{(?<=://).+@}, "")
+                   .sub(%r{https?://}, "")
+                   .split(ENVIRONMENT_VARIABLE_REGEX)
+                   .map { |part| Regexp.quote(part) }
+                   .join(".+")
+          authed_url = config_variable_urls.find { |u| u.match?(regexp) }
+          return authed_url if authed_url
+
+          cleaned_url = url.gsub(%r{#{ENVIRONMENT_VARIABLE_REGEX}/?}o, "")
+          authed_url = authed_base_url(cleaned_url)
+          return authed_url if credential_for(cleaned_url)
+
+          raise PrivateSourceAuthenticationFailure, url
+        end
+
+        def authed_base_url(base_url)
+          cred = credential_for(base_url)
+          return base_url unless cred
+
+          AuthedUrlBuilder.authed_url(credential: cred).gsub(%r{/*$}, "") + "/"
+        end
+
+        def credential_for(url)
+          credentials
+            .select { |c| c["type"] == "python_index" }
+            .find do |c|
+              cred_url = c.fetch("index-url").gsub(%r{/*$}, "") + "/"
+              cred_url.include?(url)
+            end
+        end
+
+        def pip_conf
+          dependency_files.find { |f| f.name == "pip.conf" }
+        end
+
+        def pipfile
+          dependency_files.find { |f| f.name == "Pipfile" }
+        end
+
+        def pyproject
+          dependency_files.find { |f| f.name == "pyproject.toml" }
+        end
+
+        def requirements_files
+          dependency_files.select { |f| f.name.match?(/requirements/x) }
+        end
+
+        def pip_compile_files
+          dependency_files.select { |f| f.name.end_with?(".in") }
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/uv/update_checker/latest_version_finder.rb

@@ -0,0 +1,297 @@
+# typed: true
+# frozen_string_literal: true
+
+require "cgi"
+require "excon"
+require "nokogiri"
+require "sorbet-runtime"
+
+require "dependabot/dependency"
+require "dependabot/uv/update_checker"
+require "dependabot/update_checkers/version_filters"
+require "dependabot/registry_client"
+require "dependabot/uv/authed_url_builder"
+require "dependabot/uv/name_normaliser"
+
+module Dependabot
+  module Uv
+    class UpdateChecker
+      class LatestVersionFinder
+        extend T::Sig
+
+        require_relative "index_finder"
+
+        def initialize(dependency:, dependency_files:, credentials:,
+                       ignored_versions:, raise_on_ignored: false,
+                       security_advisories:)
+          @dependency          = dependency
+          @dependency_files    = dependency_files
+          @credentials         = credentials
+          @ignored_versions    = ignored_versions
+          @raise_on_ignored    = raise_on_ignored
+          @security_advisories = security_advisories
+        end
+
+        def latest_version(python_version: nil)
+          @latest_version ||=
+            fetch_latest_version(python_version: python_version)
+        end
+
+        def latest_version_with_no_unlock(python_version: nil)
+          @latest_version_with_no_unlock ||=
+            fetch_latest_version_with_no_unlock(python_version: python_version)
+        end
+
+        def lowest_security_fix_version(python_version: nil)
+          @lowest_security_fix_version ||=
+            fetch_lowest_security_fix_version(python_version: python_version)
+        end
+
+        private
+
+        attr_reader :dependency
+        attr_reader :dependency_files
+        attr_reader :credentials
+        attr_reader :ignored_versions
+        attr_reader :security_advisories
+
+        def fetch_latest_version(python_version:)
+          versions = available_versions
+          versions = filter_yanked_versions(versions)
+          versions = filter_unsupported_versions(versions, python_version)
+          versions = filter_prerelease_versions(versions)
+          versions = filter_ignored_versions(versions)
+          versions.max
+        end
+
+        def fetch_latest_version_with_no_unlock(python_version:)
+          versions = available_versions
+          versions = filter_yanked_versions(versions)
+          versions = filter_unsupported_versions(versions, python_version)
+          versions = filter_prerelease_versions(versions)
+          versions = filter_ignored_versions(versions)
+          versions = filter_out_of_range_versions(versions)
+          versions.max
+        end
+
+        def fetch_lowest_security_fix_version(python_version:)
+          versions = available_versions
+          versions = filter_yanked_versions(versions)
+          versions = filter_unsupported_versions(versions, python_version)
+          versions = filter_prerelease_versions(versions)
+          versions = Dependabot::UpdateCheckers::VersionFilters.filter_vulnerable_versions(versions,
+                                                                                           security_advisories)
+          versions = filter_ignored_versions(versions)
+          versions = filter_lower_versions(versions)
+
+          versions.min
+        end
+
+        sig { params(versions_array: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
+        def filter_yanked_versions(versions_array)
+          filtered = versions_array.reject { |details| details.fetch(:yanked) }
+          if versions_array.count > filtered.count
+            Dependabot.logger.info("Filtered out #{versions_array.count - filtered.count} yanked versions")
+          end
+          filtered
+        end
+
+        sig do
+          params(versions_array: T::Array[T.untyped], python_version: T.nilable(T.any(String, Version)))
+            .returns(T::Array[T.untyped])
+        end
+        def filter_unsupported_versions(versions_array, python_version)
+          filtered = versions_array.filter_map do |details|
+            python_requirement = details.fetch(:python_requirement)
+            next details.fetch(:version) unless python_version
+            next details.fetch(:version) unless python_requirement
+            next unless python_requirement.satisfied_by?(python_version)
+
+            details.fetch(:version)
+          end
+          if versions_array.count > filtered.count
+            delta = versions_array.count - filtered.count
+            Dependabot.logger.info("Filtered out #{delta} unsupported Python #{python_version} versions")
+          end
+          filtered
+        end
+
+        sig { params(versions_array: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
+        def filter_prerelease_versions(versions_array)
+          return versions_array if wants_prerelease?
+
+          filtered = versions_array.reject(&:prerelease?)
+
+          if versions_array.count > filtered.count
+            Dependabot.logger.info("Filtered out #{versions_array.count - filtered.count} pre-release versions")
+          end
+
+          filtered
+        end
+
+        sig { params(versions_array: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
+        def filter_ignored_versions(versions_array)
+          filtered = versions_array
+                     .reject { |v| ignore_requirements.any? { |r| r.satisfied_by?(v) } }
+          if @raise_on_ignored && filter_lower_versions(filtered).empty? && filter_lower_versions(versions_array).any?
+            raise Dependabot::AllVersionsIgnored
+          end
+
+          if versions_array.count > filtered.count
+            Dependabot.logger.info("Filtered out #{versions_array.count - filtered.count} ignored versions")
+          end
+          filtered
+        end
+
+        def filter_lower_versions(versions_array)
+          return versions_array unless dependency.numeric_version
+
+          versions_array.select { |version| version > dependency.numeric_version }
+        end
+
+        def filter_out_of_range_versions(versions_array)
+          reqs = dependency.requirements.filter_map do |r|
+            requirement_class.requirements_array(r.fetch(:requirement))
+          end
+
+          versions_array
+            .select { |v| reqs.all? { |r| r.any? { |o| o.satisfied_by?(v) } } }
+        end
+
+        def wants_prerelease?
+          return version_class.new(dependency.version).prerelease? if dependency.version
+
+          dependency.requirements.any? do |req|
+            reqs = (req.fetch(:requirement) || "").split(",").map(&:strip)
+            reqs.any? { |r| r.match?(/[A-Za-z]/) }
+          end
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/ for details of the
+        # Simple Repository API we use here.
+        def available_versions
+          @available_versions ||=
+            index_urls.flat_map do |index_url|
+              validate_index(index_url)
+
+              sanitized_url = index_url.gsub(%r{(?<=//).*(?=@)}, "redacted")
+
+              index_response = registry_response_for_dependency(index_url)
+              if index_response.status == 401 || index_response.status == 403
+                registry_index_response = registry_index_response(index_url)
+
+                if registry_index_response.status == 401 || registry_index_response.status == 403
+                  raise PrivateSourceAuthenticationFailure, sanitized_url
+                end
+              end
+
+              version_links = []
+              index_response.body.scan(%r{<a\s.*?>.*?</a>}m) do
+                details = version_details_from_link(Regexp.last_match.to_s)
+                version_links << details if details
+              end
+
+              version_links.compact
+            rescue Excon::Error::Timeout, Excon::Error::Socket
+              raise if MAIN_PYPI_INDEXES.include?(index_url)
+
+              raise PrivateSourceTimedOut, sanitized_url
+            rescue URI::InvalidURIError
+              raise DependencyFileNotResolvable, "Invalid URL: #{sanitized_url}"
+            end
+        end
+
+        # rubocop:disable Metrics/PerceivedComplexity
+        def version_details_from_link(link)
+          doc = Nokogiri::XML(link)
+          filename = doc.at_css("a")&.content
+          url = doc.at_css("a")&.attributes&.fetch("href", nil)&.value
+          return unless filename&.match?(name_regex) || url&.match?(name_regex)
+
+          version = get_version_from_filename(filename)
+          return unless version_class.correct?(version)
+
+          {
+            version: version_class.new(version),
+            python_requirement: build_python_requirement_from_link(link),
+            yanked: link&.include?("data-yanked")
+          }
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def get_version_from_filename(filename)
+          filename
+            .gsub(/#{name_regex}-/i, "")
+            .split(/-|\.tar\.|\.zip|\.whl/)
+            .first
+        end
+
+        def build_python_requirement_from_link(link)
+          req_string = Nokogiri::XML(link)
+                               .at_css("a")
+                               &.attribute("data-requires-python")
+                               &.content
+
+          return unless req_string
+
+          requirement_class.new(CGI.unescapeHTML(req_string))
+        rescue Gem::Requirement::BadRequirementError
+          nil
+        end
+
+        def index_urls
+          @index_urls ||=
+            IndexFinder.new(
+              dependency_files: dependency_files,
+              credentials: credentials,
+              dependency: dependency
+            ).index_urls
+        end
+
+        def registry_response_for_dependency(index_url)
+          Dependabot::RegistryClient.get(
+            url: index_url + normalised_name + "/",
+            headers: { "Accept" => "text/html" }
+          )
+        end
+
+        def registry_index_response(index_url)
+          Dependabot::RegistryClient.get(
+            url: index_url,
+            headers: { "Accept" => "text/html" }
+          )
+        end
+
+        def ignore_requirements
+          ignored_versions.flat_map { |req| requirement_class.requirements_array(req) }
+        end
+
+        def normalised_name
+          NameNormaliser.normalise(dependency.name)
+        end
+
+        def name_regex
+          parts = normalised_name.split(/[\s_.-]/).map { |n| Regexp.quote(n) }
+          /#{parts.join("[\s_.-]")}/i
+        end
+
+        def version_class
+          dependency.version_class
+        end
+
+        def requirement_class
+          dependency.requirement_class
+        end
+
+        def validate_index(index_url)
+          sanitized_url = index_url.gsub(%r{(?<=//).*(?=@)}, "redacted")
+
+          return if index_url&.match?(URI::DEFAULT_PARSER.regexp[:ABS_URI])
+
+          raise Dependabot::DependencyFileNotResolvable,
+                "Invalid URL: #{sanitized_url}"
+        end
+      end
+    end
+  end
+end