dependabot-python 0.298.0 → 0.299.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d7cbbb68900510599e8ae5879bc9dc36984b212c1e0f1bf23750a374da0665a
-  data.tar.gz: 6afc9a2f6f5d4696ff8ef47669fa104fc1e580ad0719ee561ac59821715a579b
+  metadata.gz: d429e2e29dd9b40c0213abc4255d683c03153ebd78508ef1d01c8a67c0779606
+  data.tar.gz: 87e287bf9f902c64ab1d1c99c33262534b997e14b048df39215b6d9e30226e36
 SHA512:
-  metadata.gz: f9848c2b7376c9fa036812f63f2f44ee598e0f8d7746cb931c7f91c2a7e387616e207286249b896473e82df893c57ce34fd511983f7c8b5b1f9ff7ab1a9efeba
-  data.tar.gz: c5e08128f5f9eec6192e17c40cc8dc9f59e4c1ee5d3f8e7dbabb5cbba575500983a572d5eb3876833aa85b69848ae34d2ddfc1302aa86d89de2c5cd516aaab41
+  metadata.gz: 9bdeb41bf8491bc81f7970ad48a57d0e7c792e54bb010d62adec1247da87b654db158e834351632aff89968a28e1bbfacb6f6981174d73c488dc474dd95070d0
+  data.tar.gz: 31e151c0043b9ad2f634b2833ac624676ffad90071b438833036fcd4eb426307c2622645eb9735b77b3b220a1974caf49f67f97908ce1223e2f429c54c0045ae

data/lib/dependabot/python/package/package_details_fetcher.rb

@@ -0,0 +1,485 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require "cgi"
+require "excon"
+require "nokogiri"
+require "sorbet-runtime"
+require "dependabot/registry_client"
+require "dependabot/python/name_normaliser"
+require "dependabot/package/package_release"
+require "dependabot/package/package_details"
+require "dependabot/python/package/package_registry_finder"
+
+# Stores metadata for a package, including all its available versions
+module Dependabot
+  module Python
+    module Package
+      CREDENTIALS_USERNAME = "username"
+      CREDENTIALS_PASSWORD = "password"
+
+      APPLICATION_JSON = "application/json"
+      APPLICATION_TEXT = "text/html"
+      CPYTHON = "cpython"
+      PYTHON = "python"
+      UNKNOWN = "unknown"
+
+      MAIN_PYPI_INDEXES = %w(
+        https://pypi.python.org/simple/
+        https://pypi.org/simple/
+      ).freeze
+      VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/
+
+      class PackageDetailsFetcher
+        extend T::Sig
+
+        sig do
+          params(
+            dependency: Dependabot::Dependency,
+            dependency_files: T::Array[Dependabot::DependencyFile],
+            credentials: T::Array[Dependabot::Credential]
+          ).void
+        end
+        def initialize(
+          dependency:,
+          dependency_files:,
+          credentials:
+        )
+          @dependency          = dependency
+          @dependency_files    = dependency_files
+          @credentials         = credentials
+
+          @registry_urls = T.let(nil, T.nilable(T::Array[String]))
+        end
+
+        sig { returns(Dependabot::Dependency) }
+        attr_reader :dependency
+
+        sig { returns(T::Array[T.untyped]) }
+        attr_reader :dependency_files
+
+        sig { returns(T::Array[T.untyped]) }
+        attr_reader :credentials
+
+        sig { returns(Dependabot::Package::PackageDetails) }
+        def fetch
+          package_releases = registry_urls
+                             .select { |index_url| validate_index(index_url) } # Ensure only valid URLs
+                             .flat_map do |index_url|
+            fetch_from_registry(index_url) || [] # Ensure it always returns an array
+            rescue Excon::Error::Timeout, Excon::Error::Socket
+              raise if MAIN_PYPI_INDEXES.include?(index_url)
+
+              raise PrivateSourceTimedOut, sanitized_url(index_url)
+            rescue URI::InvalidURIError
+              raise DependencyFileNotResolvable, "Invalid URL: #{sanitized_url(index_url)}"
+          end
+
+          Dependabot::Package::PackageDetails.new(
+            dependency: dependency,
+            releases: package_releases.reverse.uniq(&:version)
+          )
+        end
+
+        private
+
+        sig do
+          params(index_url: String)
+            .returns(T.nilable(T::Array[Dependabot::Package::PackageRelease]))
+        end
+        def fetch_from_registry(index_url)
+          if Dependabot::Experiments.enabled?(:enable_cooldown_for_python)
+            metadata = fetch_from_json_registry(index_url)
+
+            return metadata if metadata&.any?
+
+            Dependabot.logger.warn("No valid versions found via JSON API. Falling back to HTML.")
+          end
+          fetch_from_html_registry(index_url)
+        rescue StandardError => e
+          Dependabot.logger.warn("Unexpected error in JSON fetch: #{e.message}. Falling back to HTML.")
+          fetch_from_html_registry(index_url)
+        end
+
+        # Example JSON Response Format:
+        #
+        # {
+        #   "info": {
+        #     "name": "requests",
+        #     "summary": "Python HTTP for Humans.",
+        #     "author": "Kenneth Reitz",
+        #     "license": "Apache-2.0"
+        #   },
+        #   "releases": {
+        #     "2.32.3": [
+        #       {
+        #         "filename": "requests-2.32.3-py3-none-any.whl",
+        #         "version": "2.32.3",
+        #         "requires_python": ">=3.8",
+        #         "yanked": false,
+        #         "url": "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl"
+        #       },
+        #       {
+        #         "filename": "requests-2.32.3.tar.gz",
+        #         "version": "2.32.3",
+        #         "requires_python": ">=3.8",
+        #         "yanked": false,
+        #         "url": "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz"
+        #       }
+        #     ],
+        #     "2.27.0": [
+        #       {
+        #         "filename": "requests-2.27.0-py2.py3-none-any.whl",
+        #         "version": "2.27.0",
+        #         "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*",
+        #         "yanked": false,
+        #         "url": "https://files.pythonhosted.org/packages/47/01/f420e7add78110940639a958e5af0e3f8e07a8a8b62049bac55ee117aa91/requests-2.27.0-py2.py3-none-any.whl"
+        #       },
+        #       {
+        #         "filename": "requests-2.27.0.tar.gz",
+        #         "version": "2.27.0",
+        #         "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*",
+        #         "yanked": false,
+        #         "url": "https://files.pythonhosted.org/packages/c0/e3/826e27b942352a74b656e8f58b4dc7ed9495ce2d4eeb498181167c615303/requests-2.27.0.tar.gz"
+        #       }
+        #     ]
+        #   }
+        # }
+        sig do
+          params(index_url: String)
+            .returns(T.nilable(T::Array[Dependabot::Package::PackageRelease]))
+        end
+        def fetch_from_json_registry(index_url)
+          json_url = index_url.sub(%r{/simple/?$}i, "/pypi/")
+
+          Dependabot.logger.info(
+            "Fetching release information from json registry at #{sanitized_url(json_url)} for #{dependency.name}"
+          )
+
+          response = registry_json_response_for_dependency(json_url)
+
+          return nil unless response.status == 200
+
+          begin
+            data = JSON.parse(response.body)
+
+            version_releases = data["releases"]
+
+            releases = format_version_releases(version_releases)
+
+            releases.sort_by(&:version).reverse
+          rescue JSON::ParserError
+            Dependabot.logger.warn("JSON parsing error for #{json_url}. Falling back to HTML.")
+            nil
+          rescue StandardError => e
+            Dependabot.logger.warn("Unexpected error while fetching JSON data: #{e.message}.")
+            nil
+          end
+        end
+
+        # This URL points to the Simple Index API for the "requests" package on PyPI.
+        # It provides an HTML listing of available package versions following PEP 503 (Simple Repository API).
+        # The information found here is useful for dependency resolution and package version retrieval.
+        #
+        # ✅ Information available in the Simple Index:
+        # - A list of package versions as anchor (`<a>`) elements.
+        # - URLs to distribution files (e.g., `.tar.gz`, `.whl`).
+        # - The `data-requires-python` attribute (if present) specifying the required Python version.
+        # - An optional `data-yanked` attribute indicating a yanked (withdrawn) version.
+        #
+        # ❌ Information NOT available in the Simple Index:
+        # - Release timestamps (upload time).
+        # - File digests (hashes like SHA256, MD5).
+        # - Package metadata such as description, author, or dependencies.
+        # - Download statistics.
+        # - Package type (`sdist` or `bdist_wheel`).
+        #
+        # To obtain full package metadata, use the PyPI JSON API:
+        # - JSON API: https://pypi.org/pypi/requests/json
+        #
+        # More details: https://www.python.org/dev/peps/pep-0503/
+        sig { params(index_url: String).returns(T::Array[Dependabot::Package::PackageRelease]) }
+        def fetch_from_html_registry(index_url)
+          Dependabot.logger.info(
+            "Fetching release information from html registry at #{sanitized_url(index_url)} for #{dependency.name}"
+          )
+          index_response = registry_response_for_dependency(index_url)
+          if index_response.status == 401 || index_response.status == 403
+            registry_index_response = registry_index_response(index_url)
+
+            if registry_index_response.status == 401 || registry_index_response.status == 403
+              raise PrivateSourceAuthenticationFailure, sanitized_url(index_url)
+            end
+          end
+
+          version_releases = extract_release_details_json_from_html(index_response.body)
+          releases = format_version_releases(version_releases)
+
+          releases.sort_by(&:version).reverse
+        end
+
+        sig do
+          params(html_body: String)
+            .returns(T::Hash[String, T::Array[T::Hash[String, T.untyped]]]) # Returns JSON-like format
+        end
+        def extract_release_details_json_from_html(html_body)
+          doc = Nokogiri::HTML(html_body)
+
+          releases = {}
+
+          doc.css("a").each do |a_tag|
+            details = version_details_from_link(a_tag.to_s)
+            if details && details["version"]
+              releases[details["version"]] ||= []
+              releases[details["version"]] << details
+            end
+          end
+
+          releases
+        end
+
+        # rubocop:disable Metrics/PerceivedComplexity
+        sig do
+          params(link: T.nilable(String))
+            .returns(T.nilable(T::Hash[String, T.untyped]))
+        end
+        def version_details_from_link(link)
+          return unless link
+
+          doc = Nokogiri::XML(link)
+          filename = doc.at_css("a")&.content
+          url = doc.at_css("a")&.attributes&.fetch("href", nil)&.value
+
+          return unless filename&.match?(name_regex) || url&.match?(name_regex)
+
+          version = get_version_from_filename(filename)
+          return unless version_class.correct?(version)
+
+          {
+            "version" => version,
+            "requires_python" => requires_python_from_link(link),
+            "yanked" => link.include?("data-yanked"),
+            "url" => link
+          }
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        sig do
+          params(
+            releases_json: T::Hash[String, T::Array[T::Hash[String, T.untyped]]]
+          )
+            .returns(T::Array[Dependabot::Package::PackageRelease])
+        end
+        def format_version_releases(releases_json)
+          releases_json.each_with_object([]) do |(version, release_data_array), versions|
+            release_data = release_data_array.last
+
+            next unless release_data
+
+            release = format_version_release(version, release_data)
+
+            next unless release
+
+            versions << release
+          end
+        end
+
+        sig do
+          params(
+            version: String,
+            release_data: T::Hash[String, T.untyped]
+          )
+            .returns(T.nilable(Dependabot::Package::PackageRelease))
+        end
+        def format_version_release(version, release_data)
+          upload_time = release_data["upload_time"]
+          released_at = Time.parse(upload_time) if upload_time
+          yanked = release_data["yanked"] || false
+          yanked_reason = release_data["yanked_reason"]
+          downloads = release_data["downloads"] || -1
+          url = release_data["url"]
+          package_type = release_data["packagetype"]
+          language = package_language(
+            python_version: release_data["python_version"],
+            requires_python: release_data["requires_python"]
+          )
+
+          release = Dependabot::Package::PackageRelease.new(
+            version: Dependabot::Python::Version.new(version),
+            released_at: released_at,
+            yanked: yanked,
+            yanked_reason: yanked_reason,
+            downloads: downloads,
+            url: url,
+            package_type: package_type,
+            language: language
+          )
+          release
+        end
+
+        sig do
+          params(
+            python_version: T.nilable(String),
+            requires_python: T.nilable(String)
+          )
+            .returns(T.nilable(Dependabot::Package::PackageLanguage))
+        end
+        def package_language(python_version:, requires_python:)
+          # Extract language name and version
+          language_name, language_version = convert_language_version(python_version)
+
+          # Extract language requirement
+          language_requirement = build_python_requirement(requires_python)
+
+          return nil unless language_version || language_requirement
+
+          # Return a Language object with all details
+          Dependabot::Package::PackageLanguage.new(
+            name: language_name,
+            version: language_version,
+            requirement: language_requirement
+          )
+        end
+
+        sig { params(version: T.nilable(String)).returns([String, T.nilable(Dependabot::Version)]) }
+        def convert_language_version(version)
+          return ["python", nil] if version.nil? || version == "source"
+
+          # Extract numeric parts dynamically (e.g., "cp37" -> "3.7", "py38" -> "3.8")
+          extracted_version = version.scan(/\d+/).join(".")
+
+          # Detect the language implementation
+          language_name = if version.start_with?("cp")
+                            "cpython" # CPython implementation
+                          elsif version.start_with?("py")
+                            "python" # General Python compatibility
+                          else
+                            "unknown" # Fallback for unknown cases
+                          end
+
+          # Ensure extracted version is valid before converting
+          language_version =
+            extracted_version.match?(/^\d+(\.\d+)*$/) ? Dependabot::Version.new(extracted_version) : nil
+
+          Dependabot.logger.warn("Skipping invalid language_version: #{version.inspect}") if language_version.nil?
+
+          [language_name, language_version]
+        end
+
+        sig { returns(T::Array[String]) }
+        def registry_urls
+          @registry_urls ||=
+            Package::PackageRegistryFinder.new(
+              dependency_files: dependency_files,
+              credentials: credentials,
+              dependency: dependency
+            ).registry_urls
+        end
+
+        sig { returns(String) }
+        def normalised_name
+          NameNormaliser.normalise(dependency.name)
+        end
+
+        sig { params(json_url: String).returns(Excon::Response) }
+        def registry_json_response_for_dependency(json_url)
+          Dependabot::RegistryClient.get(
+            url: "#{json_url.chomp('/')}/#{@dependency.name}/json",
+            headers: { "Accept" => APPLICATION_JSON }
+          )
+        end
+
+        sig { params(index_url: String).returns(Excon::Response) }
+        def registry_response_for_dependency(index_url)
+          Dependabot::RegistryClient.get(
+            url: index_url + normalised_name + "/",
+            headers: { "Accept" => APPLICATION_TEXT }
+          )
+        end
+
+        sig { params(index_url: String).returns(Excon::Response) }
+        def registry_index_response(index_url)
+          Dependabot::RegistryClient.get(
+            url: index_url,
+            headers: { "Accept" => APPLICATION_TEXT }
+          )
+        end
+
+        sig { params(filename: String).returns(T.nilable(String)) }
+        def get_version_from_filename(filename)
+          filename
+            .gsub(/#{name_regex}-/i, "")
+            .split(/-|\.tar\.|\.zip|\.whl/)
+            .first
+        end
+
+        sig do
+          params(req_string: T.nilable(String))
+            .returns(T.nilable(Dependabot::Requirement))
+        end
+        def build_python_requirement(req_string)
+          return nil unless req_string
+
+          requirement_class.new(CGI.unescapeHTML(req_string))
+        rescue Gem::Requirement::BadRequirementError
+          nil
+        end
+
+        sig { params(link: String).returns(T.nilable(String)) }
+        def requires_python_from_link(link)
+          raw_value = Nokogiri::XML(link)
+                              .at_css("a")
+                              &.attribute("data-requires-python")
+                              &.content
+
+          return nil unless raw_value
+
+          CGI.unescapeHTML(raw_value) # Decodes HTML entities like &gt;=3 → >=3
+        end
+
+        sig { returns(T.class_of(Dependabot::Version)) }
+        def version_class
+          dependency.version_class
+        end
+
+        sig { returns(T.class_of(Dependabot::Requirement)) }
+        def requirement_class
+          dependency.requirement_class
+        end
+
+        sig { params(index_url: String).returns(T::Hash[String, String]) }
+        def auth_headers_for(index_url)
+          credential = @credentials.find { |cred| cred["index-url"] == index_url }
+          return {} unless credential
+
+          { "Authorization" => "Basic #{Base64.strict_encode64(
+            "#{credential[CREDENTIALS_USERNAME]}:#{credential[CREDENTIALS_PASSWORD]}"
+          )}" }
+        end
+
+        sig { returns(Regexp) }
+        def name_regex
+          parts = normalised_name.split(/[\s_.-]/).map { |n| Regexp.quote(n) }
+          /#{parts.join("[\s_.-]")}/i
+        end
+
+        sig { params(index_url: T.nilable(String)).returns(T::Boolean) }
+        def validate_index(index_url)
+          return false unless index_url
+
+          return true if index_url.match?(URI::DEFAULT_PARSER.regexp[:ABS_URI])
+
+          raise Dependabot::DependencyFileNotResolvable,
+                "Invalid URL: #{sanitized_url(index_url)}"
+        end
+
+        sig { params(index_url: String).returns(String) }
+        def sanitized_url(index_url)
+          index_url.sub(%r{//([^/@]+)@}, "//redacted@")
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/{update_checker/index_finder.rb → package/package_registry_finder.rb}

@@ -7,8 +7,8 @@ require "dependabot/errors"
 
 module Dependabot
   module Python
-    class UpdateChecker
-      class IndexFinder
+    module Package
+      class PackageRegistryFinder
         PYPI_BASE_URL = "https://pypi.org/simple/"
         ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/
 
@@ -18,7 +18,7 @@ module Dependabot
           @dependency       = dependency
         end
 
-        def index_urls
+        def registry_urls
           extra_index_urls =
             config_variable_index_urls[:extra] +
             pipfile_index_urls[:extra] +
@@ -158,7 +158,7 @@ module Dependabot
         end
 
         def clean_check_and_remove_environment_variables(url)
-          url = url.strip.gsub(%r{/*$}, "") + "/"
+          url = url.strip.sub(%r{/+$}, "") + "/"
 
           return authed_base_url(url) unless url.match?(ENVIRONMENT_VARIABLE_REGEX)
 

data/lib/dependabot/python/update_checker/latest_version_finder.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: strong
 # frozen_string_literal: true
 
 require "cgi"
@@ -12,284 +12,25 @@ require "dependabot/update_checkers/version_filters"
 require "dependabot/registry_client"
 require "dependabot/python/authed_url_builder"
 require "dependabot/python/name_normaliser"
+require "dependabot/python/package/package_registry_finder"
+require "dependabot/python/package/package_details_fetcher"
+require "dependabot/package/package_latest_version_finder"
 
 module Dependabot
   module Python
     class UpdateChecker
-      class LatestVersionFinder
+      class LatestVersionFinder < Dependabot::Package::PackageLatestVersionFinder
         extend T::Sig
 
-        require_relative "index_finder"
-
-        def initialize(dependency:, dependency_files:, credentials:,
-                       ignored_versions:, raise_on_ignored: false,
-                       security_advisories:)
-          @dependency          = dependency
-          @dependency_files    = dependency_files
-          @credentials         = credentials
-          @ignored_versions    = ignored_versions
-          @raise_on_ignored    = raise_on_ignored
-          @security_advisories = security_advisories
-        end
-
-        def latest_version(python_version: nil)
-          @latest_version ||=
-            fetch_latest_version(python_version: python_version)
-        end
-
-        def latest_version_with_no_unlock(python_version: nil)
-          @latest_version_with_no_unlock ||=
-            fetch_latest_version_with_no_unlock(python_version: python_version)
-        end
-
-        def lowest_security_fix_version(python_version: nil)
-          @lowest_security_fix_version ||=
-            fetch_lowest_security_fix_version(python_version: python_version)
-        end
-
-        private
-
-        attr_reader :dependency
-        attr_reader :dependency_files
-        attr_reader :credentials
-        attr_reader :ignored_versions
-        attr_reader :security_advisories
-
-        def fetch_latest_version(python_version:)
-          versions = available_versions
-          versions = filter_yanked_versions(versions)
-          versions = filter_unsupported_versions(versions, python_version)
-          versions = filter_prerelease_versions(versions)
-          versions = filter_ignored_versions(versions)
-          versions.max
-        end
-
-        def fetch_latest_version_with_no_unlock(python_version:)
-          versions = available_versions
-          versions = filter_yanked_versions(versions)
-          versions = filter_unsupported_versions(versions, python_version)
-          versions = filter_prerelease_versions(versions)
-          versions = filter_ignored_versions(versions)
-          versions = filter_out_of_range_versions(versions)
-          versions.max
-        end
-
-        def fetch_lowest_security_fix_version(python_version:)
-          versions = available_versions
-          versions = filter_yanked_versions(versions)
-          versions = filter_unsupported_versions(versions, python_version)
-          versions = filter_prerelease_versions(versions)
-          versions = Dependabot::UpdateCheckers::VersionFilters.filter_vulnerable_versions(versions,
-                                                                                           security_advisories)
-          versions = filter_ignored_versions(versions)
-          versions = filter_lower_versions(versions)
-
-          versions.min
-        end
-
-        sig { params(versions_array: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
-        def filter_yanked_versions(versions_array)
-          filtered = versions_array.reject { |details| details.fetch(:yanked) }
-          if versions_array.count > filtered.count
-            Dependabot.logger.info("Filtered out #{versions_array.count - filtered.count} yanked versions")
-          end
-          filtered
-        end
-
         sig do
-          params(versions_array: T::Array[T.untyped], python_version: T.nilable(T.any(String, Version)))
-            .returns(T::Array[T.untyped])
-        end
-        def filter_unsupported_versions(versions_array, python_version)
-          filtered = versions_array.filter_map do |details|
-            python_requirement = details.fetch(:python_requirement)
-            next details.fetch(:version) unless python_version
-            next details.fetch(:version) unless python_requirement
-            next unless python_requirement.satisfied_by?(python_version)
-
-            details.fetch(:version)
-          end
-          if versions_array.count > filtered.count
-            delta = versions_array.count - filtered.count
-            Dependabot.logger.info("Filtered out #{delta} unsupported Python #{python_version} versions")
-          end
-          filtered
-        end
-
-        sig { params(versions_array: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
-        def filter_prerelease_versions(versions_array)
-          return versions_array if wants_prerelease?
-
-          filtered = versions_array.reject(&:prerelease?)
-
-          if versions_array.count > filtered.count
-            Dependabot.logger.info("Filtered out #{versions_array.count - filtered.count} pre-release versions")
-          end
-
-          filtered
-        end
-
-        sig { params(versions_array: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
-        def filter_ignored_versions(versions_array)
-          filtered = versions_array
-                     .reject { |v| ignore_requirements.any? { |r| r.satisfied_by?(v) } }
-          if @raise_on_ignored && filter_lower_versions(filtered).empty? && filter_lower_versions(versions_array).any?
-            raise Dependabot::AllVersionsIgnored
-          end
-
-          if versions_array.count > filtered.count
-            Dependabot.logger.info("Filtered out #{versions_array.count - filtered.count} ignored versions")
-          end
-          filtered
-        end
-
-        def filter_lower_versions(versions_array)
-          return versions_array unless dependency.numeric_version
-
-          versions_array.select { |version| version > dependency.numeric_version }
-        end
-
-        def filter_out_of_range_versions(versions_array)
-          reqs = dependency.requirements.filter_map do |r|
-            requirement_class.requirements_array(r.fetch(:requirement))
-          end
-
-          versions_array
-            .select { |v| reqs.all? { |r| r.any? { |o| o.satisfied_by?(v) } } }
-        end
-
-        def wants_prerelease?
-          return version_class.new(dependency.version).prerelease? if dependency.version
-
-          dependency.requirements.any? do |req|
-            reqs = (req.fetch(:requirement) || "").split(",").map(&:strip)
-            reqs.any? { |r| r.match?(/[A-Za-z]/) }
-          end
-        end
-
-        # See https://www.python.org/dev/peps/pep-0503/ for details of the
-        # Simple Repository API we use here.
-        def available_versions
-          @available_versions ||=
-            index_urls.flat_map do |index_url|
-              validate_index(index_url)
-
-              sanitized_url = index_url.gsub(%r{(?<=//).*(?=@)}, "redacted")
-
-              index_response = registry_response_for_dependency(index_url)
-              if index_response.status == 401 || index_response.status == 403
-                registry_index_response = registry_index_response(index_url)
-
-                if registry_index_response.status == 401 || registry_index_response.status == 403
-                  raise PrivateSourceAuthenticationFailure, sanitized_url
-                end
-              end
-
-              version_links = []
-              index_response.body.scan(%r{<a\s.*?>.*?</a>}m) do
-                details = version_details_from_link(Regexp.last_match.to_s)
-                version_links << details if details
-              end
-
-              version_links.compact
-            rescue Excon::Error::Timeout, Excon::Error::Socket
-              raise if MAIN_PYPI_INDEXES.include?(index_url)
-
-              raise PrivateSourceTimedOut, sanitized_url
-            rescue URI::InvalidURIError
-              raise DependencyFileNotResolvable, "Invalid URL: #{sanitized_url}"
-            end
-        end
-
-        # rubocop:disable Metrics/PerceivedComplexity
-        def version_details_from_link(link)
-          doc = Nokogiri::XML(link)
-          filename = doc.at_css("a")&.content
-          url = doc.at_css("a")&.attributes&.fetch("href", nil)&.value
-          return unless filename&.match?(name_regex) || url&.match?(name_regex)
-
-          version = get_version_from_filename(filename)
-          return unless version_class.correct?(version)
-
-          {
-            version: version_class.new(version),
-            python_requirement: build_python_requirement_from_link(link),
-            yanked: link&.include?("data-yanked")
-          }
-        end
-        # rubocop:enable Metrics/PerceivedComplexity
-
-        def get_version_from_filename(filename)
-          filename
-            .gsub(/#{name_regex}-/i, "")
-            .split(/-|\.tar\.|\.zip|\.whl/)
-            .first
-        end
-
-        def build_python_requirement_from_link(link)
-          req_string = Nokogiri::XML(link)
-                               .at_css("a")
-                               &.attribute("data-requires-python")
-                               &.content
-
-          return unless req_string
-
-          requirement_class.new(CGI.unescapeHTML(req_string))
-        rescue Gem::Requirement::BadRequirementError
-          nil
-        end
-
-        def index_urls
-          @index_urls ||=
-            IndexFinder.new(
-              dependency_files: dependency_files,
-              credentials: credentials,
-              dependency: dependency
-            ).index_urls
-        end
-
-        def registry_response_for_dependency(index_url)
-          Dependabot::RegistryClient.get(
-            url: index_url + normalised_name + "/",
-            headers: { "Accept" => "text/html" }
-          )
-        end
-
-        def registry_index_response(index_url)
-          Dependabot::RegistryClient.get(
-            url: index_url,
-            headers: { "Accept" => "text/html" }
-          )
-        end
-
-        def ignore_requirements
-          ignored_versions.flat_map { |req| requirement_class.requirements_array(req) }
-        end
-
-        def normalised_name
-          NameNormaliser.normalise(dependency.name)
-        end
-
-        def name_regex
-          parts = normalised_name.split(/[\s_.-]/).map { |n| Regexp.quote(n) }
-          /#{parts.join("[\s_.-]")}/i
-        end
-
-        def version_class
-          dependency.version_class
-        end
-
-        def requirement_class
-          dependency.requirement_class
-        end
-
-        def validate_index(index_url)
-          sanitized_url = index_url.gsub(%r{(?<=//).*(?=@)}, "redacted")
-
-          return if index_url&.match?(URI::DEFAULT_PARSER.regexp[:ABS_URI])
-
-          raise Dependabot::DependencyFileNotResolvable,
-                "Invalid URL: #{sanitized_url}"
+          override.returns(T.nilable(Dependabot::Package::PackageDetails))
+        end
+        def package_details
+          @package_details ||= Package::PackageDetailsFetcher.new(
+            dependency: dependency,
+            dependency_files: dependency_files,
+            credentials: credentials
+          ).fetch
         end
       end
     end

data/lib/dependabot/python/update_checker/pip_version_resolver.rb

@@ -22,17 +22,17 @@ module Dependabot
         end
 
         def latest_resolvable_version
-          latest_version_finder.latest_version(python_version: language_version_manager.python_version)
+          latest_version_finder.latest_version(language_version: language_version_manager.python_version)
         end
 
         def latest_resolvable_version_with_no_unlock
           latest_version_finder
-            .latest_version_with_no_unlock(python_version: language_version_manager.python_version)
+            .latest_version_with_no_unlock(language_version: language_version_manager.python_version)
         end
 
         def lowest_resolvable_security_fix_version
           latest_version_finder
-            .lowest_security_fix_version(python_version: language_version_manager.python_version)
+            .lowest_security_fix_version(language_version: language_version_manager.python_version)
         end
 
         private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-python
 version: !ruby/object:Gem::Version
-  version: 0.298.0
+  version: 0.299.0
 platform: ruby
 authors:
 - Dependabot
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-20 00:00:00.000000000 Z
+date: 2025-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.298.0
+        version: 0.299.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.298.0
+        version: 0.299.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -271,13 +271,14 @@ files:
 - lib/dependabot/python/metadata_finder.rb
 - lib/dependabot/python/name_normaliser.rb
 - lib/dependabot/python/native_helpers.rb
+- lib/dependabot/python/package/package_details_fetcher.rb
+- lib/dependabot/python/package/package_registry_finder.rb
 - lib/dependabot/python/package_manager.rb
 - lib/dependabot/python/pip_compile_file_matcher.rb
 - lib/dependabot/python/pipenv_runner.rb
 - lib/dependabot/python/requirement.rb
 - lib/dependabot/python/requirement_parser.rb
 - lib/dependabot/python/update_checker.rb
-- lib/dependabot/python/update_checker/index_finder.rb
 - lib/dependabot/python/update_checker/latest_version_finder.rb
 - lib/dependabot/python/update_checker/pip_compile_version_resolver.rb
 - lib/dependabot/python/update_checker/pip_version_resolver.rb
@@ -290,7 +291,7 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
-  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.298.0
+  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.299.0
 post_install_message:
 rdoc_options: []
 require_paths: