dependabot-uv 0.300.0 → 0.301.1

data/lib/dependabot/uv/package/package_details_fetcher.rb

@@ -0,0 +1,488 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require "cgi"
+require "excon"
+require "nokogiri"
+require "sorbet-runtime"
+require "dependabot/registry_client"
+require "dependabot/uv/name_normaliser"
+require "dependabot/package/package_release"
+require "dependabot/package/package_details"
+require "dependabot/uv/package/package_registry_finder"
+
+# Stores metadata for a package, including all its available versions
+module Dependabot
+  module Uv
+    module Package
+      CREDENTIALS_USERNAME = "username"
+      CREDENTIALS_PASSWORD = "password"
+
+      APPLICATION_JSON = "application/json"
+      APPLICATION_TEXT = "text/html"
+      CPYTHON = "cpython"
+      PYTHON = "python"
+      UNKNOWN = "unknown"
+
+      MAIN_PYPI_INDEXES = %w(
+        https://pypi.python.org/simple/
+        https://pypi.org/simple/
+      ).freeze
+      VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/
+
+      class PackageDetailsFetcher
+        extend T::Sig
+
+        sig do
+          params(
+            dependency: Dependabot::Dependency,
+            dependency_files: T::Array[Dependabot::DependencyFile],
+            credentials: T::Array[Dependabot::Credential]
+          ).void
+        end
+        def initialize(
+          dependency:,
+          dependency_files:,
+          credentials:
+        )
+          @dependency          = dependency
+          @dependency_files    = dependency_files
+          @credentials         = credentials
+
+          @registry_urls = T.let(nil, T.nilable(T::Array[String]))
+        end
+
+        sig { returns(Dependabot::Dependency) }
+        attr_reader :dependency
+
+        sig { returns(T::Array[T.untyped]) }
+        attr_reader :dependency_files
+
+        sig { returns(T::Array[T.untyped]) }
+        attr_reader :credentials
+
+        sig { returns(Dependabot::Package::PackageDetails) }
+        def fetch
+          package_releases = registry_urls
+                             .select { |index_url| validate_index(index_url) } # Ensure only valid URLs
+                             .flat_map do |index_url|
+            fetch_from_registry(index_url) || [] # Ensure it always returns an array
+            rescue Excon::Error::Timeout, Excon::Error::Socket
+              raise if MAIN_PYPI_INDEXES.include?(index_url)
+
+              raise PrivateSourceTimedOut, sanitized_url(index_url)
+            rescue URI::InvalidURIError
+              raise DependencyFileNotResolvable, "Invalid URL: #{sanitized_url(index_url)}"
+          end
+
+          Dependabot::Package::PackageDetails.new(
+            dependency: dependency,
+            releases: package_releases.reverse.uniq(&:version)
+          )
+        end
+
+        private
+
+        sig do
+          params(index_url: String)
+            .returns(T.nilable(T::Array[Dependabot::Package::PackageRelease]))
+        end
+        def fetch_from_registry(index_url)
+          if Dependabot::Experiments.enabled?(:enable_cooldown_for_uv)
+            metadata = fetch_from_json_registry(index_url)
+
+            return metadata if metadata&.any?
+
+            Dependabot.logger.warn("No valid versions found via JSON API. Falling back to HTML.")
+          end
+          fetch_from_html_registry(index_url)
+        rescue StandardError => e
+          Dependabot.logger.warn("Unexpected error in JSON fetch: #{e.message}. Falling back to HTML.")
+          fetch_from_html_registry(index_url)
+        end
+
+        # Example JSON Response Format:
+        #
+        # {
+        #   "info": {
+        #     "name": "requests",
+        #     "summary": "Python HTTP for Humans.",
+        #     "author": "Kenneth Reitz",
+        #     "license": "Apache-2.0"
+        #   },
+        #   "releases": {
+        #     "2.32.3": [
+        #       {
+        #         "filename": "requests-2.32.3-py3-none-any.whl",
+        #         "version": "2.32.3",
+        #         "requires_python": ">=3.8",
+        #         "yanked": false,
+        #         "url": "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl"
+        #       },
+        #       {
+        #         "filename": "requests-2.32.3.tar.gz",
+        #         "version": "2.32.3",
+        #         "requires_python": ">=3.8",
+        #         "yanked": false,
+        #         "url": "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz"
+        #       }
+        #     ],
+        #     "2.27.0": [
+        #       {
+        #         "filename": "requests-2.27.0-py2.py3-none-any.whl",
+        #         "version": "2.27.0",
+        #         "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*",
+        #         "yanked": false,
+        #         "url": "https://files.pythonhosted.org/packages/47/01/f420e7add78110940639a958e5af0e3f8e07a8a8b62049bac55ee117aa91/requests-2.27.0-py2.py3-none-any.whl"
+        #       },
+        #       {
+        #         "filename": "requests-2.27.0.tar.gz",
+        #         "version": "2.27.0",
+        #         "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*",
+        #         "yanked": false,
+        #         "url": "https://files.pythonhosted.org/packages/c0/e3/826e27b942352a74b656e8f58b4dc7ed9495ce2d4eeb498181167c615303/requests-2.27.0.tar.gz"
+        #       }
+        #     ]
+        #   }
+        # }
+        sig do
+          params(index_url: String)
+            .returns(T.nilable(T::Array[Dependabot::Package::PackageRelease]))
+        end
+        def fetch_from_json_registry(index_url)
+          json_url = index_url.sub(%r{/simple/?$}i, "/pypi/")
+
+          Dependabot.logger.info(
+            "Fetching release information from json registry at #{sanitized_url(json_url)} for #{dependency.name}"
+          )
+
+          response = registry_json_response_for_dependency(json_url)
+
+          return nil unless response.status == 200
+
+          begin
+            data = JSON.parse(response.body)
+
+            version_releases = data["releases"]
+
+            releases = format_version_releases(version_releases)
+
+            releases.sort_by(&:version).reverse
+          rescue JSON::ParserError
+            Dependabot.logger.warn("JSON parsing error for #{json_url}. Falling back to HTML.")
+            nil
+          rescue StandardError => e
+            Dependabot.logger.warn("Unexpected error while fetching JSON data: #{e.message}.")
+            nil
+          end
+        end
+
+        # This URL points to the Simple Index API for the "requests" package on PyPI.
+        # It provides an HTML listing of available package versions following PEP 503 (Simple Repository API).
+        # The information found here is useful for dependency resolution and package version retrieval.
+        #
+        # ✅ Information available in the Simple Index:
+        # - A list of package versions as anchor (`<a>`) elements.
+        # - URLs to distribution files (e.g., `.tar.gz`, `.whl`).
+        # - The `data-requires-python` attribute (if present) specifying the required Python version.
+        # - An optional `data-yanked` attribute indicating a yanked (withdrawn) version.
+        #
+        # ❌ Information NOT available in the Simple Index:
+        # - Release timestamps (upload time).
+        # - File digests (hashes like SHA256, MD5).
+        # - Package metadata such as description, author, or dependencies.
+        # - Download statistics.
+        # - Package type (`sdist` or `bdist_wheel`).
+        #
+        # To obtain full package metadata, use the PyPI JSON API:
+        # - JSON API: https://pypi.org/pypi/requests/json
+        #
+        # More details: https://www.python.org/dev/peps/pep-0503/
+        sig { params(index_url: String).returns(T::Array[Dependabot::Package::PackageRelease]) }
+        def fetch_from_html_registry(index_url)
+          Dependabot.logger.info(
+            "Fetching release information from html registry at #{sanitized_url(index_url)} for #{dependency.name}"
+          )
+          index_response = registry_response_for_dependency(index_url)
+          if index_response.status == 401 || index_response.status == 403
+            registry_index_response = registry_index_response(index_url)
+
+            if registry_index_response.status == 401 || registry_index_response.status == 403
+              raise PrivateSourceAuthenticationFailure, sanitized_url(index_url)
+            end
+          end
+
+          version_releases = extract_release_details_json_from_html(index_response.body)
+          releases = format_version_releases(version_releases)
+
+          releases.sort_by(&:version).reverse
+        end
+
+        sig do
+          params(html_body: String)
+            .returns(T::Hash[String, T::Array[T::Hash[String, T.untyped]]]) # Returns JSON-like format
+        end
+        def extract_release_details_json_from_html(html_body)
+          doc = Nokogiri::HTML(html_body)
+
+          releases = {}
+
+          doc.css("a").each do |a_tag|
+            details = version_details_from_link(a_tag.to_s)
+            if details && details["version"]
+              releases[details["version"]] ||= []
+              releases[details["version"]] << details
+            end
+          end
+
+          releases
+        end
+
+        # rubocop:disable Metrics/PerceivedComplexity
+        sig do
+          params(link: T.nilable(String))
+            .returns(T.nilable(T::Hash[String, T.untyped]))
+        end
+        def version_details_from_link(link)
+          return unless link
+
+          doc = Nokogiri::XML(link)
+          filename = doc.at_css("a")&.content
+          url = doc.at_css("a")&.attributes&.fetch("href", nil)&.value
+
+          return unless filename&.match?(name_regex) || url&.match?(name_regex)
+
+          version = get_version_from_filename(filename)
+          return unless version_class.correct?(version)
+
+          {
+            "version" => version,
+            "requires_python" => requires_python_from_link(link),
+            "yanked" => link.include?("data-yanked"),
+            "url" => link
+          }
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        sig do
+          params(
+            releases_json: T.nilable(T::Hash[String, T::Array[T::Hash[String, T.untyped]]])
+          )
+            .returns(T::Array[Dependabot::Package::PackageRelease])
+        end
+        def format_version_releases(releases_json)
+          return [] unless releases_json
+
+          releases_json.each_with_object([]) do |(version, release_data_array), versions|
+            release_data = release_data_array.last
+
+            next unless release_data
+
+            release = format_version_release(version, release_data)
+
+            next unless release
+
+            versions << release
+          end
+        end
+
+        sig do
+          params(
+            version: String,
+            release_data: T::Hash[String, T.untyped]
+          )
+            .returns(T.nilable(Dependabot::Package::PackageRelease))
+        end
+        def format_version_release(version, release_data)
+          upload_time = release_data["upload_time"]
+          released_at = Time.parse(upload_time) if upload_time
+          yanked = release_data["yanked"] || false
+          yanked_reason = release_data["yanked_reason"]
+          downloads = release_data["downloads"] || -1
+          url = release_data["url"]
+          package_type = release_data["packagetype"]
+          language = package_language(
+            python_version: release_data["python_version"],
+            requires_python: release_data["requires_python"]
+          )
+
+          release = Dependabot::Package::PackageRelease.new(
+            version: Dependabot::Uv::Version.new(version),
+            released_at: released_at,
+            yanked: yanked,
+            yanked_reason: yanked_reason,
+            downloads: downloads,
+            url: url,
+            package_type: package_type,
+            language: language
+          )
+          release
+        end
+
+        sig do
+          params(
+            python_version: T.nilable(String),
+            requires_python: T.nilable(String)
+          )
+            .returns(T.nilable(Dependabot::Package::PackageLanguage))
+        end
+        def package_language(python_version:, requires_python:)
+          # Extract language name and version
+          language_name, language_version = convert_language_version(python_version)
+
+          # Extract language requirement
+          language_requirement = build_python_requirement(requires_python)
+
+          return nil unless language_version || language_requirement
+
+          # Return a Language object with all details
+          Dependabot::Package::PackageLanguage.new(
+            name: language_name,
+            version: language_version,
+            requirement: language_requirement
+          )
+        end
+
+        sig { params(version: T.nilable(String)).returns([String, T.nilable(Dependabot::Version)]) }
+        def convert_language_version(version)
+          return ["python", nil] if version.nil? || version == "source"
+
+          # Extract numeric parts dynamically (e.g., "cp37" -> "3.7", "py38" -> "3.8")
+          extracted_version = version.scan(/\d+/).join(".")
+
+          # Detect the language implementation
+          language_name = if version.start_with?("cp")
+                            "cpython" # CPython implementation
+                          elsif version.start_with?("py")
+                            "python" # General Python compatibility
+                          else
+                            "unknown" # Fallback for unknown cases
+                          end
+
+          # Ensure extracted version is valid before converting
+          language_version =
+            extracted_version.match?(/^\d+(\.\d+)*$/) ? Dependabot::Version.new(extracted_version) : nil
+
+          Dependabot.logger.warn("Skipping invalid language_version: #{version.inspect}") if language_version.nil?
+
+          [language_name, language_version]
+        end
+
+        sig { returns(T::Array[String]) }
+        def registry_urls
+          @registry_urls ||=
+            Package::PackageRegistryFinder.new(
+              dependency_files: dependency_files,
+              credentials: credentials,
+              dependency: dependency
+            ).registry_urls
+        end
+
+        sig { returns(String) }
+        def normalised_name
+          NameNormaliser.normalise(dependency.name)
+        end
+
+        sig { params(json_url: String).returns(Excon::Response) }
+        def registry_json_response_for_dependency(json_url)
+          url = "#{json_url.chomp('/')}/#{@dependency.name}/json"
+          Dependabot::RegistryClient.get(
+            url: url,
+            headers: { "Accept" => APPLICATION_JSON }
+          )
+        end
+
+        sig { params(index_url: String).returns(Excon::Response) }
+        def registry_response_for_dependency(index_url)
+          Dependabot::RegistryClient.get(
+            url: index_url + normalised_name + "/",
+            headers: { "Accept" => APPLICATION_TEXT }
+          )
+        end
+
+        sig { params(index_url: String).returns(Excon::Response) }
+        def registry_index_response(index_url)
+          Dependabot::RegistryClient.get(
+            url: index_url,
+            headers: { "Accept" => APPLICATION_TEXT }
+          )
+        end
+
+        sig { params(filename: String).returns(T.nilable(String)) }
+        def get_version_from_filename(filename)
+          filename
+            .gsub(/#{name_regex}-/i, "")
+            .split(/-|\.tar\.|\.zip|\.whl/)
+            .first
+        end
+
+        sig do
+          params(req_string: T.nilable(String))
+            .returns(T.nilable(Dependabot::Requirement))
+        end
+        def build_python_requirement(req_string)
+          return nil unless req_string
+
+          requirement_class.new(CGI.unescapeHTML(req_string))
+        rescue Gem::Requirement::BadRequirementError
+          nil
+        end
+
+        sig { params(link: String).returns(T.nilable(String)) }
+        def requires_python_from_link(link)
+          raw_value = Nokogiri::XML(link)
+                              .at_css("a")
+                              &.attribute("data-requires-python")
+                              &.content
+
+          return nil unless raw_value
+
+          CGI.unescapeHTML(raw_value) # Decodes HTML entities like &gt;=3 → >=3
+        end
+
+        sig { returns(T.class_of(Dependabot::Version)) }
+        def version_class
+          dependency.version_class
+        end
+
+        sig { returns(T.class_of(Dependabot::Requirement)) }
+        def requirement_class
+          dependency.requirement_class
+        end
+
+        sig { params(index_url: String).returns(T::Hash[String, String]) }
+        def auth_headers_for(index_url)
+          credential = @credentials.find { |cred| cred["index-url"] == index_url }
+          return {} unless credential
+
+          { "Authorization" => "Basic #{Base64.strict_encode64(
+            "#{credential[CREDENTIALS_USERNAME]}:#{credential[CREDENTIALS_PASSWORD]}"
+          )}" }
+        end
+
+        sig { returns(Regexp) }
+        def name_regex
+          parts = normalised_name.split(/[\s_.-]/).map { |n| Regexp.quote(n) }
+          /#{parts.join("[\s_.-]")}/i
+        end
+
+        sig { params(index_url: T.nilable(String)).returns(T::Boolean) }
+        def validate_index(index_url)
+          return false unless index_url
+
+          return true if index_url.match?(URI::DEFAULT_PARSER.regexp[:ABS_URI])
+
+          raise Dependabot::DependencyFileNotResolvable,
+                "Invalid URL: #{sanitized_url(index_url)}"
+        end
+
+        sig { params(index_url: String).returns(String) }
+        def sanitized_url(index_url)
+          index_url.sub(%r{//([^/@]+)@}, "//redacted@")
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/uv/{update_checker/index_finder.rb → package/package_registry_finder.rb}

@@ -7,8 +7,9 @@ require "dependabot/errors"
 
 module Dependabot
   module Uv
-    class UpdateChecker
-      class IndexFinder
+    module Package
+      class PackageRegistryFinder
+        extend T::Sig
         PYPI_BASE_URL = "https://pypi.org/simple/"
         ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/
 
@@ -18,7 +19,8 @@ module Dependabot
           @dependency       = dependency
         end
 
-        def index_urls
+        sig { returns(T::Array[String]) }
+        def registry_urls
           extra_index_urls =
             config_variable_index_urls[:extra] +
             pipfile_index_urls[:extra] +
@@ -140,7 +142,7 @@ module Dependabot
         end
 
         def config_variable_index_urls
-          urls = { main: nil, extra: [] }
+          urls = { main: T.let(nil, T.nilable(String)), extra: [] }
 
           index_url_creds = credentials
                             .select { |cred| cred["type"] == "python_index" }
@@ -158,7 +160,7 @@ module Dependabot
         end
 
         def clean_check_and_remove_environment_variables(url)
-          url = url.strip.gsub(%r{/*$}, "") + "/"
+          url = url.strip.sub(%r{/+$}, "") + "/"
 
           return authed_base_url(url) unless url.match?(ENVIRONMENT_VARIABLE_REGEX)
 
@@ -190,7 +192,11 @@ module Dependabot
           cred = credential_for(base_url)
           return base_url unless cred
 
-          AuthedUrlBuilder.authed_url(credential: cred).gsub(%r{/*$}, "") + "/"
+          builder = AuthedUrlBuilder.authed_url(credential: cred)
+
+          return base_url unless builder
+
+          builder.gsub(%r{/*$}, "") + "/"
         end
 
         def credential_for(url)