dependabot-python 0.392.0 → 0.394.0

data/lib/dependabot/python/package/package_details_fetcher.rb

@@ -12,6 +12,7 @@ require "dependabot/python/name_normaliser"
 require "dependabot/package/package_release"
 require "dependabot/package/package_details"
 require "dependabot/python/package/package_registry_finder"
+require "dependabot/python/package/simple_api_parser"
 
 # Stores metadata for a package, including all its available versions
 module Dependabot
@@ -21,6 +22,7 @@ module Dependabot
       CREDENTIALS_PASSWORD = "password"
 
       APPLICATION_JSON = "application/json"
+      APPLICATION_PYPI_SIMPLE_JSON = "application/vnd.pypi.simple.v1+json"
       APPLICATION_TEXT = "text/html"
       CPYTHON = "cpython"
       PYTHON = "python"
@@ -42,11 +44,7 @@ module Dependabot
             credentials: T::Array[Dependabot::Credential]
           ).void
         end
-        def initialize(
-          dependency:,
-          dependency_files:,
-          credentials:
-        )
+        def initialize(dependency:, dependency_files:, credentials:)
           @dependency          = dependency
           @dependency_files    = dependency_files
           @credentials         = credentials
@@ -70,7 +68,7 @@ module Dependabot
                              .flat_map do |index_url|
             fetch_from_registry(index_url) || [] # Ensure it always returns an array
           rescue Excon::Error::Timeout, Excon::Error::Socket
-            raise if MAIN_PYPI_INDEXES.include?(index_url)
+            raise if main_pypi_index?(index_url)
 
             raise PrivateSourceTimedOut, sanitized_url(index_url)
           rescue URI::InvalidURIError
@@ -90,15 +88,46 @@ module Dependabot
             .returns(T.nilable(T::Array[Dependabot::Package::PackageRelease]))
         end
         def fetch_from_registry(index_url)
-          metadata = fetch_from_json_registry(index_url)
+          return fetch_from_simple_registry(index_url) unless main_pypi_index?(index_url)
 
-          return metadata if metadata&.any?
+          begin
+            metadata = fetch_from_json_registry(index_url)
+
+            return metadata if metadata&.any?
 
-          Dependabot.logger.warn("No valid versions found via JSON API. Falling back to HTML.")
-          fetch_from_html_registry(index_url)
-        rescue StandardError => e
-          Dependabot.logger.warn("Unexpected error in JSON fetch: #{e.message}. Falling back to HTML.")
-          fetch_from_html_registry(index_url)
+            Dependabot.logger.warn("No valid versions found via JSON API. Falling back to HTML.")
+            fetch_from_html_registry(index_url)
+          rescue Dependabot::DependencyFileNotResolvable
+            raise
+          rescue StandardError => e
+            Dependabot.logger.warn("Unexpected error in JSON fetch: #{e.message}. Falling back to HTML.")
+            fetch_from_html_registry(index_url)
+          end
+        end
+
+        sig { params(index_url: String).returns(T::Array[Dependabot::Package::PackageRelease]) }
+        def fetch_from_simple_registry(index_url)
+          Dependabot.logger.info(
+            "Fetching release information from simple registry at #{sanitized_url(index_url)} for #{dependency.name}"
+          )
+          response = registry_response_for_dependency(index_url, accept: simple_api_accept)
+          project_url = response.path ? Excon::Utils.request_uri(response.data) : index_url + normalised_name + "/"
+          check_authentication_response(response, index_url)
+
+          version_releases = if simple_json_response?(response)
+                               SimpleApiParser.new(
+                                 dependency: dependency,
+                                 project_url: project_url
+                               ).parse(response.body)
+                             else
+                               extract_release_details_json_from_html(response.body)
+                             end
+
+          format_version_releases(version_releases, preserve_distributions: simple_json_response?(response))
+            .sort_by(&:version).reverse
+        rescue JSON::ParserError
+          Dependabot.logger.warn("JSON parsing error for #{sanitized_url(index_url)}.")
+          []
         end
 
         # Example JSON Response Format:
@@ -203,14 +232,8 @@ module Dependabot
           Dependabot.logger.info(
             "Fetching release information from html registry at #{sanitized_url(index_url)} for #{dependency.name}"
           )
-          index_response = registry_response_for_dependency(index_url)
-          if index_response.status == 401 || index_response.status == 403
-            registry_index_response = registry_index_response(index_url)
-
-            if registry_index_response.status == 401 || registry_index_response.status == 403
-              raise PrivateSourceAuthenticationFailure, sanitized_url(index_url)
-            end
-          end
+          index_response = registry_response_for_dependency(index_url, accept: APPLICATION_TEXT)
+          check_authentication_response(index_response, index_url)
 
           version_releases = extract_release_details_json_from_html(index_response.body)
           releases = format_version_releases(version_releases)
@@ -218,6 +241,27 @@ module Dependabot
           releases.sort_by(&:version).reverse
         end
 
+        sig { params(response: Excon::Response).returns(T::Boolean) }
+        def simple_json_response?(response)
+          response.headers["Content-Type"].to_s.split(";", 2).first.to_s.strip.downcase == APPLICATION_PYPI_SIMPLE_JSON
+        end
+
+        sig { returns(String) }
+        def simple_api_accept
+          "application/vnd.pypi.simple.v1+json, " \
+            "application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01"
+        end
+
+        sig { params(response: Excon::Response, index_url: String).void }
+        def check_authentication_response(response, index_url)
+          return unless [401, 403].include?(response.status)
+
+          index_response = registry_index_response(index_url)
+          return unless [401, 403].include?(index_response.status)
+
+          raise PrivateSourceAuthenticationFailure, sanitized_url(index_url)
+        end
+
         sig do
           params(html_body: String)
             .returns(T::Hash[String, T::Array[T::Hash[String, T.untyped]]]) # Returns JSON-like format
@@ -266,23 +310,17 @@ module Dependabot
 
         sig do
           params(
-            releases_json: T.nilable(T::Hash[String, T::Array[T::Hash[String, T.untyped]]])
+            releases_json: T.nilable(T::Hash[String, T::Array[T::Hash[String, T.untyped]]]),
+            preserve_distributions: T::Boolean
           )
             .returns(T::Array[Dependabot::Package::PackageRelease])
         end
-        def format_version_releases(releases_json)
+        def format_version_releases(releases_json, preserve_distributions: false)
           return [] unless releases_json
 
           releases_json.each_with_object([]) do |(version, release_data_array), versions|
-            release_data = release_data_array.last
-
-            next unless release_data
-
-            release = format_version_release(version, release_data)
-
-            next unless release
-
-            versions << release
+            release_data_array = [release_data_array.last].compact unless preserve_distributions
+            versions.concat(release_data_array.filter_map { |data| format_version_release(version, data) })
           end
         end
 
@@ -397,11 +435,11 @@ module Dependabot
           )
         end
 
-        sig { params(index_url: String).returns(Excon::Response) }
-        def registry_response_for_dependency(index_url)
+        sig { params(index_url: String, accept: String).returns(Excon::Response) }
+        def registry_response_for_dependency(index_url, accept:)
           Dependabot::RegistryClient.get(
             url: index_url + normalised_name + "/",
-            headers: { "Accept" => APPLICATION_TEXT }
+            headers: { "Accept" => accept }
           )
         end
 
@@ -487,6 +525,14 @@ module Dependabot
           index_url.sub(%r{//([^/@]+)@}, "//redacted@")
         end
 
+        sig { params(index_url: String).returns(T::Boolean) }
+        def main_pypi_index?(index_url)
+          uri = URI.parse(index_url)
+          T.must(uri.scheme.to_s.casecmp("https")).zero? && uri.port == 443 &&
+            uri.path.to_s.split("/").reject(&:empty?) == ["simple"] &&
+            %w(pypi.org pypi.python.org).include?(uri.host.to_s.downcase)
+        end
+
         sig { params(dep_name: String).returns(String) }
         def remove_optional(dep_name)
           # Remove any optional dependencies postfix, e.g., "pyvista[io] "

data/lib/dependabot/python/package/package_registry_finder.rb

@@ -193,7 +193,7 @@ module Dependabot
 
         sig { params(url: String).returns(String) }
         def clean_check_and_remove_environment_variables(url)
-          url = url.strip.sub(%r{/+$}, "") + "/"
+          url = with_single_trailing_slash(url.strip)
 
           return authed_base_url(url) unless url.match?(ENVIRONMENT_VARIABLE_REGEX)
 
@@ -203,7 +203,7 @@ module Dependabot
               *config_variable_index_urls[:extra]
             ]
             .compact
-            .map { |u| u.strip.gsub(%r{/*$}, "") + "/" }
+            .map { |u| with_single_trailing_slash(u.strip) }
 
           regexp = url
                    .sub(%r{(?<=://).+@}, "")
@@ -226,7 +226,7 @@ module Dependabot
           cred = credential_for(base_url)
           return base_url unless cred
 
-          AuthedUrlBuilder.authed_url(credential: cred).gsub(%r{/*$}, "") + "/"
+          with_single_trailing_slash(AuthedUrlBuilder.authed_url(credential: cred))
         end
 
         sig { params(url: String).returns(T.nilable(Dependabot::Credential)) }
@@ -234,11 +234,18 @@ module Dependabot
           credentials
             .select { |c| c["type"] == "python_index" }
             .find do |c|
-              cred_url = c.fetch("index-url").gsub(%r{/*$}, "") + "/"
+              cred_url = with_single_trailing_slash(c.fetch("index-url"))
               cred_url.include?(url)
             end
         end
 
+        sig { params(url: String).returns(String) }
+        def with_single_trailing_slash(url)
+          normalized_url = url.dup
+          normalized_url.chop! while normalized_url.end_with?("/")
+          normalized_url + "/"
+        end
+
         sig { returns(T.nilable(Dependabot::DependencyFile)) }
         def pip_conf
           dependency_files.find { |f| f.name == "pip.conf" }

data/lib/dependabot/python/package/simple_api_parser.rb

@@ -0,0 +1,91 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+require "sorbet-runtime"
+require "dependabot/errors"
+require "dependabot/python/name_normaliser"
+
+module Dependabot
+  module Python
+    module Package
+      class SimpleApiParser
+        extend T::Sig
+
+        ReleaseDetail = T.type_alias { T::Hash[String, T.nilable(T.any(String, T::Boolean))] }
+
+        sig { params(dependency: Dependabot::Dependency, project_url: String).void }
+        def initialize(dependency:, project_url:)
+          @dependency = dependency
+          @project_url = project_url
+        end
+
+        sig do
+          params(json_body: String)
+            .returns(T::Hash[String, T::Array[ReleaseDetail]])
+        end
+        def parse(json_body)
+          data = JSON.parse(json_body)
+          validate_api_version!(data.dig("meta", "api-version").to_s)
+
+          data.fetch("files", []).each_with_object({}) do |file, releases|
+            filename = file["filename"]
+            next unless filename&.match?(name_regex)
+
+            version = version_from_filename(filename)
+            next unless dependency.version_class.correct?(version)
+
+            yanked = file["yanked"] || false
+            details = {
+              "version" => version,
+              "requires_python" => file["requires-python"],
+              "yanked" => yanked != false,
+              "yanked_reason" => yanked.is_a?(String) ? yanked : nil,
+              "upload_time" => file["upload-time"],
+              "url" => resolve_url(file["url"])
+            }
+            releases[version] ||= []
+            releases[version] << details
+          end
+        end
+
+        private
+
+        sig { returns(Dependabot::Dependency) }
+        attr_reader :dependency
+
+        sig { returns(String) }
+        attr_reader :project_url
+
+        sig { params(url: T.nilable(String)).returns(T.nilable(String)) }
+        def resolve_url(url)
+          return unless url
+
+          resolved_url = URI.join(project_url, url)
+          resolved_url.user = nil
+          resolved_url.password = nil
+          resolved_url.to_s
+        end
+
+        sig { params(api_version: String).void }
+        def validate_api_version!(api_version)
+          return unless api_version.split(".").first.to_i > 1
+
+          raise Dependabot::DependencyFileNotResolvable, "Unsupported PEP 691 API version: #{api_version}"
+        end
+
+        sig { returns(Regexp) }
+        def name_regex
+          parts = NameNormaliser.normalise(dependency.name).split(/[\s_.-]/).map { |name| Regexp.quote(name) }
+          /#{parts.join("[\s_.-]")}/i
+        end
+
+        sig { params(filename: String).returns(T.nilable(String)) }
+        def version_from_filename(filename)
+          filename.strip.gsub(/#{name_regex}-/i, "").split(/-|\.tar\.|\.zip|\.whl/).first
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/shared_file_fetcher.rb

@@ -249,8 +249,7 @@ module Dependabot
           next if previously_fetched_files.map(&:name).include?(path)
           next if file.name == path
 
-          if Dependabot::Experiments.enabled?(:enable_exclude_paths_subdirectory_manifest_files) &&
-             !@exclude_paths.empty? && Dependabot::FileFiltering.exclude_path?(path, @exclude_paths)
+          if !@exclude_paths.empty? && Dependabot::FileFiltering.exclude_path?(path, @exclude_paths)
             raise Dependabot::DependencyFileNotEvaluatable,
                   "Cannot process requirements: '#{file.name}' references excluded file '#{path}'. " \
                   "Please either remove the reference from '#{file.name}' " \

data/lib/dependabot/python/update_checker/latest_version_finder.rb

@@ -42,6 +42,22 @@ module Dependabot
             cooldown.semver_minor_days.to_i.positive? ||
             cooldown.semver_patch_days.to_i.positive?
         end
+
+        sig do
+          params(language_version: T.nilable(T.any(String, Dependabot::Version)))
+            .returns(T.nilable(T::Array[Dependabot::Package::PackageRelease]))
+        end
+        def eligible_releases(language_version: nil)
+          releases = available_versions
+          return unless releases
+
+          releases = filter_yanked_versions(releases)
+          releases = filter_by_cooldown(releases)
+          releases = filter_unsupported_versions(releases, language_version)
+          releases = filter_prerelease_versions(releases)
+          releases = filter_ignored_versions(releases)
+          apply_post_fetch_latest_versions_filter(releases)
+        end
       end
     end
   end

data/lib/dependabot/python/update_checker.rb

@@ -129,7 +129,7 @@ module Dependabot
 
       sig { returns(T.nilable(Gem::Version)) }
       def latest_version_for_git_dependency
-        latest_git_version_details&.fetch(:version)
+        latest_git_version_details&.version
       end
 
       sig { returns(T.nilable(Gem::Version)) }
@@ -152,7 +152,7 @@ module Dependabot
       def updated_git_source
         # Update the git tag if a new version is available
         if git_commit_checker.pinned_ref_looks_like_version? && latest_git_version_details
-          new_tag = T.must(latest_git_version_details).fetch(:tag)
+          new_tag = T.must(latest_git_version_details).tag
           source_details = dependency.source_details
           return source_details.transform_keys(&:to_sym).merge(ref: new_tag) if source_details
         end
@@ -161,11 +161,11 @@ module Dependabot
         dependency.source_details&.transform_keys(&:to_sym)
       end
 
-      sig { returns(T.nilable(T::Hash[Symbol, T.untyped])) }
+      sig { returns(T.nilable(Dependabot::GitTagDetails)) }
       def latest_git_version_details
         @latest_git_version_details ||= T.let(
           git_commit_checker.local_tag_for_latest_version(@update_cooldown),
-          T.nilable(T::Hash[Symbol, T.untyped])
+          T.nilable(Dependabot::GitTagDetails)
         )
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-python
 version: !ruby/object:Gem::Version
-  version: 0.392.0
+  version: 0.394.0
 platform: ruby
 authors:
 - Dependabot
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.392.0
+        version: 0.394.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.392.0
+        version: 0.394.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -169,14 +169,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.22'
+        version: '1.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.22'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: turbo_tests
   requirement: !ruby/object:Gem::Requirement
@@ -280,7 +280,10 @@ files:
 - lib/dependabot/python/dependency_grapher/requirements_layers.rb
 - lib/dependabot/python/file_fetcher.rb
 - lib/dependabot/python/file_parser.rb
+- lib/dependabot/python/file_parser/pep_dependency.rb
 - lib/dependabot/python/file_parser/pipfile_files_parser.rb
+- lib/dependabot/python/file_parser/poetry_lock.rb
+- lib/dependabot/python/file_parser/pyproject_document.rb
 - lib/dependabot/python/file_parser/pyproject_files_parser.rb
 - lib/dependabot/python/file_parser/python_requirement_parser.rb
 - lib/dependabot/python/file_parser/setup_file_parser.rb
@@ -302,6 +305,7 @@ files:
 - lib/dependabot/python/native_helpers.rb
 - lib/dependabot/python/package/package_details_fetcher.rb
 - lib/dependabot/python/package/package_registry_finder.rb
+- lib/dependabot/python/package/simple_api_parser.rb
 - lib/dependabot/python/package_manager.rb
 - lib/dependabot/python/pip_compile_file_matcher.rb
 - lib/dependabot/python/pipenv_runner.rb
@@ -323,7 +327,7 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
-  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.392.0
+  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.394.0
 rdoc_options: []
 require_paths:
 - lib