dependabot-python 0.367.0 → 0.368.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e54cc0660ba9d6007661da639e66151ac7cdab9dafb2ea460ee544b825e7790
-  data.tar.gz: eaaec236cd949ffa2a446b5056a56378a2bcf954eb32da78e0ec0cb9f7b0bb4c
+  metadata.gz: 5b03d84191a73f9cd803b3db1fd32bc70638a4f607c2e9424a3900f143119f5f
+  data.tar.gz: 6d04a235c38eb76a1a86eb2f60e2f34c80c82ff5f342d997ff3c8a25011e42f3
 SHA512:
-  metadata.gz: 88f0d4fc4032adab9165c2d0187ef56660ebc219472723ad97c61c128e764cfeac49d4a49abfa4999f05ac85f9ccee3189dec442fc5701bf1afd41d18d706fba
-  data.tar.gz: 931b2880cdfda0184098b61416963e02542a13993752f83e8682f3c25b7de3e2f6c45affe5a80e231a331fc6bcd4b62b0eec3bab33aae065a92d70932e31c844
+  metadata.gz: 793a849e50f2162b65f1da65521dfb7450081b84507fbb4134145ab5c72b63a916a59c68602bf41dea530d373cb0d2dd5e9a38fee9b6a5347b17848a8891054f
+  data.tar.gz: db96c2fd7be7c4b6ac473f29dc429f3c4163fb2dd38d4b1d48c19ea62727022ff6822166dd10e9c4a680eae24bc5f16f447bf9f8ec55b7909b8ae663698b29c6

data/lib/dependabot/python/file_updater/requirement_file_updater.rb

@@ -55,33 +55,56 @@ module Dependabot
 
         sig { returns(T::Array[Dependabot::DependencyFile]) }
         def fetch_updated_dependency_files
-          reqs = dependency.requirements.zip(dependency.previous_requirements || [])
+          updated_contents = T.let({}, T::Hash[String, String])
+
+          unique_requirement_changes.each do |pair|
+            new_req = T.must(pair[0])
+            old_req = pair[1]
+            filename = new_req.fetch(:file)
+            content = updated_contents[filename] || T.must(T.must(get_original_file(filename)).content)
+            updated_contents[filename] = updated_requirement_or_setup_file_content(content, new_req, old_req)
+          end
+
+          updated_contents.filter_map do |filename, content|
+            file = T.must(get_original_file(filename)).dup
+            next if content == T.must(file.content)
+
+            file.content = content
+            file
+          end
+        end
+
+        # Deduplicates requirements that share the same file and requirement strings.
+        # The replacer's regex matches all extras variants at once, so one call per
+        # unique (file, old_requirement, new_requirement) is sufficient.
+        sig { returns(T::Array[T::Array[T.nilable(T::Hash[Symbol, T.untyped])]]) }
+        def unique_requirement_changes
+          previous_reqs = dependency.previous_requirements || []
 
-          reqs.filter_map do |(new_req, old_req)|
+          changes = dependency.requirements.filter_map do |new_req|
+            old_req = previous_reqs.find { |r| r[:file] == new_req[:file] && r[:groups] == new_req[:groups] }
             next if new_req == old_req
 
-            file = T.must(get_original_file(new_req.fetch(:file))).dup
-            updated_content =
-              updated_requirement_or_setup_file_content(new_req, old_req)
-            next if updated_content == file.content
+            [new_req, old_req]
+          end
 
-            file.content = updated_content
-            file
+          changes.uniq do |pair|
+            new_req = pair[0]
+            old_req = pair[1]
+            [new_req[:file], old_req&.fetch(:requirement), new_req.fetch(:requirement)]
           end
         end
 
         sig do
           params(
+            content: String,
             new_req: T::Hash[Symbol, T.untyped],
             old_req: T.nilable(T::Hash[Symbol, T.untyped])
           ).returns(String)
         end
-        def updated_requirement_or_setup_file_content(new_req, old_req)
-          original_file = get_original_file(new_req.fetch(:file))
-          raise "Could not find a dependency file for #{new_req}" unless original_file
-
+        def updated_requirement_or_setup_file_content(content, new_req, old_req)
           RequirementReplacer.new(
-            content: T.must(original_file.content),
+            content: content,
             dependency_name: dependency.name,
             old_requirement: old_req&.fetch(:requirement),
             new_requirement: new_req.fetch(:requirement),

data/lib/dependabot/python/file_updater/requirement_replacer.rb

@@ -53,7 +53,7 @@ module Dependabot
               # ignore it, since it isn't actually a declaration
               next mtch if Regexp.last_match&.pre_match&.match?(/--.*\z/)
 
-              updated_dependency_declaration_string
+              updated_matched_declaration(mtch)
             end
 
           raise "Expected content to change!" if old_requirement != new_requirement && content == updated_content
@@ -98,29 +98,30 @@ module Dependabot
           new_req_string
         end
 
-        sig { returns(String) }
-        def updated_dependency_declaration_string
-          old_req = old_requirement
-          updated_string =
-            if old_req
-              original_dependency_declaration_string(old_req)
-                .sub(RequirementParser::REQUIREMENTS, updated_requirement_string || "")
-            else
-              original_dependency_declaration_string(old_req)
-                .sub(RequirementParser::NAME_WITH_EXTRAS) do |nm|
-                  nm + (updated_requirement_string || "")
-                end
-            end
-
-          return updated_string unless update_hashes? && requirement_includes_hashes?(old_req)
-
-          updated_string.sub(
+        # Builds updated declaration from the actual matched text, preserving
+        # whatever extras (or lack thereof) appeared in the original match.
+        sig { params(matched_declaration: String).returns(String) }
+        def updated_matched_declaration(matched_declaration)
+          updated = if old_requirement
+                      matched_declaration
+                        .sub(RequirementParser::REQUIREMENTS, updated_requirement_string || "")
+                    else
+                      matched_declaration
+                        .sub(RequirementParser::NAME_WITH_EXTRAS) { |nm| nm + (updated_requirement_string || "") }
+                    end
+
+          return updated unless update_hashes? && matched_declaration.match?(RequirementParser::HASHES)
+
+          algorithm = T.must(matched_declaration.match(RequirementParser::HASHES))
+                       .named_captures.fetch("algorithm")
+          separator = hash_separator(old_requirement)
+          updated.sub(
             RequirementParser::HASHES,
             package_hashes_for(
               name: dependency_name,
               version: new_hash_version,
-              algorithm: hash_algorithm(old_req)
-            ).join(hash_separator(old_req))
+              algorithm: algorithm
+            ).join(separator)
           )
         end
 
@@ -142,7 +143,14 @@ module Dependabot
         def original_declaration_replacement_regex
           original_string =
             original_dependency_declaration_string(old_requirement)
-          /(?<![\-\w\.\[])#{Regexp.escape(original_string)}(?![\-\w\.])/
+          match_data = T.must(original_string.match(RequirementParser::NAME_WITH_EXTRAS))
+          name_escaped = Regexp.escape(T.must(match_data[:name]))
+          # Everything after name+extras (the requirement/markers/hashes portion)
+          after_name_extras = T.must(original_string[T.must(match_data[0]).length..]).strip
+          after_escaped = Regexp.escape(after_name_extras)
+          # Match the dependency name with any extras (or none), followed by the requirement.
+          # This ensures a single gsub handles all extras variants of the same dependency.
+          /(?<![\-\w\.\[])#{name_escaped}\s*\\?\s*(?:\[[^\]]*\])?\s*\\?\s*#{after_escaped}(?![\-\w\.])/
         end
 
         sig { params(requirement: T.nilable(String)).returns(T::Boolean) }

data/lib/dependabot/python/metadata_finder.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "excon"
+require "openssl"
 require "uri"
 
 require "dependabot/metadata_finders"
@@ -27,6 +28,7 @@ module Dependabot
       def initialize(dependency:, credentials:)
         super
         @pypi_listing = T.let(nil, T.nilable(T::Hash[String, T.untyped]))
+        @parsed_source_urls = T.let({}, T::Hash[String, T.nilable(Dependabot::Source)])
       end
 
       sig { returns(T.nilable(String)) }
@@ -37,26 +39,88 @@ module Dependabot
           super
       end
 
+      sig { override.returns(T.nilable(String)) }
+      def maintainer_changes
+        return unless dependency.previous_version
+        return unless dependency.version
+
+        previous_ownership = ownership_for_version(T.must(dependency.previous_version))
+        current_ownership = ownership_for_version(T.must(dependency.version))
+
+        if previous_ownership.nil? || current_ownership.nil?
+          Dependabot.logger.info("Unable to determine ownership changes for #{dependency.name}")
+          return
+        end
+
+        previous_org = previous_ownership["organization"]
+        current_org = current_ownership["organization"]
+
+        if previous_org != current_org && !(previous_org.nil? && current_org)
+          return "The organization that maintains #{dependency.name} on PyPI has " \
+                 "changed since your current version."
+        end
+
+        previous_users = ownership_users(previous_ownership)
+        current_users = ownership_users(current_ownership)
+
+        # Warn only when there were previous maintainers and none of them remain
+        return unless previous_users.any? && !previous_users.intersect?(current_users)
+
+        "None of the maintainers for your current version of #{dependency.name} are " \
+          "listed as maintainers for the new version on PyPI."
+      end
+
       private
 
       sig { override.returns(T.nilable(Dependabot::Source)) }
       def look_up_source
+        source_url = exact_match_source_url_from_project_urls
+        source_url ||= labelled_source_url_from_project_urls
+        source_url ||= fallback_source_url
+        source_url ||= source_from_description
+        source_url ||= source_from_homepage
+
+        parsed_source_from_url(source_url)
+      end
+
+      sig { returns(T.nilable(String)) }
+      def exact_match_source_url_from_project_urls
+        project_urls.values.find do |url|
+          repo = parsed_source_from_url(url)&.repo
+          repo&.downcase&.end_with?(normalised_dependency_name)
+        end
+      end
+
+      sig { returns(T.nilable(String)) }
+      def labelled_source_url_from_project_urls
+        source_urls = source_like_project_url_labels.filter_map do |label|
+          project_urls[label]
+        end
+
+        source_urls.find { |url| parsed_source_from_url(url) }
+      end
+
+      sig { returns(T.nilable(String)) }
+      def fallback_source_url
         potential_source_urls = [
-          pypi_listing.dig("info", "project_urls", "Source"),
-          pypi_listing.dig("info", "project_urls", "Repository"),
           pypi_listing.dig("info", "home_page"),
           pypi_listing.dig("info", "download_url"),
           pypi_listing.dig("info", "docs_url")
         ].compact
 
-        potential_source_urls +=
-          (pypi_listing.dig("info", "project_urls") || {}).values
+        potential_source_urls += project_urls.values
 
-        source_url = potential_source_urls.find { |url| Source.from_url(url) }
-        source_url ||= source_from_description
-        source_url ||= source_from_homepage
+        potential_source_urls.find { |url| parsed_source_from_url(url) }
+      end
 
-        Source.from_url(source_url)
+      sig { returns(T::Hash[String, String]) }
+      def project_urls
+        pypi_listing.dig("info", "project_urls") || {}
+      end
+
+      sig { returns(T::Array[String]) }
+      def source_like_project_url_labels
+        ["Source", "Source Code", "Repository", "Code", "Homepage"]
       end
 
       # rubocop:disable Metrics/PerceivedComplexity
@@ -73,7 +137,7 @@ module Dependabot
         # Looking for a source where the repo name exactly matches the
         # dependency name
         match_url = potential_source_urls.find do |url|
-          repo = Source.from_url(url)&.repo
+          repo = parsed_source_from_url(url)&.repo
           repo&.downcase&.end_with?(normalised_dependency_name)
         end
 
@@ -83,11 +147,14 @@ module Dependabot
         # mentioned when the link is followed
         @source_from_description ||= T.let(
           potential_source_urls.find do |url|
-            full_url = Source.from_url(url)&.url
+            full_url = parsed_source_from_url(url)&.url
             next unless full_url
 
             response = Dependabot::RegistryClient.get(url: full_url)
-            next unless response.status == 200
+            unless response.status == 200
+              Dependabot.logger.warn("Error fetching source URL #{full_url}: HTTP #{response.status}")
+              next
+            end
 
             response.body.include?(normalised_dependency_name)
           end,
@@ -108,7 +175,7 @@ module Dependabot
         end
 
         match_url = potential_source_urls.find do |url|
-          repo = Source.from_url(url)&.repo
+          repo = parsed_source_from_url(url)&.repo
           repo&.downcase&.end_with?(normalised_dependency_name)
         end
 
@@ -116,7 +183,7 @@ module Dependabot
 
         @source_from_homepage ||= T.let(
           potential_source_urls.find do |url|
-            full_url = Source.from_url(url)&.url
+            full_url = parsed_source_from_url(url)&.url
             next unless full_url
 
             response = Dependabot::RegistryClient.get(url: full_url)
@@ -143,7 +210,8 @@ module Dependabot
           begin
             Dependabot::RegistryClient.get(url: homepage_url)
           rescue Excon::Error::Timeout, Excon::Error::Socket,
-                 Excon::Error::TooManyRedirects, ArgumentError
+                 Excon::Error::TooManyRedirects, OpenSSL::SSL::SSLError, ArgumentError => e
+            Dependabot.logger.warn("Error fetching Python homepage URL #{homepage_url}: #{e.class}: #{e.message}")
             nil
           end,
           T.nilable(Excon::Response)
@@ -165,9 +233,8 @@ module Dependabot
 
           @pypi_listing = JSON.parse(response.body)
           return @pypi_listing
-        rescue JSON::ParserError
-          next
-        rescue Excon::Error::Timeout
+        rescue JSON::ParserError, Excon::Error::Timeout, Excon::Error::Socket, OpenSSL::SSL::SSLError => e
+          Dependabot.logger.warn("Error fetching Python package listing from #{url}: #{e.class}: #{e.message}")
           next
         end
 
@@ -194,23 +261,88 @@ module Dependabot
 
       sig { returns(T::Array[String]) }
       def possible_listing_urls
-        credential_urls =
+        index_credentials =
           credentials
           .select { |cred| cred["type"] == "python_index" }
-          .map { |c| AuthedUrlBuilder.authed_url(credential: c) }
 
-        (credential_urls + [MAIN_PYPI_URL]).map do |base_url|
+        credential_urls = index_credentials
+                          .map { |c| AuthedUrlBuilder.authed_url(credential: c) }
+                          .reject { |url| url.strip.empty? }
+
+        base_urls = if index_credentials.any?(&:replaces_base?)
+                      credential_urls
+                    else
+                      credential_urls + [MAIN_PYPI_URL]
+                    end
+        base_urls = [MAIN_PYPI_URL] if base_urls.empty?
+
+        base_urls.map do |base_url|
           # Convert /simple/ endpoints to /pypi/ for JSON API access
           json_base_url = base_url.sub(%r{/simple/?$}i, "/pypi")
           json_base_url.gsub(%r{/$}, "") + "/#{normalised_dependency_name}/json"
         end
       end
 
+      sig { params(version: String).returns(T.nilable(T::Hash[String, T.untyped])) }
+      def ownership_for_version(version)
+        if version.include?("+")
+          Dependabot.logger.info("Version #{version} includes a local version identifier, skipping ownership check")
+          return nil
+        end
+
+        possible_version_listing_urls(version).each do |url|
+          response = fetch_authed_url(url)
+          unless response.status == 200
+            Dependabot.logger.warn(
+              "Error fetching Python package ownership from #{url} for version #{version}: " \
+              "HTTP #{response.status}"
+            )
+            next
+          end
+
+          data = JSON.parse(response.body)
+          ownership = data["ownership"]
+          Dependabot.logger.debug("Found ownership for #{dependency.name} version #{version}")
+          return ownership
+        rescue JSON::ParserError, Excon::Error::Timeout, Excon::Error::Socket,
+               Excon::Error::TooManyRedirects, OpenSSL::SSL::SSLError, ArgumentError => e
+          Dependabot.logger.warn(
+            "Error fetching Python package ownership from #{url} for version #{version}: #{e.class}: #{e.message}"
+          )
+          next
+        end
+
+        nil
+      end
+
+      sig { params(version: String).returns(T::Array[String]) }
+      def possible_version_listing_urls(version)
+        possible_listing_urls.map do |url|
+          url.sub(%r{/json$}, "/#{URI::DEFAULT_PARSER.escape(version)}/json")
+        end
+      end
+
+      sig { params(ownership: T::Hash[String, T.untyped]).returns(T::Array[String]) }
+      def ownership_users(ownership)
+        roles = ownership["roles"]
+        return [] unless roles.is_a?(Array)
+
+        roles.filter_map { |role| role["user"] if role.is_a?(Hash) }
+      end
+
       # Strip [extras] from name (dependency_name[extra_dep,other_extra])
       sig { returns(String) }
       def normalised_dependency_name
         NameNormaliser.normalise(dependency.name)
       end
+
+      sig { params(url: T.nilable(String)).returns(T.nilable(Dependabot::Source)) }
+      def parsed_source_from_url(url)
+        return unless url
+        return @parsed_source_urls[url] if @parsed_source_urls.key?(url)
+
+        @parsed_source_urls[url] = Source.from_url(url)
+      end
     end
   end
 end

data/lib/dependabot/python/shared_file_fetcher.rb

@@ -238,7 +238,8 @@ module Dependabot
         content = file.content
         return [] if content.nil?
 
-        paths = content.scan(CHILD_REQUIREMENT_REGEX).flatten
+        paths = content.scan(CHILD_REQUIREMENT_REGEX).flatten +
+                content.scan(CONSTRAINT_REGEX).flatten
         current_dir = File.dirname(file.name)
 
         paths.flat_map do |path|
@@ -268,7 +269,8 @@ module Dependabot
       sig { returns(T::Array[Dependabot::DependencyFile]) }
       def constraints_files
         all_requirement_files = requirements_txt_files +
-                                child_requirement_txt_files
+                                child_requirement_txt_files +
+                                requirements_in_files
 
         constraints_paths = all_requirement_files.map do |req_file|
           current_dir = File.dirname(req_file.name)
@@ -283,7 +285,10 @@ module Dependabot
           end
         end.flatten.uniq
 
-        constraints_paths.map { |path| fetch_file_from_host(path) }
+        already_fetched_names = child_requirement_files.map(&:name)
+        constraints_paths
+          .reject { |path| already_fetched_names.include?(path) }
+          .map { |path| fetch_file_from_host(path) }
       end
 
       sig { returns(T::Array[Dependabot::DependencyFile]) }
@@ -355,7 +360,7 @@ module Dependabot
 
         uneditable_reqs =
           content
-          .scan(/(?<name>^['"]?(?:file:)?(?<path>\.[^\[#'"\n]*))/)
+          .scan(/(?<name>^['"]?(?:file:)?(?<path>\.[^\[#'"\n;]*))/)
           .filter_map do |match_array|
             n, p = match_array
             { name: n.to_s.strip, path: p.to_s.strip, file: req_file.name } unless p.to_s.include?("://")
@@ -363,7 +368,7 @@ module Dependabot
 
         editable_reqs =
           content
-          .scan(/(?<name>^-e\s+['"]?(?:file:)?(?<path>[^\[#'"\n]*))/)
+          .scan(/(?<name>^-e\s+['"]?(?:file:)?(?<path>[^\[#'"\n;]*))/)
           .filter_map do |match_array|
             n, p = match_array
             unless p.to_s.include?("://") || p.to_s.include?("git@")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-python
 version: !ruby/object:Gem::Version
-  version: 0.367.0
+  version: 0.368.0
 platform: ruby
 authors:
 - Dependabot
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.367.0
+        version: 0.368.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.367.0
+        version: 0.368.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -294,7 +294,7 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
-  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.367.0
+  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.368.0
 rdoc_options: []
 require_paths:
 - lib