dependabot-python 0.399.0 → 0.400.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e243e407de697945437e9818b4a451ab39ecb274ea6281b9ddcd2270d1bd16e7
-  data.tar.gz: 2ff42e6c9c205775d467942431d3c8aa51e2c4db9557b7bfbea733e2b201cfae
+  metadata.gz: ddc78c3d36f56fa3124806151a71bb16e46ccbd4fb002a814974c6d4b8fd2fad
+  data.tar.gz: b08192e8bc0a5b8c0125266571bd034029ed17e2c0f77a9d01350b7a9d30fca6
 SHA512:
-  metadata.gz: '0297caeec2ee5d809878d232089fabb5015966cb703bf59f3fcbdd295ca87908be3bffbe447d9c32b79b2945796f4c7d2a985574643496fe45e69116fb77a7fc'
-  data.tar.gz: 9f2c74c8fca0e487014943a033c030c67907c0f9135ae802bf4bd8d38dab8c876f18e0f05f98f7cae033906555ce0981c54b48671dd5d4d345e9b892576dda96
+  metadata.gz: 5062ffec97c3b221246787027d1afb91fc0da8aeba188cbf7be0f834448ec3ddf0d8ed25a5c7e18803bf986c8c0f6fea96cd4164ae6ecd81215fb4ffb0f75628
+  data.tar.gz: 932c74e3ca269747bcbd005fa0ac624d38a26bc60269953b7a8af99c03abf157b999b1d6ec5e30a64b7791a3c970aca267f964bb62dab880a083fb4021a8cbc9

data/lib/dependabot/python/dependency_grapher/requirements_layers.rb

@@ -129,7 +129,9 @@ module Dependabot
           end
 
           # When the primary is a compiled `.txt`, the `-r`/`-c` directives live in the paired `.in` files.
-          referenced = referenced_requirement_files([primary] + paired)
+          # Constraints files join every layer below, so they are seeded here too: a constraints file
+          # that references siblings needs those siblings on disk for the layer to parse.
+          referenced = referenced_requirement_files([primary] + paired + constraints_files)
 
           support = (paired + referenced + constraints_files).uniq.map do |f|
             as_support_file(f)

data/lib/dependabot/python/package/distribution.rb

@@ -0,0 +1,180 @@
+# typed: strong
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require "uri"
+require "cgi/escape"
+require "nokogiri"
+require "sorbet-runtime"
+require "dependabot/errors"
+
+module Dependabot
+  module Python
+    module Package
+      class Distribution < T::ImmutableStruct
+        extend T::Sig
+
+        ObjectHash = T.type_alias { T::Hash[String, Object] }
+
+        const :version_string, String
+        const :released_at, T.nilable(Time), default: nil
+        const :requires_python, T.nilable(String), default: nil
+        const :python_version, T.nilable(String), default: nil
+        const :yanked, T::Boolean, default: false
+        const :yanked_reason, T.nilable(String), default: nil
+        const :downloads, Integer, default: -1
+        const :url, T.nilable(String), default: nil
+        const :package_type, T.nilable(String), default: nil
+
+        sig { params(value: Object, version_string: String, context: String).returns(Distribution) }
+        def self.from_pypi(value, version_string:, context:)
+          fields = object(value, context)
+          new(
+            version_string: version_string,
+            released_at: timestamp(fields["upload_time"], "#{context}.upload_time"),
+            requires_python: optional_string(fields["requires_python"], "#{context}.requires_python"),
+            python_version: optional_string(fields["python_version"], "#{context}.python_version"),
+            yanked: boolean(fields["yanked"], "#{context}.yanked"),
+            yanked_reason: optional_string(fields["yanked_reason"], "#{context}.yanked_reason"),
+            downloads: downloads(fields["downloads"], "#{context}.downloads"),
+            url: optional_string(fields["url"], "#{context}.url"),
+            package_type: optional_string(fields["packagetype"], "#{context}.packagetype")
+          )
+        end
+
+        sig do
+          params(value: Object, version_string: String, context: String, project_url: String).returns(Distribution)
+        end
+        def self.from_simple(value, version_string:, context:, project_url:)
+          fields = object(value, context)
+          withdrawn = fields["yanked"]
+          yanked = withdrawn.is_a?(String) || boolean(withdrawn, "#{context}.yanked")
+          new(
+            version_string: version_string,
+            released_at: timestamp(fields["upload-time"], "#{context}.upload-time"),
+            requires_python: optional_string(fields["requires-python"], "#{context}.requires-python"),
+            yanked: yanked,
+            yanked_reason: withdrawn.is_a?(String) ? withdrawn : nil,
+            url: resolve_url(optional_string(fields["url"], "#{context}.url"), project_url, "#{context}.url")
+          )
+        end
+
+        sig do
+          params(
+            link: Nokogiri::XML::Node, version_string: String, context: String, project_url: String
+          ).returns(Distribution)
+        end
+        def self.from_html(link, version_string:, context:, project_url:)
+          requirement = optional_string(T.cast(link["data-requires-python"], Object), "#{context}.data-requires-python")
+          reason = optional_string(T.cast(link["data-yanked"], Object), "#{context}.data-yanked")
+          withdrawn = !T.cast(link.attribute("data-yanked"), Object).nil?
+          new(
+            version_string: version_string,
+            requires_python: requirement ? CGI.unescapeHTML(requirement) : nil,
+            yanked: withdrawn,
+            yanked_reason: reason,
+            url: resolve_url(
+              optional_string(T.cast(link["href"], Object), "#{context}.href"),
+              project_url,
+              "#{context}.href"
+            )
+          )
+        end
+
+        sig { params(format: String, source_url: String).returns(String) }
+        def self.context(format, source_url)
+          uri = URI.parse(source_url)
+          uri.userinfo = nil
+          uri.query = nil
+          uri.fragment = nil
+          "#{format} registry #{String(uri)}"
+        rescue URI::InvalidURIError
+          invalid(format, "registry URL is invalid")
+        end
+
+        sig { params(content: String, context: String).returns(Object) }
+        def self.parse_json(content, context)
+          T.cast(JSON.parse(content), Object)
+        rescue JSON::ParserError
+          invalid(context, "must contain valid JSON")
+        end
+
+        sig { params(value: Object, context: String).returns(ObjectHash) }
+        def self.object(value, context)
+          invalid(context, "must be an object") unless value.is_a?(Hash)
+
+          value.to_h do |raw_key, raw_value|
+            key = T.cast(raw_key, Object)
+            invalid(context, "keys must be strings") unless key.is_a?(String)
+
+            [key, T.cast(raw_value, Object)]
+          end
+        end
+
+        sig { params(value: Object, context: String).returns(T::Array[Object]) }
+        def self.array(value, context)
+          invalid(context, "must be an array") unless value.is_a?(Array)
+
+          value.map { |entry| T.cast(entry, Object) }
+        end
+
+        sig { params(value: Object, context: String).returns(T.nilable(String)) }
+        def self.optional_string(value, context)
+          return if value.nil?
+          return value if value.is_a?(String)
+
+          invalid(context, "must be a string or nil")
+        end
+
+        sig { params(context: String, message: String).returns(T.noreturn) }
+        def self.invalid(context, message)
+          raise DependencyFileNotResolvable.new("#{context} #{message}"), cause: nil
+        end
+
+        sig { params(value: Object, context: String).returns(T.nilable(Time)) }
+        def self.timestamp(value, context)
+          text = optional_string(value, context)
+          return if text.nil?
+
+          Time.parse(text)
+        rescue ArgumentError
+          invalid(context, "must be a valid timestamp")
+        end
+        private_class_method :timestamp
+
+        sig { params(value: Object, context: String).returns(T::Boolean) }
+        def self.boolean(value, context)
+          case value
+          when nil, false then false
+          when true then true
+          else invalid(context, "must be a boolean or nil")
+          end
+        end
+        private_class_method :boolean
+
+        sig { params(value: Object, context: String).returns(Integer) }
+        def self.downloads(value, context)
+          return -1 if value.nil?
+          return value if value.is_a?(Integer)
+
+          invalid(context, "must be an integer or nil")
+        end
+        private_class_method :downloads
+
+        sig { params(url: T.nilable(String), project_url: String, context: String).returns(T.nilable(String)) }
+        def self.resolve_url(url, project_url, context)
+          return if url.nil?
+
+          resolved = URI.join(project_url, url)
+          resolved.user = nil
+          resolved.password = nil
+          String(resolved)
+        rescue URI::InvalidURIError
+          invalid(context, "must be a valid URL")
+        end
+        private_class_method :resolve_url
+      end
+    end
+  end
+end

data/lib/dependabot/python/package/package_details_fetcher.rb

@@ -1,8 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "json"
-require "time"
 require "cgi/escape"
 require "excon"
 require "nokogiri"
@@ -13,6 +11,8 @@ require "dependabot/package/package_release"
 require "dependabot/package/package_details"
 require "dependabot/python/package/package_registry_finder"
 require "dependabot/python/package/simple_api_parser"
+require "dependabot/python/package/distribution"
+require "dependabot/python/package/pypi_json_parser"
 
 # Stores metadata for a package, including all its available versions
 module Dependabot
@@ -111,7 +111,7 @@ module Dependabot
             "Fetching release information from simple registry at #{sanitized_url(index_url)} for #{dependency.name}"
           )
           response = registry_response_for_dependency(index_url, accept: simple_api_accept)
-          project_url = response.path ? Excon::Utils.request_uri(response.data) : index_url + normalised_name + "/"
+          project_url = project_url_for(response, index_url)
           check_authentication_response(response, index_url)
 
           version_releases = if simple_json_response?(response)
@@ -120,14 +120,11 @@ module Dependabot
                                  project_url: project_url
                                ).parse(response.body)
                              else
-                               extract_release_details_json_from_html(response.body)
+                               distributions_from_html(response.body, project_url: project_url)
                              end
 
           format_version_releases(version_releases, preserve_distributions: simple_json_response?(response))
             .sort_by(&:version).reverse
-        rescue JSON::ParserError
-          Dependabot.logger.warn("JSON parsing error for #{sanitized_url(index_url)}.")
-          []
         end
 
         # Example JSON Response Format:
@@ -190,16 +187,13 @@ module Dependabot
           return nil unless response.status == 200
 
           begin
-            data = JSON.parse(response.body)
-
-            version_releases = data["releases"]
+            version_releases = PypiJsonParser.new(source_url: json_url).parse(response.body)
 
             releases = format_version_releases(version_releases)
 
             releases.sort_by(&:version).reverse
-          rescue JSON::ParserError
-            Dependabot.logger.warn("JSON parsing error for #{json_url}. Falling back to HTML.")
-            nil
+          rescue Dependabot::DependencyFileNotResolvable
+            raise
           rescue StandardError => e
             Dependabot.logger.warn("Unexpected error while fetching JSON data: #{e.message}.")
             nil
@@ -235,7 +229,9 @@ module Dependabot
           index_response = registry_response_for_dependency(index_url, accept: APPLICATION_TEXT)
           check_authentication_response(index_response, index_url)
 
-          version_releases = extract_release_details_json_from_html(index_response.body)
+          version_releases = distributions_from_html(
+            index_response.body, project_url: project_url_for(index_response, index_url)
+          )
           releases = format_version_releases(version_releases)
 
           releases.sort_by(&:version).reverse
@@ -262,102 +258,64 @@ module Dependabot
           raise PrivateSourceAuthenticationFailure, sanitized_url(index_url)
         end
 
-        sig do
-          params(html_body: String)
-            .returns(T::Hash[String, T::Array[T::Hash[String, T.untyped]]]) # Returns JSON-like format
+        sig { params(response: Excon::Response, index_url: String).returns(String) }
+        def project_url_for(response, index_url)
+          response.path ? Excon::Utils.request_uri(response.data) : index_url + normalised_name + "/"
         end
-        def extract_release_details_json_from_html(html_body)
-          doc = Nokogiri::HTML(html_body)
-
-          releases = {}
 
-          doc.css("a").each do |a_tag|
-            details = version_details_from_link(a_tag.to_s)
-            if details && details["version"]
-              releases[details["version"]] ||= []
-              releases[details["version"]] << details
-            end
+        sig { params(html_body: String, project_url: String).returns(T::Hash[String, T::Array[Distribution]]) }
+        def distributions_from_html(html_body, project_url:)
+          doc = Nokogiri::HTML(html_body)
+          context = Distribution.context("HTML index", project_url)
+          releases = T.let({}, T::Hash[String, T::Array[Distribution]])
+
+          doc.css("a").each_with_index do |link, index|
+            filename = link.content
+            url = link["href"]
+            next unless filename.match?(name_regex) || url&.match?(name_regex)
+
+            version = get_version_from_filename(filename)
+            next unless version && version_class.correct?(version)
+
+            distribution = Distribution.from_html(
+              link, version_string: version, context: "#{context}.links[#{index}]", project_url: project_url
+            )
+            releases[version] ||= []
+            T.must(releases[version]) << distribution
           end
 
           releases
         end
 
-        # rubocop:disable Metrics/PerceivedComplexity
-        sig do
-          params(link: T.nilable(String))
-            .returns(T.nilable(T::Hash[String, T.untyped]))
-        end
-        def version_details_from_link(link)
-          return unless link
-
-          doc = Nokogiri::XML(link)
-          filename = doc.at_css("a")&.content
-          url = doc.at_css("a")&.attributes&.fetch("href", nil)&.value
-
-          return unless filename&.match?(name_regex) || url&.match?(name_regex)
-
-          version = get_version_from_filename(filename)
-          return unless version_class.correct?(version)
-
-          {
-            "version" => version,
-            "requires_python" => requires_python_from_link(link),
-            "yanked" => link.include?("data-yanked"),
-            "url" => link
-          }
-        end
-        # rubocop:enable Metrics/PerceivedComplexity
-
         sig do
           params(
-            releases_json: T.nilable(T::Hash[String, T::Array[T::Hash[String, T.untyped]]]),
+            distributions: T::Hash[String, T::Array[Distribution]],
             preserve_distributions: T::Boolean
           )
             .returns(T::Array[Dependabot::Package::PackageRelease])
         end
-        def format_version_releases(releases_json, preserve_distributions: false)
-          return [] unless releases_json
-
-          releases_json.each_with_object([]) do |(version, release_data_array), versions|
-            release_data_array = [release_data_array.last].compact unless preserve_distributions
-            versions.concat(release_data_array.filter_map { |data| format_version_release(version, data) })
+        def format_version_releases(distributions, preserve_distributions: false)
+          distributions.each_value.flat_map do |files|
+            selected = preserve_distributions ? files : files.last(1)
+            selected.map { |distribution| format_version_release(distribution) }
           end
         end
 
-        sig do
-          params(
-            version: String,
-            release_data: T::Hash[String, T.untyped]
-          )
-            .returns(T.nilable(Dependabot::Package::PackageRelease))
-        end
-        def format_version_release(version, release_data)
-          # Skip versions that don't conform to PEP 440
-          unless Dependabot::Python::Version.correct?(version)
-            Dependabot.logger.warn("Skipping invalid version #{version}: does not match PEP 440")
-            return nil
-          end
-
-          upload_time = release_data["upload_time"]
-          released_at = Time.parse(upload_time) if upload_time
-          yanked = release_data["yanked"] || false
-          yanked_reason = release_data["yanked_reason"]
-          downloads = release_data["downloads"] || -1
-          url = release_data["url"]
-          package_type = release_data["packagetype"]
+        sig { params(distribution: Distribution).returns(Dependabot::Package::PackageRelease) }
+        def format_version_release(distribution)
           language = package_language(
-            python_version: release_data["python_version"],
-            requires_python: release_data["requires_python"]
+            python_version: distribution.python_version,
+            requires_python: distribution.requires_python
           )
 
           Dependabot::Package::PackageRelease.new(
-            version: Dependabot::Python::Version.new(version),
-            released_at: released_at,
-            yanked: yanked,
-            yanked_reason: yanked_reason,
-            downloads: downloads,
-            url: url,
-            package_type: package_type,
+            version: Dependabot::Python::Version.new(distribution.version_string),
+            released_at: distribution.released_at,
+            yanked: distribution.yanked,
+            yanked_reason: distribution.yanked_reason,
+            downloads: distribution.downloads,
+            url: distribution.url,
+            package_type: distribution.package_type,
             language: language
           )
         end
@@ -472,18 +430,6 @@ module Dependabot
           nil
         end
 
-        sig { params(link: String).returns(T.nilable(String)) }
-        def requires_python_from_link(link)
-          raw_value = Nokogiri::XML(link)
-                              .at_css("a")
-                              &.attribute("data-requires-python")
-                              &.content
-
-          return nil unless raw_value
-
-          CGI.unescapeHTML(raw_value) # Decodes HTML entities like &gt;=3 → >=3
-        end
-
         sig { returns(T.class_of(Dependabot::Version)) }
         def version_class
           dependency.version_class

data/lib/dependabot/python/package/pypi_json_parser.rb

@@ -0,0 +1,48 @@
+# typed: strong
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+require "dependabot/python/version"
+require "dependabot/python/package/distribution"
+
+module Dependabot
+  module Python
+    module Package
+      class PypiJsonParser
+        extend T::Sig
+
+        sig { params(source_url: String).void }
+        def initialize(source_url:)
+          @context = T.let(Distribution.context("PyPI JSON", source_url), String)
+        end
+
+        sig { params(content: String).returns(T::Hash[String, T::Array[Distribution]]) }
+        def parse(content)
+          fields = Distribution.object(Distribution.parse_json(content, @context), @context)
+          raw_releases = fields["releases"]
+          return {} if raw_releases.nil?
+
+          releases = Distribution.object(raw_releases, "#{@context}.releases")
+          parsed = T.let({}, T::Hash[String, T::Array[Distribution]])
+          releases.each_with_index do |(version, value), release_index|
+            location = "#{@context}.releases[#{release_index}]"
+            entries = Distribution.array(value, location)
+            next if entries.empty?
+
+            unless Python::Version.correct?(version)
+              Dependabot.logger.warn("Skipping invalid version #{version}: does not match PEP 440")
+              next
+            end
+
+            parsed[version] = entries.each_with_index.map do |entry, index|
+              Distribution.from_pypi(
+                entry, version_string: version, context: "#{location}[#{index}]"
+              )
+            end
+          end
+          parsed
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/package/simple_api_parser.rb

@@ -1,11 +1,10 @@
-# typed: strict
+# typed: strong
 # frozen_string_literal: true
 
-require "json"
-require "uri"
 require "sorbet-runtime"
 require "dependabot/errors"
 require "dependabot/python/name_normaliser"
+require "dependabot/python/package/distribution"
 
 module Dependabot
   module Python
@@ -13,8 +12,6 @@ module Dependabot
       class SimpleApiParser
         extend T::Sig
 
-        ReleaseDetail = T.type_alias { T::Hash[String, T.nilable(T.any(String, T::Boolean))] }
-
         sig { params(dependency: Dependabot::Dependency, project_url: String).void }
         def initialize(dependency:, project_url:)
           @dependency = dependency
@@ -23,31 +20,32 @@ module Dependabot
 
         sig do
           params(json_body: String)
-            .returns(T::Hash[String, T::Array[ReleaseDetail]])
+            .returns(T::Hash[String, T::Array[Distribution]])
         end
         def parse(json_body)
-          data = JSON.parse(json_body)
-          validate_api_version!(data.dig("meta", "api-version").to_s)
-
-          data.fetch("files", []).each_with_object({}) do |file, releases|
-            filename = file["filename"]
+          context = Distribution.context("Simple API JSON", project_url)
+          data = Distribution.object(Distribution.parse_json(json_body, context), context)
+          meta = data["meta"]
+          metadata = meta.nil? ? {} : Distribution.object(meta, "#{context}.meta")
+          validate_api_version!(metadata.fetch("api-version", "1.0"), context)
+
+          releases = T.let({}, T::Hash[String, T::Array[Distribution]])
+          Distribution.array(data.fetch("files", []), "#{context}.files").each_with_index do |entry, index|
+            location = "#{context}.files[#{index}]"
+            file = Distribution.object(entry, location)
+            filename = Distribution.optional_string(file["filename"], "#{location}.filename")
             next unless filename&.match?(name_regex)
 
             version = version_from_filename(filename)
-            next unless dependency.version_class.correct?(version)
+            next unless version && dependency.version_class.correct?(version)
 
-            yanked = file["yanked"] || false
-            details = {
-              "version" => version,
-              "requires_python" => file["requires-python"],
-              "yanked" => yanked != false,
-              "yanked_reason" => yanked.is_a?(String) ? yanked : nil,
-              "upload_time" => file["upload-time"],
-              "url" => resolve_url(file["url"])
-            }
+            distribution = Distribution.from_simple(
+              file, version_string: version, context: location, project_url: project_url
+            )
             releases[version] ||= []
-            releases[version] << details
+            T.must(releases[version]) << distribution
           end
+          releases
         end
 
         private
@@ -58,18 +56,11 @@ module Dependabot
         sig { returns(String) }
         attr_reader :project_url
 
-        sig { params(url: T.nilable(String)).returns(T.nilable(String)) }
-        def resolve_url(url)
-          return unless url
-
-          resolved_url = URI.join(project_url, url)
-          resolved_url.user = nil
-          resolved_url.password = nil
-          resolved_url.to_s
-        end
-
-        sig { params(api_version: String).void }
-        def validate_api_version!(api_version)
+        sig { params(api_version: Object, context: String).void }
+        def validate_api_version!(api_version, context)
+          unless api_version.is_a?(String) && api_version.match?(/\A[0-9]+\.[0-9]+\z/)
+            Distribution.invalid("#{context}.meta.api-version", "must be a Major.Minor string")
+          end
           return unless api_version.split(".").first.to_i > 1
 
           raise Dependabot::DependencyFileNotResolvable, "Unsupported PEP 691 API version: #{api_version}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-python
 version: !ruby/object:Gem::Version
-  version: 0.399.0
+  version: 0.400.0
 platform: ruby
 authors:
 - Dependabot
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.399.0
+        version: 0.400.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.399.0
+        version: 0.400.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -305,8 +305,10 @@ files:
 - lib/dependabot/python/metadata_finder.rb
 - lib/dependabot/python/name_normaliser.rb
 - lib/dependabot/python/native_helpers.rb
+- lib/dependabot/python/package/distribution.rb
 - lib/dependabot/python/package/package_details_fetcher.rb
 - lib/dependabot/python/package/package_registry_finder.rb
+- lib/dependabot/python/package/pypi_json_parser.rb
 - lib/dependabot/python/package/simple_api_parser.rb
 - lib/dependabot/python/package_manager.rb
 - lib/dependabot/python/pip_compile_file_matcher.rb
@@ -329,7 +331,7 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
-  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.399.0
+  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.400.0
 rdoc_options: []
 require_paths:
 - lib