dependabot-python 0.367.0 → 0.369.0

data/helpers/test/test_parser.py

@@ -0,0 +1,184 @@
+import json
+import os
+import sys
+
+# Add the helpers lib directory to the Python path so we can import parser
+sys.path.insert(
+    0, os.path.join(os.path.dirname(__file__), os.pardir, "lib")
+)
+
+from parser import parse_pep621_pep735_dependencies  # noqa: E402
+
+FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures")
+
+
+def parse(fixture_name):
+    path = os.path.join(FIXTURES, fixture_name)
+    result = json.loads(parse_pep621_pep735_dependencies(path))
+    return result["result"]
+
+
+def find_dep(deps, name):
+    return next((d for d in deps if d["name"] == name), None)
+
+
+# ---------------------------------------------------------------------------
+# PEP 621 project.dependencies
+# ---------------------------------------------------------------------------
+class TestPep621Dependencies:
+    def test_parses_runtime_dependencies(self):
+        deps = parse("pep621_dependencies.toml")
+        requests = find_dep(deps, "requests")
+        assert requests is not None
+        # packaging normalises specifier order alphabetically by operator
+        assert requests["requirement"] == "<3.0,>=2.13.0"
+        assert requests["requirement_type"] == "dependencies"
+
+    def test_parses_exact_version(self):
+        deps = parse("pep621_dependencies.toml")
+        urllib3 = find_dep(deps, "urllib3")
+        assert urllib3 is not None
+        assert urllib3["version"] == "1.26.0"
+        assert urllib3["requirement"] == "==1.26.0"
+
+    def test_parses_optional_dependencies(self):
+        deps = parse("pep621_dependencies.toml")
+        pysocks = find_dep(deps, "PySocks")
+        assert pysocks is not None
+        assert pysocks["requirement_type"] == "socks"
+
+    def test_optional_dependency_specifiers(self):
+        deps = parse("pep621_dependencies.toml")
+        pysocks = find_dep(deps, "PySocks")
+        # packaging normalises specifiers: sorted, no spaces
+        assert "!=" in pysocks["requirement"]
+        assert ">=" in pysocks["requirement"]
+
+    def test_parses_multiple_optional_groups(self):
+        deps = parse("pep621_dependencies.toml")
+        group_types = {d["requirement_type"] for d in deps}
+        assert "socks" in group_types
+        assert "tests" in group_types
+
+    def test_parses_build_system_requires(self):
+        deps = parse("pep621_dependencies.toml")
+        setuptools = find_dep(deps, "setuptools")
+        assert setuptools is not None
+        assert setuptools["requirement_type"] == "build-system.requires"
+        assert setuptools["requirement"] == ">=68.0"
+
+
+# ---------------------------------------------------------------------------
+# PEP 621 extras
+# ---------------------------------------------------------------------------
+class TestPep621Extras:
+    def test_parses_extras(self):
+        deps = parse("pep621_extras.toml")
+        cc = find_dep(deps, "cachecontrol")
+        assert cc is not None
+        assert cc["extras"] == ["filecache"]
+
+    def test_extras_requirement(self):
+        deps = parse("pep621_extras.toml")
+        cc = find_dep(deps, "cachecontrol")
+        assert cc["requirement"] == ">=0.14.0"
+
+
+# ---------------------------------------------------------------------------
+# PEP 735 dependency-groups
+# ---------------------------------------------------------------------------
+class TestPep735DependencyGroups:
+    def test_parses_dependency_group(self):
+        deps = parse("pep735_dependency_groups.toml")
+        pytest_dep = find_dep(deps, "pytest")
+        assert pytest_dep is not None
+        assert pytest_dep["requirement_type"] == "dev"
+        assert pytest_dep["version"] == "7.1.3"
+
+    def test_include_group_resolves(self):
+        deps = parse("pep735_dependency_groups.toml")
+        # "lint" group includes "dev" via include-group, plus flake8
+        lint_deps = [d for d in deps if d["requirement_type"] == "lint"]
+        lint_names = {d["name"] for d in lint_deps}
+        assert "flake8" in lint_names
+        # include-group pulls dev deps into lint but they keep their
+        # original group name, so they appear under "dev" requirement_type
+        all_names = {d["name"] for d in deps}
+        assert "pytest" in all_names
+        assert "black" in all_names
+
+    def test_include_group_lists_all_resolved_deps(self):
+        deps = parse("pep735_dependency_groups.toml")
+        # pytest appears twice: once from direct "dev" processing and
+        # once from "lint" including "dev" (both with requirement_type="dev"
+        # because included deps keep their original group name)
+        pytests = [d for d in deps if d["name"] == "pytest"]
+        assert len(pytests) == 2
+        assert all(d["requirement_type"] == "dev" for d in pytests)
+
+
+# ---------------------------------------------------------------------------
+# Markers
+# ---------------------------------------------------------------------------
+class TestMarkers:
+    def test_parses_markers(self):
+        deps = parse("pep621_markers.toml")
+        requests = find_dep(deps, "requests")
+        assert requests is not None
+        assert requests["markers"] == 'python_version >= "3.8"'
+
+    def test_requirement_with_markers(self):
+        deps = parse("pep621_markers.toml")
+        requests = find_dep(deps, "requests")
+        assert requests["requirement"] == ">=2.13.0"
+
+
+# ---------------------------------------------------------------------------
+# Edge cases
+# ---------------------------------------------------------------------------
+class TestEdgeCases:
+    def test_no_dependencies(self):
+        deps = parse("no_dependencies.toml")
+        assert deps == []
+
+    def test_only_build_system(self):
+        deps = parse("pep621_only_build_system.toml")
+        assert len(deps) == 2
+        names = {d["name"] for d in deps}
+        assert "setuptools" in names
+        assert "wheel" in names
+        assert all(
+            d["requirement_type"] == "build-system.requires" for d in deps
+        )
+
+    def test_empty_dependency_lists(self):
+        deps = parse("pep621_empty_deps.toml")
+        assert deps == []
+
+    def test_multiple_extras_on_single_dep(self):
+        deps = parse("pep621_multiple_extras.toml")
+        boto3 = find_dep(deps, "boto3")
+        assert boto3 is not None
+        assert boto3["extras"] == ["crt", "s3"]
+        assert boto3["requirement"] == ">=1.28.0"
+
+    def test_arbitrary_equality_operator(self):
+        deps = parse("pep621_arbitrary_equality.toml")
+        numpy = find_dep(deps, "numpy")
+        assert numpy is not None
+        assert numpy["version"] == "1.24.0rc1"
+        assert numpy["requirement"] == "===1.24.0rc1"
+
+    def test_no_version_specifier(self):
+        deps = parse("pep621_no_version.toml")
+        requests = find_dep(deps, "requests")
+        assert requests is not None
+        assert requests["version"] is None
+        assert requests["requirement"] == ""
+
+    def test_cyclic_include_group_does_not_loop(self):
+        deps = parse("pep735_cycle.toml")
+        names = [d["name"] for d in deps]
+        # Both groups should be processed without infinite recursion
+        assert "requests" in names
+        assert "flask" in names

data/helpers/test/test_run.py

@@ -0,0 +1,49 @@
+import json
+import os
+import subprocess
+import sys
+
+FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures")
+HELPERS_DIR = os.path.join(os.path.dirname(__file__), os.pardir)
+RUN_PY = os.path.join(HELPERS_DIR, "run.py")
+
+
+def run_helper(function, args):
+    input_json = json.dumps({"function": function, "args": args})
+    result = subprocess.run(
+        [sys.executable, RUN_PY],
+        input=input_json,
+        capture_output=True,
+        text=True,
+        cwd=HELPERS_DIR,
+    )
+    assert result.returncode == 0, (
+        f"run.py failed: {result.stderr}"
+    )
+    return json.loads(result.stdout)
+
+
+class TestRunRouting:
+    def test_parse_pep621_routing(self):
+        fixture = os.path.join(FIXTURES, "pep621_dependencies.toml")
+        result = run_helper("parse_pep621_pep735_dependencies", [fixture])
+
+        assert "result" in result
+        names = {d["name"] for d in result["result"]}
+        assert "requests" in names
+
+    def test_parse_setup_routing(self):
+        fixture_dir = os.path.join(FIXTURES, "setup_py")
+        result = run_helper("parse_setup", [fixture_dir])
+
+        assert "result" in result
+        names = {d["name"] for d in result["result"]}
+        assert "requests" in names
+
+    def test_parse_requirements_routing(self):
+        fixture_dir = os.path.join(FIXTURES, "requirements")
+        result = run_helper("parse_requirements", [fixture_dir])
+
+        assert "result" in result
+        names = {d["name"] for d in result["result"]}
+        assert "requests" in names

data/lib/dependabot/python/file_updater/poetry_file_updater.rb

@@ -124,8 +124,10 @@ module Dependabot
           declaration_match = content.match(declaration_regex)
           if declaration_match
             declaration = declaration_match[:declaration]
-            new_declaration = T.must(declaration).sub(old_req, new_req)
-            return content.sub(T.must(declaration), new_declaration)
+            if T.must(declaration).include?(old_req)
+              new_declaration = T.must(declaration).sub(old_req, new_req)
+              return content.sub(T.must(declaration), new_declaration)
+            end
           end
 
           # Try Poetry table format
@@ -220,9 +222,8 @@ module Dependabot
 
         sig { params(pyproject_content: String).returns(String) }
         def freeze_other_dependencies(pyproject_content)
-          PyprojectPreparer
-            .new(pyproject_content: pyproject_content, lockfile: lockfile)
-            .freeze_top_level_dependencies_except(dependencies)
+          PyprojectPreparer.new(pyproject_content: pyproject_content, lockfile: lockfile)
+                           .freeze_top_level_dependencies_except(dependencies)
         end
 
         sig { params(pyproject_content: String).returns(String) }
@@ -244,14 +245,18 @@ module Dependabot
             end
           end
 
+          # Freeze PEP 621 project.dependencies and project.optional-dependencies
+          PyprojectPreparer.freeze_pep621_deps!(pyproject_object, dependencies) do |dep|
+            !git_dependency_being_updated?(dep)
+          end
+
           TomlRB.dump(pyproject_object)
         end
 
         sig { params(pyproject_content: String).returns(String) }
         def update_python_requirement(pyproject_content)
-          PyprojectPreparer
-            .new(pyproject_content: pyproject_content)
-            .update_python_requirement(language_version_manager.python_version)
+          PyprojectPreparer.new(pyproject_content: pyproject_content)
+                           .update_python_requirement(language_version_manager.python_version)
         end
 
         sig { params(poetry_object: T::Hash[String, T.untyped], dep: Dependabot::Dependency).returns(T::Array[String]) }
@@ -262,6 +267,8 @@ module Dependabot
             next unless pkg_name
 
             if poetry_object[type][pkg_name].is_a?(Hash)
+              next unless poetry_object[type][pkg_name].key?("version") # skip enrichment-only entries
+
               poetry_object[type][pkg_name]["version"] = dep.version
             else
               poetry_object[type][pkg_name] = dep.version
@@ -284,9 +291,7 @@ module Dependabot
 
         sig { params(pyproject_content: String).returns(String) }
         def sanitize(pyproject_content)
-          PyprojectPreparer
-            .new(pyproject_content: pyproject_content)
-            .sanitize
+          PyprojectPreparer.new(pyproject_content: pyproject_content).sanitize
         end
 
         sig { params(pyproject_content: String).returns(String) }

data/lib/dependabot/python/file_updater/pyproject_preparer.rb

@@ -16,6 +16,80 @@ module Dependabot
       class PyprojectPreparer
         extend T::Sig
 
+        # Builds a regex pattern that matches a PEP 508 package name,
+        # treating hyphens, underscores, and dots as interchangeable per PEP 508.
+        sig { params(name: String).returns(String) }
+        def self.pep508_name_pattern(name)
+          Regexp.escape(name).gsub("\\-", "[-_.]").gsub("_", "[-_.]").gsub("\\.", "[-_.]")
+        end
+        private_class_method :pep508_name_pattern
+
+        # Pins a single PEP 508 dependency entry string to a specific version,
+        # preserving extras and environment markers.
+        sig { params(entry: String, name_pattern: String, version: String).returns(String) }
+        def self.pin_pep508_entry(entry, name_pattern, version)
+          if entry.match?(/\A#{name_pattern}(\[.*?\])?\s*[><=!~]/i)
+            entry.sub(
+              /(?<pre>#{name_pattern}(?:\[.*?\])?)\s*[><=!~][^;]*?(?=\s*;|\s*\z)/i,
+              "\\k<pre>==#{version}"
+            )
+          else
+            entry.sub(
+              /(?<pre>#{name_pattern}(?:\[.*?\])?)(?<rest>\s*(?:;.*)?)/i,
+              "\\k<pre>==#{version}\\k<rest>"
+            )
+          end
+        end
+        private_class_method :pin_pep508_entry
+
+        # Freezes PEP 621 dependencies in-place within a parsed pyproject object.
+        # Replaces version specifiers with ==dep.version for each matching dep.
+        # Accepts an optional block to filter which dependencies to freeze.
+        sig do
+          params(
+            pyproject_object: T::Hash[String, T.untyped],
+            deps: T::Array[Dependabot::Dependency],
+            blk: T.nilable(T.proc.params(dep: Dependabot::Dependency).returns(T::Boolean))
+          ).void
+        end
+        def self.freeze_pep621_deps!(pyproject_object, deps, &blk)
+          dep_arrays = collect_pep621_dep_arrays(pyproject_object)
+          return if dep_arrays.empty?
+
+          deps.each do |dep|
+            next if blk && !yield(dep)
+            next unless dep.version
+
+            pin_pep621_dep_in_arrays!(dep_arrays, dep)
+          end
+        end
+
+        sig { params(pyproject_object: T::Hash[String, T.untyped]).returns(T::Array[T::Array[String]]) }
+        def self.collect_pep621_dep_arrays(pyproject_object)
+          project_object = pyproject_object["project"]
+          return [] unless project_object
+
+          dep_arrays = [project_object["dependencies"]]
+          project_object["optional-dependencies"]&.each_value { |opt| dep_arrays << opt }
+          dep_arrays.compact!
+          dep_arrays.select! { |arr| arr.is_a?(Array) && arr.all?(String) }
+          dep_arrays
+        end
+        private_class_method :collect_pep621_dep_arrays
+
+        sig { params(dep_arrays: T::Array[T::Array[String]], dep: Dependabot::Dependency).void }
+        def self.pin_pep621_dep_in_arrays!(dep_arrays, dep)
+          name_pattern = pep508_name_pattern(dep.name)
+          dep_arrays.each do |arr|
+            arr.each_with_index do |entry, i|
+              next unless entry.match?(/\A#{name_pattern}(\[.*?\])?\s*(\z|[><=!~;,])/i)
+
+              arr[i] = pin_pep508_entry(entry, name_pattern, T.must(dep.version))
+            end
+          end
+        end
+        private_class_method :pin_pep621_dep_in_arrays!
+
         sig { params(pyproject_content: String, lockfile: T.nilable(Dependabot::DependencyFile)).void }
         def initialize(pyproject_content:, lockfile: nil)
           @pyproject_content = pyproject_content
@@ -109,6 +183,9 @@ module Dependabot
             end
           end
 
+          # Freeze PEP 621 project.dependencies and project.optional-dependencies
+          freeze_pep621_top_level_deps!(pyproject_object, excluded_names)
+
           TomlRB.dump(pyproject_object)
         end
         # rubocop:enable Metrics/AbcSize
@@ -122,6 +199,38 @@ module Dependabot
         sig { returns(T.nilable(Dependabot::DependencyFile)) }
         attr_reader :lockfile
 
+        sig { params(pyproject_object: T::Hash[String, T.untyped], excluded_names: T::Array[String]).void }
+        def freeze_pep621_top_level_deps!(pyproject_object, excluded_names)
+          project_object = pyproject_object["project"]
+          return unless project_object
+
+          freeze_pep621_dep_array!(project_object["dependencies"], excluded_names)
+
+          project_object["optional-dependencies"]&.each_value do |opt_deps|
+            freeze_pep621_dep_array!(opt_deps, excluded_names)
+          end
+        end
+
+        sig { params(dep_array: T.nilable(T::Array[String]), excluded_names: T::Array[String]).void }
+        def freeze_pep621_dep_array!(dep_array, excluded_names)
+          return unless dep_array
+
+          dep_array.each_with_index do |entry, index|
+            # Extract dependency name from PEP 508 string
+            match = entry.match(/\A([a-zA-Z0-9](?:[a-zA-Z0-9._-]*[a-zA-Z0-9])?)/i)
+            next unless match
+
+            dep_name = normalise(T.must(match[1]))
+            next if excluded_names.include?(dep_name)
+
+            locked_details = locked_details(dep_name)
+            next unless (locked_version = locked_details&.fetch("version"))
+
+            name_pattern = self.class.send(:pep508_name_pattern, T.must(match[1]))
+            dep_array[index] = self.class.send(:pin_pep508_entry, entry, name_pattern, locked_version)
+          end
+        end
+
         sig { params(dep_name: String).returns(T.nilable(T::Hash[String, T.untyped])) }
         def locked_details(dep_name)
           parsed_lockfile.fetch("package")

data/lib/dependabot/python/file_updater/requirement_file_updater.rb

@@ -55,33 +55,56 @@ module Dependabot
 
         sig { returns(T::Array[Dependabot::DependencyFile]) }
         def fetch_updated_dependency_files
-          reqs = dependency.requirements.zip(dependency.previous_requirements || [])
+          updated_contents = T.let({}, T::Hash[String, String])
+
+          unique_requirement_changes.each do |pair|
+            new_req = T.must(pair[0])
+            old_req = pair[1]
+            filename = new_req.fetch(:file)
+            content = updated_contents[filename] || T.must(T.must(get_original_file(filename)).content)
+            updated_contents[filename] = updated_requirement_or_setup_file_content(content, new_req, old_req)
+          end
+
+          updated_contents.filter_map do |filename, content|
+            file = T.must(get_original_file(filename)).dup
+            next if content == T.must(file.content)
+
+            file.content = content
+            file
+          end
+        end
+
+        # Deduplicates requirements that share the same file and requirement strings.
+        # The replacer's regex matches all extras variants at once, so one call per
+        # unique (file, old_requirement, new_requirement) is sufficient.
+        sig { returns(T::Array[T::Array[T.nilable(T::Hash[Symbol, T.untyped])]]) }
+        def unique_requirement_changes
+          previous_reqs = dependency.previous_requirements || []
 
-          reqs.filter_map do |(new_req, old_req)|
+          changes = dependency.requirements.filter_map do |new_req|
+            old_req = previous_reqs.find { |r| r[:file] == new_req[:file] && r[:groups] == new_req[:groups] }
             next if new_req == old_req
 
-            file = T.must(get_original_file(new_req.fetch(:file))).dup
-            updated_content =
-              updated_requirement_or_setup_file_content(new_req, old_req)
-            next if updated_content == file.content
+            [new_req, old_req]
+          end
 
-            file.content = updated_content
-            file
+          changes.uniq do |pair|
+            new_req = pair[0]
+            old_req = pair[1]
+            [new_req[:file], old_req&.fetch(:requirement), new_req.fetch(:requirement)]
           end
         end
 
         sig do
           params(
+            content: String,
             new_req: T::Hash[Symbol, T.untyped],
             old_req: T.nilable(T::Hash[Symbol, T.untyped])
           ).returns(String)
         end
-        def updated_requirement_or_setup_file_content(new_req, old_req)
-          original_file = get_original_file(new_req.fetch(:file))
-          raise "Could not find a dependency file for #{new_req}" unless original_file
-
+        def updated_requirement_or_setup_file_content(content, new_req, old_req)
           RequirementReplacer.new(
-            content: T.must(original_file.content),
+            content: content,
             dependency_name: dependency.name,
             old_requirement: old_req&.fetch(:requirement),
             new_requirement: new_req.fetch(:requirement),

data/lib/dependabot/python/file_updater/requirement_replacer.rb

@@ -53,7 +53,7 @@ module Dependabot
               # ignore it, since it isn't actually a declaration
               next mtch if Regexp.last_match&.pre_match&.match?(/--.*\z/)
 
-              updated_dependency_declaration_string
+              updated_matched_declaration(mtch)
             end
 
           raise "Expected content to change!" if old_requirement != new_requirement && content == updated_content
@@ -98,29 +98,30 @@ module Dependabot
           new_req_string
         end
 
-        sig { returns(String) }
-        def updated_dependency_declaration_string
-          old_req = old_requirement
-          updated_string =
-            if old_req
-              original_dependency_declaration_string(old_req)
-                .sub(RequirementParser::REQUIREMENTS, updated_requirement_string || "")
-            else
-              original_dependency_declaration_string(old_req)
-                .sub(RequirementParser::NAME_WITH_EXTRAS) do |nm|
-                  nm + (updated_requirement_string || "")
-                end
-            end
-
-          return updated_string unless update_hashes? && requirement_includes_hashes?(old_req)
-
-          updated_string.sub(
+        # Builds updated declaration from the actual matched text, preserving
+        # whatever extras (or lack thereof) appeared in the original match.
+        sig { params(matched_declaration: String).returns(String) }
+        def updated_matched_declaration(matched_declaration)
+          updated = if old_requirement
+                      matched_declaration
+                        .sub(RequirementParser::REQUIREMENTS, updated_requirement_string || "")
+                    else
+                      matched_declaration
+                        .sub(RequirementParser::NAME_WITH_EXTRAS) { |nm| nm + (updated_requirement_string || "") }
+                    end
+
+          return updated unless update_hashes? && matched_declaration.match?(RequirementParser::HASHES)
+
+          algorithm = T.must(matched_declaration.match(RequirementParser::HASHES))
+                       .named_captures.fetch("algorithm")
+          separator = hash_separator(old_requirement)
+          updated.sub(
             RequirementParser::HASHES,
             package_hashes_for(
               name: dependency_name,
               version: new_hash_version,
-              algorithm: hash_algorithm(old_req)
-            ).join(hash_separator(old_req))
+              algorithm: algorithm
+            ).join(separator)
           )
         end
 
@@ -142,7 +143,14 @@ module Dependabot
         def original_declaration_replacement_regex
           original_string =
             original_dependency_declaration_string(old_requirement)
-          /(?<![\-\w\.\[])#{Regexp.escape(original_string)}(?![\-\w\.])/
+          match_data = T.must(original_string.match(RequirementParser::NAME_WITH_EXTRAS))
+          name_escaped = Regexp.escape(T.must(match_data[:name]))
+          # Everything after name+extras (the requirement/markers/hashes portion)
+          after_name_extras = T.must(original_string[T.must(match_data[0]).length..]).strip
+          after_escaped = Regexp.escape(after_name_extras)
+          # Match the dependency name with any extras (or none), followed by the requirement.
+          # This ensures a single gsub handles all extras variants of the same dependency.
+          /(?<![\-\w\.\[])#{name_escaped}\s*\\?\s*(?:\[[^\]]*\])?\s*\\?\s*#{after_escaped}(?![\-\w\.])/
         end
 
         sig { params(requirement: T.nilable(String)).returns(T::Boolean) }