dependabot-python 0.368.0 → 0.370.0

data/helpers/test/test_parser.py

@@ -0,0 +1,265 @@
+import json
+import os
+import sys
+
+# Add the helpers lib directory to the Python path so we can import parser
+sys.path.insert(
+    0, os.path.join(os.path.dirname(__file__), os.pardir, "lib")
+)
+
+from parser import parse_pep621_pep735_dependencies  # noqa: E402
+
+FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures")
+
+
+def parse(fixture_name):
+    path = os.path.join(FIXTURES, fixture_name)
+    result = json.loads(parse_pep621_pep735_dependencies(path))
+    return result["result"]
+
+
+def find_dep(deps, name):
+    return next((d for d in deps if d["name"] == name), None)
+
+
+# ---------------------------------------------------------------------------
+# PEP 621 project.dependencies
+# ---------------------------------------------------------------------------
+class TestPep621Dependencies:
+    def test_parses_runtime_dependencies(self):
+        deps = parse("pep621_dependencies.toml")
+        requests = find_dep(deps, "requests")
+        assert requests is not None
+        # packaging normalises specifier order alphabetically by operator
+        assert requests["requirement"] == "<3.0,>=2.13.0"
+        assert requests["requirement_type"] == "dependencies"
+
+    def test_parses_exact_version(self):
+        deps = parse("pep621_dependencies.toml")
+        urllib3 = find_dep(deps, "urllib3")
+        assert urllib3 is not None
+        assert urllib3["version"] == "1.26.0"
+        assert urllib3["requirement"] == "==1.26.0"
+
+    def test_parses_optional_dependencies(self):
+        deps = parse("pep621_dependencies.toml")
+        pysocks = find_dep(deps, "PySocks")
+        assert pysocks is not None
+        assert pysocks["requirement_type"] == "socks"
+
+    def test_optional_dependency_specifiers(self):
+        deps = parse("pep621_dependencies.toml")
+        pysocks = find_dep(deps, "PySocks")
+        # packaging normalises specifiers: sorted, no spaces
+        assert "!=" in pysocks["requirement"]
+        assert ">=" in pysocks["requirement"]
+
+    def test_parses_multiple_optional_groups(self):
+        deps = parse("pep621_dependencies.toml")
+        group_types = {d["requirement_type"] for d in deps}
+        assert "socks" in group_types
+        assert "tests" in group_types
+
+    def test_parses_build_system_requires(self):
+        deps = parse("pep621_dependencies.toml")
+        setuptools = find_dep(deps, "setuptools")
+        assert setuptools is not None
+        assert setuptools["requirement_type"] == "build-system.requires"
+        assert setuptools["requirement"] == ">=68.0"
+
+
+# ---------------------------------------------------------------------------
+# PEP 621 extras
+# ---------------------------------------------------------------------------
+class TestPep621Extras:
+    def test_parses_extras(self):
+        deps = parse("pep621_extras.toml")
+        cc = find_dep(deps, "cachecontrol")
+        assert cc is not None
+        assert cc["extras"] == ["filecache"]
+
+    def test_extras_requirement(self):
+        deps = parse("pep621_extras.toml")
+        cc = find_dep(deps, "cachecontrol")
+        assert cc["requirement"] == ">=0.14.0"
+
+
+# ---------------------------------------------------------------------------
+# PEP 735 dependency-groups
+# ---------------------------------------------------------------------------
+class TestPep735DependencyGroups:
+    def test_parses_dependency_group(self):
+        deps = parse("pep735_dependency_groups.toml")
+        pytest_dep = find_dep(deps, "pytest")
+        assert pytest_dep is not None
+        assert pytest_dep["requirement_type"] == "dev"
+        assert pytest_dep["version"] == "7.1.3"
+
+    def test_include_group_resolves(self):
+        deps = parse("pep735_dependency_groups.toml")
+        # "lint" group includes "dev" via include-group, plus flake8
+        lint_deps = [d for d in deps if d["requirement_type"] == "lint"]
+        lint_names = {d["name"] for d in lint_deps}
+        assert "flake8" in lint_names
+        # include-group pulls dev deps into lint but they keep their
+        # original group name, so they appear under "dev" requirement_type
+        all_names = {d["name"] for d in deps}
+        assert "pytest" in all_names
+        assert "black" in all_names
+
+    def test_include_group_lists_all_resolved_deps(self):
+        deps = parse("pep735_dependency_groups.toml")
+        # pytest appears twice: once from direct "dev" processing and
+        # once from "lint" including "dev" (both with requirement_type="dev"
+        # because included deps keep their original group name)
+        pytests = [d for d in deps if d["name"] == "pytest"]
+        assert len(pytests) == 2
+        assert all(d["requirement_type"] == "dev" for d in pytests)
+
+
+# ---------------------------------------------------------------------------
+# Markers
+# ---------------------------------------------------------------------------
+class TestMarkers:
+    def test_parses_markers(self):
+        deps = parse("pep621_markers.toml")
+        requests = find_dep(deps, "requests")
+        assert requests is not None
+        assert requests["markers"] == 'python_version >= "3.8"'
+
+    def test_requirement_with_markers(self):
+        deps = parse("pep621_markers.toml")
+        requests = find_dep(deps, "requests")
+        assert requests["requirement"] == ">=2.13.0"
+
+
+# ---------------------------------------------------------------------------
+# Edge cases
+# ---------------------------------------------------------------------------
+class TestEdgeCases:
+    def test_no_dependencies(self):
+        deps = parse("no_dependencies.toml")
+        assert deps == []
+
+    def test_only_build_system(self):
+        deps = parse("pep621_only_build_system.toml")
+        assert len(deps) == 2
+        names = {d["name"] for d in deps}
+        assert "setuptools" in names
+        assert "wheel" in names
+        assert all(
+            d["requirement_type"] == "build-system.requires" for d in deps
+        )
+
+    def test_empty_dependency_lists(self):
+        deps = parse("pep621_empty_deps.toml")
+        assert deps == []
+
+    def test_multiple_extras_on_single_dep(self):
+        deps = parse("pep621_multiple_extras.toml")
+        boto3 = find_dep(deps, "boto3")
+        assert boto3 is not None
+        assert boto3["extras"] == ["crt", "s3"]
+        assert boto3["requirement"] == ">=1.28.0"
+
+    def test_arbitrary_equality_operator(self):
+        deps = parse("pep621_arbitrary_equality.toml")
+        numpy = find_dep(deps, "numpy")
+        assert numpy is not None
+        assert numpy["version"] == "1.24.0rc1"
+        assert numpy["requirement"] == "===1.24.0rc1"
+
+    def test_no_version_specifier(self):
+        deps = parse("pep621_no_version.toml")
+        requests = find_dep(deps, "requests")
+        assert requests is not None
+        assert requests["version"] is None
+        assert requests["requirement"] == ""
+
+    def test_cyclic_include_group_does_not_loop(self):
+        deps = parse("pep735_cycle.toml")
+        names = [d["name"] for d in deps]
+        # Both groups should be processed without infinite recursion
+        assert "requests" in names
+        assert "flask" in names
+
+
+# ---------------------------------------------------------------------------
+# source_requirement — preserves original specifier string from the TOML
+# ---------------------------------------------------------------------------
+class TestSourceRequirement:
+    def test_simple_specifier_preserves_order(self):
+        deps = parse("pep621_dependencies.toml")
+        requests = find_dep(deps, "requests")
+        # Original: "requests>=2.13.0,<3.0" — order preserved
+        assert requests["source_requirement"] == ">=2.13.0,<3.0"
+        # Normalized by packaging: "<3.0,>=2.13.0" (alphabetical by operator)
+        assert requests["requirement"] == "<3.0,>=2.13.0"
+
+    def test_exact_version(self):
+        deps = parse("pep621_dependencies.toml")
+        urllib3 = find_dep(deps, "urllib3")
+        assert urllib3["source_requirement"] == "==1.26.0"
+
+    def test_extras_stripped_from_source(self):
+        deps = parse("pep621_extras.toml")
+        cc = find_dep(deps, "cachecontrol")
+        assert cc["source_requirement"] == ">=0.14.0"
+
+    def test_markers_stripped_from_source(self):
+        deps = parse("pep621_markers.toml")
+        requests = find_dep(deps, "requests")
+        assert requests["source_requirement"] == ">=2.13.0"
+
+    def test_no_version_specifier(self):
+        deps = parse("pep621_no_version.toml")
+        requests = find_dep(deps, "requests")
+        assert requests["source_requirement"] == ""
+
+    def test_multiple_extras_stripped(self):
+        deps = parse("pep621_multiple_extras.toml")
+        boto3 = find_dep(deps, "boto3")
+        assert boto3["source_requirement"] == ">=1.28.0"
+
+    def test_arbitrary_equality(self):
+        deps = parse("pep621_arbitrary_equality.toml")
+        numpy = find_dep(deps, "numpy")
+        assert numpy["source_requirement"] == "===1.24.0rc1"
+
+    def test_spaced_specifiers_preserved(self):
+        deps = parse("pep621_spaced_specifiers.toml")
+        requests = find_dep(deps, "requests")
+        # Spaces are preserved from the original entry
+        assert requests["source_requirement"] == ">= 2.13.0, < 3.0"
+
+    def test_spaced_exact_version_preserved(self):
+        deps = parse("pep621_spaced_specifiers.toml")
+        urllib3 = find_dep(deps, "urllib3")
+        assert urllib3["source_requirement"] == "== 1.26.0"
+
+    def test_spaced_extras_and_specifier(self):
+        deps = parse("pep621_spaced_specifiers.toml")
+        cc = find_dep(deps, "cachecontrol")
+        assert cc["source_requirement"] == ">= 0.14.0"
+
+    def test_spaced_specifier_with_markers(self):
+        deps = parse("pep621_spaced_specifiers.toml")
+        idna = find_dep(deps, "idna")
+        # Markers stripped, original spacing kept
+        assert idna["source_requirement"] == ">= 2.5"
+
+    def test_optional_deps_with_spaces(self):
+        deps = parse("pep621_dependencies.toml")
+        pysocks = find_dep(deps, "PySocks")
+        # Original: "PySocks >= 1.5.6, != 1.5.7, < 2"
+        assert pysocks["source_requirement"] == ">= 1.5.6, != 1.5.7, < 2"
+
+    def test_build_system_source_requirement(self):
+        deps = parse("pep621_dependencies.toml")
+        setuptools = find_dep(deps, "setuptools")
+        assert setuptools["source_requirement"] == ">=68.0"
+
+    def test_dependency_group_source_requirement(self):
+        deps = parse("pep735_dependency_groups.toml")
+        pytest_dep = find_dep(deps, "pytest")
+        assert pytest_dep["source_requirement"] == "==7.1.3"

data/helpers/test/test_run.py

@@ -0,0 +1,49 @@
+import json
+import os
+import subprocess
+import sys
+
+FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures")
+HELPERS_DIR = os.path.join(os.path.dirname(__file__), os.pardir)
+RUN_PY = os.path.join(HELPERS_DIR, "run.py")
+
+
+def run_helper(function, args):
+    input_json = json.dumps({"function": function, "args": args})
+    result = subprocess.run(
+        [sys.executable, RUN_PY],
+        input=input_json,
+        capture_output=True,
+        text=True,
+        cwd=HELPERS_DIR,
+    )
+    assert result.returncode == 0, (
+        f"run.py failed: {result.stderr}"
+    )
+    return json.loads(result.stdout)
+
+
+class TestRunRouting:
+    def test_parse_pep621_routing(self):
+        fixture = os.path.join(FIXTURES, "pep621_dependencies.toml")
+        result = run_helper("parse_pep621_pep735_dependencies", [fixture])
+
+        assert "result" in result
+        names = {d["name"] for d in result["result"]}
+        assert "requests" in names
+
+    def test_parse_setup_routing(self):
+        fixture_dir = os.path.join(FIXTURES, "setup_py")
+        result = run_helper("parse_setup", [fixture_dir])
+
+        assert "result" in result
+        names = {d["name"] for d in result["result"]}
+        assert "requests" in names
+
+    def test_parse_requirements_routing(self):
+        fixture_dir = os.path.join(FIXTURES, "requirements")
+        result = run_helper("parse_requirements", [fixture_dir])
+
+        assert "result" in result
+        names = {d["name"] for d in result["result"]}
+        assert "requests" in names

data/lib/dependabot/python/dependency_grapher/lockfile_generator.rb

@@ -7,6 +7,7 @@ require "dependabot/dependency_file"
 require "dependabot/shared_helpers"
 require "dependabot/python/file_parser/python_requirement_parser"
 require "dependabot/python/language_version_manager"
+require "dependabot/python/poetry_plugin_installer"
 
 module Dependabot
   module Python
@@ -33,6 +34,10 @@ module Dependabot
             SharedHelpers.with_git_configured(credentials: credentials) do
               write_temporary_files
               language_version_manager.install_required_python
+
+              # Install any required Poetry plugins declared in pyproject.toml
+              poetry_plugin_installer.install_required_plugins
+
               run_poetry_lock
               read_generated_lockfile
             end
@@ -119,6 +124,14 @@ module Dependabot
           )
         end
 
+        sig { returns(PoetryPluginInstaller) }
+        def poetry_plugin_installer
+          @poetry_plugin_installer ||= T.let(
+            PoetryPluginInstaller.from_dependency_files(dependency_files),
+            T.nilable(PoetryPluginInstaller)
+          )
+        end
+
         sig { params(error: SharedHelpers::HelperSubprocessFailed).void }
         def handle_generation_error(error)
           Dependabot.logger.error(

data/lib/dependabot/python/file_parser/pyproject_files_parser.rb

@@ -25,6 +25,7 @@ module Dependabot
         sig { params(dependency_files: T::Array[Dependabot::DependencyFile]).void }
         def initialize(dependency_files:)
           @dependency_files = dependency_files
+          @dynamic_fields = T.let(nil, T.nilable(T::Array[String]))
         end
 
         sig { returns(Dependabot::FileParsers::Base::DependencySet) }
@@ -84,16 +85,7 @@ module Dependabot
           return dependencies if using_pdm?
 
           parse_pep621_pep735_dependencies.each do |dep|
-            # If a requirement has a `<` or `<=` marker then updating it is
-            # probably blocked. Ignore it.
-            next if dep["markers"]&.include?("<")
-
-            # If no requirement, don't add it
-            next if dep["requirement"].empty?
-
-            # Skip build-system.requires dependencies when using Poetry
-            # Poetry manages its own build system dependencies
-            next if using_poetry? && dep["requirement_type"] == "build-system.requires"
+            next if skip_pep621_dep?(dep)
 
             dependencies <<
               Dependency.new(
@@ -106,7 +98,9 @@ module Dependabot
                   groups: [dep["requirement_type"]].compact
                 }],
                 package_manager: "pip",
-                metadata: extras_metadata(dep["extras"])
+                metadata: extras_metadata(dep["extras"]).merge(
+                  source_requirement: dep["source_requirement"]
+                ).compact
               )
           end
 
@@ -229,6 +223,43 @@ module Dependabot
           using_pep621? && pdm_lock
         end
 
+        sig { returns(T::Array[String]) }
+        def dynamic_fields
+          @dynamic_fields ||= parsed_pyproject.dig("project", "dynamic") || []
+        end
+
+        sig { params(dep: T::Hash[String, T.untyped]).returns(T::Boolean) }
+        def skip_pep621_dep?(dep)
+          # If a requirement has a `<` or `<=` marker then updating it is
+          # probably blocked. Ignore it.
+          return true if dep["markers"]&.include?("<")
+
+          # If no requirement, don't add it
+          return true if dep["requirement"].empty?
+
+          # Skip build-system.requires dependencies when using Poetry
+          # Poetry manages its own build system dependencies
+          return true if using_poetry? && dep["requirement_type"] == "build-system.requires"
+
+          # When dependencies or optional-dependencies are listed in project.dynamic,
+          # they are managed by the build backend (e.g. Poetry) — skip the PEP 621 path
+          dynamic_pep621_dep?(dep["requirement_type"])
+        end
+
+        sig { params(requirement_type: T.nilable(String)).returns(T::Boolean) }
+        def dynamic_pep621_dep?(requirement_type)
+          return false unless using_poetry?
+          return false unless requirement_type
+
+          if requirement_type == "dependencies"
+            dynamic_fields.include?("dependencies")
+          elsif parsed_pyproject.dig("project", "optional-dependencies")&.key?(requirement_type)
+            dynamic_fields.include?("optional-dependencies")
+          else
+            false
+          end
+        end
+
         # Create a DependencySet where each element has no requirement. Any
         # requirements will be added when combining the DependencySet with
         # other DependencySets.

data/lib/dependabot/python/file_parser.rb

@@ -1,6 +1,7 @@
 # typed: strict
 # frozen_string_literal: true
 
+require "toml-rb"
 require "dependabot/dependency"
 require "dependabot/file_parsers"
 require "dependabot/file_parsers/base"
@@ -110,7 +111,13 @@ module Dependabot
 
         return PipenvPackageManager.new(T.must(detect_pipenv_version)) if detect_pipenv_version
 
-        return PoetryPackageManager.new(T.must(detect_poetry_version)) if detect_poetry_version
+        poetry_version = detect_poetry_version
+        if poetry_version
+          return PoetryPackageManager.new(
+            poetry_version,
+            requires_poetry_version_constraint
+          )
+        end
 
         return PipCompilePackageManager.new(T.must(detect_pipcompile_version)) if detect_pipcompile_version
 
@@ -135,6 +142,19 @@ module Dependabot
         nil
       end
 
+      sig { returns(T.nilable(Dependabot::Python::Requirement)) }
+      def requires_poetry_version_constraint
+        return nil unless pyproject&.content
+
+        parsed = TomlRB.parse(T.must(pyproject).content)
+        constraint = parsed.dig("tool", "poetry", "requires-poetry")
+        return nil unless constraint.is_a?(String) && !constraint.strip.empty?
+
+        Dependabot::Python::Requirement.new(constraint.strip)
+      rescue TomlRB::ParseError, TomlRB::ValueOverwriteError, Gem::Requirement::BadRequirementError
+        nil
+      end
+
       # Detects the version of pip-compile. If the version cannot be detected, it returns nil
       sig { returns(T.nilable(String)) }
       def detect_pipcompile_version

data/lib/dependabot/python/file_updater/poetry_file_updater/pep621_updater.rb

@@ -0,0 +1,162 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+require "dependabot/python/file_updater/poetry_file_updater"
+
+module Dependabot
+  module Python
+    class FileUpdater
+      class PoetryFileUpdater
+        class Pep621Updater
+          extend T::Sig
+
+          sig { params(dep: Dependabot::Dependency).void }
+          def initialize(dep:)
+            @dep = dep
+          end
+
+          sig do
+            params(
+              content: String,
+              new_r: T::Hash[Symbol, T.untyped],
+              old_r: T::Hash[Symbol, T.untyped]
+            ).returns(T.nilable(String))
+          end
+          def replace(content, new_r, old_r)
+            source_req = dep.metadata[:source_requirement]
+
+            if source_req
+              replace_with_source_requirement(content, source_req, new_r, old_r)
+            else
+              replace_with_normalized_requirement(content, new_r, old_r)
+            end
+          end
+
+          sig { params(source_req: String, old_req: String, new_req: String).returns(String) }
+          def rewrite_pep508_requirement(source_req, old_req, new_req)
+            old_specifiers = parse_specifiers(old_req)
+            new_specifiers = parse_specifiers(new_req)
+
+            old_versions_by_op = group_versions_by_operator(old_specifiers)
+            new_versions_by_op = group_versions_by_operator(new_specifiers)
+
+            replacements = T.let([], T::Array[T::Hash[Symbol, String]])
+            new_versions_by_op.each do |operator, new_versions|
+              old_versions = old_versions_by_op[operator]
+              next unless old_versions
+              next unless old_versions.length == new_versions.length
+
+              old_versions.zip(new_versions).each do |old_version, new_version|
+                next if old_version == new_version
+
+                replacements << {
+                  operator: operator,
+                  old_version: old_version,
+                  new_version: T.must(new_version)
+                }
+              end
+            end
+
+            result = source_req.dup
+            replacements.each do |replacement|
+              op = Regexp.escape(T.must(replacement[:operator]))
+              ver = Regexp.escape(T.must(replacement[:old_version]))
+              result = result.sub(/(#{op}\s*)#{ver}/, "\\1#{replacement[:new_version]}")
+            end
+            result
+          end
+
+          sig { params(specifiers: T::Array[T::Hash[Symbol, String]]).returns(T::Hash[String, T::Array[String]]) }
+          def group_versions_by_operator(specifiers)
+            specifiers.each_with_object(
+              T.let({}, T::Hash[String, T::Array[String]])
+            ) do |specifier, grouped_versions|
+              operator = T.must(specifier[:operator])
+              version = T.must(specifier[:version])
+
+              grouped_versions[operator] ||= []
+              T.must(grouped_versions[operator]) << version
+            end
+          end
+
+          sig { params(req: String).returns(T::Array[T::Hash[Symbol, String]]) }
+          def parse_specifiers(req)
+            req.scan(/([!<>=~]+)\s*([^\s,]+)/).map do |op, ver|
+              { operator: op, version: ver }
+            end
+          end
+
+          private
+
+          sig { returns(Dependabot::Dependency) }
+          attr_reader :dep
+
+          sig do
+            params(
+              content: String,
+              source_req: String,
+              new_r: T::Hash[Symbol, T.untyped],
+              old_r: T::Hash[Symbol, T.untyped]
+            ).returns(T.nilable(String))
+          end
+          def replace_with_source_requirement(content, source_req, new_r, old_r)
+            match = content.match(declaration_regex(source_req))
+            return unless match
+
+            declaration = T.must(match[:declaration])
+            new_req_str = rewrite_pep508_requirement(source_req, old_r[:requirement], new_r[:requirement])
+            content.sub(declaration, declaration.sub(source_req, new_req_str))
+          end
+
+          # Fallback when source_requirement metadata is absent (e.g. after
+          # DependencySet merge or deserialization). Matches using the
+          # normalized requirement string, which may fail on whitespace
+          # differences but is better than skipping the update entirely.
+          sig do
+            params(
+              content: String,
+              new_r: T::Hash[Symbol, T.untyped],
+              old_r: T::Hash[Symbol, T.untyped]
+            ).returns(T.nilable(String))
+          end
+          def replace_with_normalized_requirement(content, new_r, old_r)
+            old_req = old_r[:requirement]
+            new_req = new_r[:requirement]
+
+            match = content.match(normalized_declaration_regex(old_req))
+            return unless match
+
+            declaration = T.must(match[:declaration])
+            return unless declaration.include?(old_req)
+
+            content.sub(declaration, declaration.sub(old_req, new_req))
+          end
+
+          sig { params(old_req: String).returns(Regexp) }
+          def declaration_regex(old_req)
+            /(?<declaration>["']#{escape}\s*#{extras_pattern}\s*#{Regexp.escape(old_req)}["'])/mi
+          end
+
+          sig { params(old_req: String).returns(Regexp) }
+          def normalized_declaration_regex(old_req)
+            /(?<declaration>["']#{escape}#{extras_pattern}#{Regexp.escape(old_req)}["'])/mi
+          end
+
+          sig { returns(String) }
+          def extras_pattern
+            extras_str = dep.metadata[:extras]
+            return "" unless extras_str.is_a?(String) && !extras_str.empty?
+
+            "\\[" + extras_str.split(",").map { |e| Regexp.escape(e.strip) }.join(",\\s*") + "\\]"
+          end
+
+          sig { returns(String) }
+          def escape
+            Regexp.escape(dep.name).gsub("\\-", "[-_.]")
+          end
+        end
+      end
+    end
+  end
+end