dependabot-python 0.236.0 → 0.238.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9f6ef3950d4558611300b72d3839af555ea94e389aeb114a1257290901419e8
-  data.tar.gz: c896fc1ec74464344a70a3bf00add5181424b599c4553bf1d8dcb1e9708d8fcc
+  metadata.gz: b05bd36f835c00c6533153183b23d848d6b06759d20dc4e643cc2fd5e9a8c5e6
+  data.tar.gz: 0c08d7fdb367c16636cd558dca7ad36e16bba8851b4f906ac076d8a28b45bbdb
 SHA512:
-  metadata.gz: 0c25f7375610f6be67b98731d756af8c16b551971dfef4661f59e48fd3b8cfb09ac962b60458bcb99ddcc40c58a6246c5e95062eb9d3724d0d7d93be5860e71a
-  data.tar.gz: 54538b41299885f7187c6c3fe8d2967eb2bf6001fdc18dfb897cc1b70841060de2b48f834e480406d4e8c595c5897d734856e549e9cfe04d820a66dee8e39c09
+  metadata.gz: 1a7faaff71e67be34ad8d152bb5f869313f69a7a031cf4ffe4f1b2e5cc565e40ca0d9e9e5e2c470ff5477b5ab8c166ac29fc2323ac2c7548ec7e684348fac98b
+  data.tar.gz: 78adf7610fff0b22a46ac7d105a6463e1040505de29ecb15e9b087930bdfe5ba54cf6755bdefbff610a9009ae6a6bac6ae59997cae050c9dae06cd31ec1eda0d

data/helpers/requirements.txt

@@ -1,10 +1,10 @@
-pip==23.2.1
+pip==23.3.1
 pip-tools==7.3.0
 flake8==6.1.0
 hashin==0.17.0
-pipenv==2023.8.28
+pipenv@git+https://github.com/pypa/pipenv@main
 pipfile==0.0.2
-poetry==1.6.1
+poetry==1.7.1
 
 # Some dependencies will only install if Cython is present
-Cython==3.0.3
+Cython==3.0.5

data/lib/dependabot/python/file_fetcher.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "toml-rb"
+require "sorbet-runtime"
 
 require "dependabot/file_fetchers"
 require "dependabot/file_fetchers/base"
@@ -15,6 +16,9 @@ require "dependabot/errors"
 module Dependabot
   module Python
     class FileFetcher < Dependabot::FileFetchers::Base
+      extend T::Sig
+      extend T::Helpers
+
       CHILD_REQUIREMENT_REGEX = /^-r\s?(?<path>.*\.(?:txt|in))/
       CONSTRAINT_REGEX = /^-c\s?(?<path>.*\.(?:txt|in))/
       DEPENDENCY_TYPES = %w(packages dev-packages).freeze
@@ -63,8 +67,7 @@ module Dependabot
         }
       end
 
-      private
-
+      sig { override.returns(T::Array[DependencyFile]) }
       def fetch_files
         fetched_files = []
 
@@ -84,6 +87,8 @@ module Dependabot
         uniq_files(fetched_files)
       end
 
+      private
+
       def uniq_files(fetched_files)
         uniq_files = fetched_files.reject(&:support_file?).uniq
         uniq_files += fetched_files

data/lib/dependabot/python/file_parser/pipfile_files_parser.rb

@@ -67,7 +67,8 @@ module Dependabot
                     source: nil,
                     groups: [group]
                   }],
-                  package_manager: "pip"
+                  package_manager: "pip",
+                  metadata: { original_name: dep_name }
                 )
             end
           end

data/lib/dependabot/python/file_parser/pyproject_files_parser.rb

@@ -137,12 +137,21 @@ module Dependabot
 
             check_requirements(requirement)
 
-            {
-              requirement: requirement.is_a?(String) ? requirement : requirement["version"],
-              file: pyproject.name,
-              source: nil,
-              groups: [type]
-            }
+            if requirement.is_a?(String)
+              {
+                requirement: requirement,
+                file: pyproject.name,
+                source: nil,
+                groups: [type]
+              }
+            else
+              {
+                requirement: requirement["version"],
+                file: pyproject.name,
+                source: requirement.fetch("source", nil),
+                groups: [type]
+              }
+            end
           end
         end
 

data/lib/dependabot/python/file_parser/python_requirement_parser.rb

@@ -6,6 +6,7 @@ require "open3"
 require "dependabot/errors"
 require "dependabot/shared_helpers"
 require "dependabot/python/file_parser"
+require "dependabot/python/pip_compile_file_matcher"
 require "dependabot/python/requirement"
 
 module Dependabot
@@ -22,6 +23,7 @@ module Dependabot
           [
             pipfile_python_requirement,
             pyproject_python_requirement,
+            pip_compile_python_requirement,
             python_version_file_version,
             runtime_file_python_version,
             setup_file_requirement
@@ -64,6 +66,20 @@ module Dependabot
             poetry_object&.dig("dev-dependencies", "python")
         end
 
+        def pip_compile_python_requirement
+          requirement_files.each do |file|
+            next unless pip_compile_file_matcher.lockfile_for_pip_compile_file?(file)
+
+            marker = /^# This file is autogenerated by pip-compile with [pP]ython (?<version>\d+.\d+)$/m
+            match = marker.match(file.content)
+            next unless match
+
+            return match[:version]
+          end
+
+          nil
+        end
+
         def python_version_file_version
           return unless python_version_file
 
@@ -106,6 +122,10 @@ module Dependabot
           SharedHelpers.run_shell_command(command, env: env, stderr_to_stdout: true)
         end
 
+        def pip_compile_file_matcher
+          @pip_compile_file_matcher ||= PipCompileFileMatcher.new(pip_compile_files)
+        end
+
         def requirement_class
           Dependabot::Python::Requirement
         end
@@ -144,6 +164,10 @@ module Dependabot
         def requirement_files
           dependency_files.select { |f| f.name.end_with?(".txt") }
         end
+
+        def pip_compile_files
+          dependency_files.select { |f| f.name.end_with?(".in") }
+        end
       end
     end
   end

data/lib/dependabot/python/file_updater/pipfile_file_updater.rb

@@ -1,7 +1,6 @@
 # typed: true
 # frozen_string_literal: true
 
-require "toml-rb"
 require "open3"
 require "dependabot/dependency"
 require "dependabot/python/requirement_parser"
@@ -10,7 +9,7 @@ require "dependabot/python/file_updater"
 require "dependabot/python/language_version_manager"
 require "dependabot/shared_helpers"
 require "dependabot/python/native_helpers"
-require "dependabot/python/name_normaliser"
+require "dependabot/python/pipenv_runner"
 
 module Dependabot
   module Python
@@ -22,12 +21,13 @@ module Dependabot
 
         DEPENDENCY_TYPES = %w(packages dev-packages).freeze
 
-        attr_reader :dependencies, :dependency_files, :credentials
+        attr_reader :dependencies, :dependency_files, :credentials, :repo_contents_path
 
-        def initialize(dependencies:, dependency_files:, credentials:)
+        def initialize(dependencies:, dependency_files:, credentials:, repo_contents_path:)
           @dependencies = dependencies
           @dependency_files = dependency_files
           @credentials = credentials
+          @repo_contents_path = repo_contents_path
         end
 
         def updated_dependency_files
@@ -83,7 +83,6 @@ module Dependabot
           return [] unless lockfile
 
           pipfile_lock_deps = parsed_lockfile[type]&.keys&.sort || []
-          pipfile_lock_deps = pipfile_lock_deps.map { |n| normalise(n) }
           return [] unless pipfile_lock_deps.any?
 
           regex = RequirementParser::INSTALL_REQ_WITH_REQUIREMENT
@@ -94,7 +93,7 @@ module Dependabot
           requirements_files.select do |req_file|
             deps = []
             req_file.content.scan(regex) { deps << Regexp.last_match }
-            deps = deps.map { |m| normalise(m[:name]) }
+            deps = deps.map { |m| m[:name] }
             deps.sort == pipfile_lock_deps
           end
         end
@@ -129,61 +128,17 @@ module Dependabot
 
         def prepared_pipfile_content
           content = updated_pipfile_content
-          content = freeze_other_dependencies(content)
-          content = freeze_dependencies_being_updated(content)
           content = add_private_sources(content)
           content = update_python_requirement(content)
           content
         end
 
-        def freeze_other_dependencies(pipfile_content)
-          PipfilePreparer
-            .new(pipfile_content: pipfile_content, lockfile: lockfile)
-            .freeze_top_level_dependencies_except(dependencies)
-        end
-
         def update_python_requirement(pipfile_content)
           PipfilePreparer
             .new(pipfile_content: pipfile_content)
             .update_python_requirement(language_version_manager.python_major_minor)
         end
 
-        # rubocop:disable Metrics/PerceivedComplexity
-        def freeze_dependencies_being_updated(pipfile_content)
-          pipfile_object = TomlRB.parse(pipfile_content)
-
-          dependencies.each do |dep|
-            DEPENDENCY_TYPES.each do |type|
-              names = pipfile_object[type]&.keys || []
-              pkg_name = names.find { |nm| normalise(nm) == dep.name }
-              next unless pkg_name || subdep_type?(type)
-
-              pkg_name ||= dependency.name
-              if pipfile_object[type][pkg_name].is_a?(Hash)
-                pipfile_object[type][pkg_name]["version"] =
-                  "==#{dep.version}"
-              else
-                pipfile_object[type][pkg_name] = "==#{dep.version}"
-              end
-            end
-          end
-
-          TomlRB.dump(pipfile_object)
-        end
-        # rubocop:enable Metrics/PerceivedComplexity
-
-        def subdep_type?(type)
-          return false if dependency.top_level?
-
-          lockfile_type = Python::FileParser::DEPENDENCY_GROUP_KEYS
-                          .find { |i| i.fetch(:pipfile) == type }
-                          .fetch(:lockfile)
-
-          JSON.parse(lockfile.content)
-              .fetch(lockfile_type, {})
-              .keys.any? { |k| normalise(k) == dependency.name }
-        end
-
         def add_private_sources(pipfile_content)
           PipfilePreparer
             .new(pipfile_content: pipfile_content)
@@ -192,14 +147,12 @@ module Dependabot
 
         def updated_generated_files
           @updated_generated_files ||=
-            SharedHelpers.in_a_temporary_directory do
+            SharedHelpers.in_a_temporary_repo_directory(dependency_files.first.directory, repo_contents_path) do
               SharedHelpers.with_git_configured(credentials: credentials) do
                 write_temporary_dependency_files(prepared_pipfile_content)
                 install_required_python
 
-                run_pipenv_command(
-                  "pyenv exec pipenv lock"
-                )
+                pipenv_runner.run_upgrade("==#{dependency.version}")
 
                 result = { lockfile: File.read("Pipfile.lock") }
                 result[:lockfile] = post_process_lockfile(result[:lockfile])
@@ -245,17 +198,12 @@ module Dependabot
           File.write("dev-req.txt", dev_req_content)
         end
 
-        def run_command(command, env: {}, fingerprint: nil)
-          SharedHelpers.run_shell_command(command, env: env, fingerprint: fingerprint)
+        def run_command(command)
+          SharedHelpers.run_shell_command(command)
         end
 
-        def run_pipenv_command(command, env: pipenv_env_variables)
-          run_command(
-            "pyenv local #{language_version_manager.python_major_minor}",
-            fingerprint: "pyenv local <python_major_minor>"
-          )
-
-          run_command(command, env: env)
+        def run_pipenv_command(command)
+          pipenv_runner.run(command)
         end
 
         def write_temporary_dependency_files(pipfile_content)
@@ -317,7 +265,7 @@ module Dependabot
             SharedHelpers.run_helper_subprocess(
               command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
               function: "get_pipfile_hash",
-              args: [dir]
+              args: [T.cast(dir, Pathname).to_s]
             )
           end
         end
@@ -328,10 +276,6 @@ module Dependabot
           updated_file
         end
 
-        def normalise(name)
-          NameNormaliser.normalise(name)
-        end
-
         def python_requirement_parser
           @python_requirement_parser ||=
             FileParser::PythonRequirementParser.new(
@@ -346,6 +290,15 @@ module Dependabot
             )
         end
 
+        def pipenv_runner
+          @pipenv_runner ||=
+            PipenvRunner.new(
+              dependency: dependency,
+              lockfile: lockfile,
+              language_version_manager: language_version_manager
+            )
+        end
+
         def parsed_lockfile
           @parsed_lockfile ||= JSON.parse(lockfile.content)
         end
@@ -369,16 +322,6 @@ module Dependabot
         def requirements_files
           dependency_files.select { |f| f.name.end_with?(".txt") }
         end
-
-        def pipenv_env_variables
-          {
-            "PIPENV_YES" => "true",       # Install new Python ver if needed
-            "PIPENV_MAX_RETRIES" => "3",  # Retry timeouts
-            "PIPENV_NOSPIN" => "1",       # Don't pollute logs with spinner
-            "PIPENV_TIMEOUT" => "600",    # Set install timeout to 10 minutes
-            "PIP_DEFAULT_TIMEOUT" => "60" # Set pip timeout to 1 minute
-          }
-        end
       end
     end
   end

data/lib/dependabot/python/file_updater/pipfile_preparer.rb

@@ -7,15 +7,13 @@ require "dependabot/dependency"
 require "dependabot/python/file_parser"
 require "dependabot/python/file_updater"
 require "dependabot/python/authed_url_builder"
-require "dependabot/python/name_normaliser"
 
 module Dependabot
   module Python
     class FileUpdater
       class PipfilePreparer
-        def initialize(pipfile_content:, lockfile: nil)
+        def initialize(pipfile_content:)
           @pipfile_content = pipfile_content
-          @lockfile = lockfile
         end
 
         def replace_sources(credentials)
@@ -28,45 +26,6 @@ module Dependabot
           TomlRB.dump(pipfile_object)
         end
 
-        def freeze_top_level_dependencies_except(dependencies)
-          return pipfile_content unless lockfile
-
-          pipfile_object = TomlRB.parse(pipfile_content)
-          excluded_names = dependencies.map(&:name)
-
-          Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys|
-            next unless pipfile_object[keys[:pipfile]]
-
-            pipfile_object.fetch(keys[:pipfile]).each do |dep_name, _|
-              next if excluded_names.include?(normalise(dep_name))
-
-              freeze_dependency(dep_name, pipfile_object, keys)
-            end
-          end
-
-          TomlRB.dump(pipfile_object)
-        end
-
-        def freeze_dependency(dep_name, pipfile_object, keys)
-          locked_version = version_from_lockfile(
-            keys[:lockfile],
-            normalise(dep_name)
-          )
-          locked_ref = ref_from_lockfile(
-            keys[:lockfile],
-            normalise(dep_name)
-          )
-
-          pipfile_req = pipfile_object[keys[:pipfile]][dep_name]
-          if pipfile_req.is_a?(Hash) && locked_version
-            pipfile_req["version"] = "==#{locked_version}"
-          elsif pipfile_req.is_a?(Hash) && locked_ref && !pipfile_req["ref"]
-            pipfile_req["ref"] = locked_ref
-          elsif locked_version
-            pipfile_object[keys[:pipfile]][dep_name] = "==#{locked_version}"
-          end
-        end
-
         def update_python_requirement(requirement)
           pipfile_object = TomlRB.parse(pipfile_content)
 
@@ -84,31 +43,6 @@ module Dependabot
 
         attr_reader :pipfile_content, :lockfile
 
-        def version_from_lockfile(dep_type, dep_name)
-          details = parsed_lockfile.dig(dep_type, normalise(dep_name))
-
-          case details
-          when String then details.gsub(/^==/, "")
-          when Hash then details["version"]&.gsub(/^==/, "")
-          end
-        end
-
-        def ref_from_lockfile(dep_type, dep_name)
-          details = parsed_lockfile.dig(dep_type, normalise(dep_name))
-
-          case details
-          when Hash then details["ref"]
-          end
-        end
-
-        def parsed_lockfile
-          @parsed_lockfile ||= JSON.parse(lockfile.content)
-        end
-
-        def normalise(name)
-          NameNormaliser.normalise(name)
-        end
-
         def pipfile_sources
           @pipfile_sources ||= TomlRB.parse(pipfile_content).fetch("source", [])
         end

data/lib/dependabot/python/file_updater/poetry_file_updater.rb

@@ -238,7 +238,7 @@ module Dependabot
               SharedHelpers.run_helper_subprocess(
                 command: "pyenv exec python3 #{python_helper_path}",
                 function: "get_pyproject_hash",
-                args: [dir]
+                args: [T.cast(dir, Pathname).to_s]
               )
             end
           end
@@ -247,7 +247,8 @@ module Dependabot
         def declaration_regex(dep, old_req)
           group = old_req[:groups].first
 
-          /#{group}(?:\.dependencies)?\]\s*\n.*?(?<declaration>(?:^\s*|["'])#{escape(dep)}["']?\s*=[^\n]*)$/mi
+          header_regex = "#{group}(?:\\.dependencies)?\\]\s*(?:\s*#.*?)*?"
+          /#{header_regex}\n.*?(?<declaration>(?:^\s*|["'])#{escape(dep)}["']?\s*=[^\n]*)$/mi
         end
 
         def table_declaration_regex(dep, old_req)

data/lib/dependabot/python/file_updater.rb

@@ -88,7 +88,8 @@ module Dependabot
         PipfileFileUpdater.new(
           dependencies: dependencies,
           dependency_files: dependency_files,
-          credentials: credentials
+          credentials: credentials,
+          repo_contents_path: repo_contents_path
         ).updated_dependency_files
       end
 

data/lib/dependabot/python/pipenv_runner.rb

@@ -0,0 +1,84 @@
+# typed: true
+# frozen_string_literal: true
+
+require "dependabot/shared_helpers"
+require "dependabot/python/file_parser"
+require "json"
+
+module Dependabot
+  module Python
+    class PipenvRunner
+      def initialize(dependency:, lockfile:, language_version_manager:)
+        @dependency = dependency
+        @lockfile = lockfile
+        @language_version_manager = language_version_manager
+      end
+
+      def run_upgrade(constraint)
+        constraint = "" if constraint == "*"
+        command = "pyenv exec pipenv upgrade --verbose #{dependency_name}#{constraint}"
+        command << " --dev" if lockfile_section == "develop"
+
+        run(command, fingerprint: "pyenv exec pipenv upgrade --verbose <dependency_name><constraint>")
+      end
+
+      def run_upgrade_and_fetch_version(constraint)
+        run_upgrade(constraint)
+
+        updated_lockfile = JSON.parse(File.read("Pipfile.lock"))
+
+        fetch_version_from_parsed_lockfile(updated_lockfile)
+      end
+
+      def run(command, fingerprint: nil)
+        run_command(
+          "pyenv local #{language_version_manager.python_major_minor}",
+          fingerprint: "pyenv local <python_major_minor>"
+        )
+
+        run_command(command, fingerprint: fingerprint)
+      end
+
+      private
+
+      attr_reader :dependency, :lockfile, :language_version_manager
+
+      def fetch_version_from_parsed_lockfile(updated_lockfile)
+        deps = updated_lockfile[lockfile_section] || {}
+
+        deps.dig(dependency_name, "version")
+            &.gsub(/^==/, "")
+      end
+
+      def run_command(command, fingerprint: nil)
+        SharedHelpers.run_shell_command(command, env: pipenv_env_variables, fingerprint: fingerprint)
+      end
+
+      def lockfile_section
+        if dependency.requirements.any?
+          dependency.requirements.first[:groups].first
+        else
+          Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys|
+            section = keys.fetch(:lockfile)
+            return section if JSON.parse(lockfile.content)[section].keys.any?(dependency_name)
+          end
+        end
+      end
+
+      def dependency_name
+        dependency.metadata[:original_name] || dependency.name
+      end
+
+      def pipenv_env_variables
+        {
+          "PIPENV_YES" => "true",        # Install new Python ver if needed
+          "PIPENV_MAX_RETRIES" => "3",   # Retry timeouts
+          "PIPENV_NOSPIN" => "1",        # Don't pollute logs with spinner
+          "PIPENV_TIMEOUT" => "600",     # Set install timeout to 10 minutes
+          "PIP_DEFAULT_TIMEOUT" => "60", # Set pip timeout to 1 minute
+          "COLUMNS" => "250"             # Avoid line wrapping
+        }
+      end
+    end
+  end
+end

data/lib/dependabot/python/update_checker/index_finder.rb

@@ -12,9 +12,10 @@ module Dependabot
         PYPI_BASE_URL = "https://pypi.org/simple/"
         ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/
 
-        def initialize(dependency_files:, credentials:)
+        def initialize(dependency_files:, credentials:, dependency:)
           @dependency_files = dependency_files
           @credentials      = credentials
+          @dependency       = dependency
         end
 
         def index_urls
@@ -122,9 +123,13 @@ module Dependabot
             # If source is PyPI, skip it, and let it pick the default URI
             next if source["name"].casecmp?("PyPI")
 
-            if source["default"]
+            if @dependency.all_sources.include?(source["name"])
+              # If dependency has specified this source, use it
+              return { main: source["url"], extra: [] }
+            elsif source["default"]
               urls[:main] = source["url"]
-            else
+            elsif source["priority"] != "explicit"
+              # if source is not explicit, add it to extra
               urls[:extra] << source["url"]
             end
           end

data/lib/dependabot/python/update_checker/latest_version_finder.rb

@@ -213,7 +213,8 @@ module Dependabot
           @index_urls ||=
             IndexFinder.new(
               dependency_files: dependency_files,
-              credentials: credentials
+              credentials: credentials,
+              dependency: dependency
             ).index_urls
         end
 

data/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb

@@ -33,12 +33,13 @@ module Dependabot
         RESOLUTION_IMPOSSIBLE_ERROR = "ResolutionImpossible"
         ERROR_REGEX = /(?<=ERROR\:\W).*$/
 
-        attr_reader :dependency, :dependency_files, :credentials
+        attr_reader :dependency, :dependency_files, :credentials, :repo_contents_path
 
-        def initialize(dependency:, dependency_files:, credentials:)
+        def initialize(dependency:, dependency_files:, credentials:, repo_contents_path:)
           @dependency               = dependency
           @dependency_files         = dependency_files
           @credentials              = credentials
+          @repo_contents_path       = repo_contents_path
           @build_isolation = true
         end
 

data/lib/dependabot/python/update_checker/pipenv_version_resolver.rb

@@ -1,8 +1,7 @@
-# typed: false
+# typed: true
 # frozen_string_literal: true
 
 require "excon"
-require "toml-rb"
 require "open3"
 require "dependabot/dependency"
 require "dependabot/errors"
@@ -13,23 +12,14 @@ require "dependabot/python/file_updater/pipfile_preparer"
 require "dependabot/python/file_updater/setup_file_sanitizer"
 require "dependabot/python/update_checker"
 require "dependabot/python/native_helpers"
-require "dependabot/python/name_normaliser"
+require "dependabot/python/pipenv_runner"
 require "dependabot/python/version"
 
 module Dependabot
   module Python
     class UpdateChecker
-      # This class does version resolution for Pipfiles. Its current approach
-      # is somewhat crude:
-      # - Unlock the dependency we're checking in the Pipfile
-      # - Freeze all of the other dependencies in the Pipfile
-      # - Run `pipenv lock` and see what the result is
-      #
-      # Unfortunately, Pipenv doesn't resolve how we'd expect - it appears to
-      # just raise if the latest version can't be resolved. Knowing that is
-      # still better than nothing, though.
       class PipenvVersionResolver
-        GIT_DEPENDENCY_UNREACHABLE_REGEX = /git clone --filter=blob:none (?<url>[^\s]+).*/
+        GIT_DEPENDENCY_UNREACHABLE_REGEX = /git clone --filter=blob:none --quiet (?<url>[^\s]+).*/
         GIT_REFERENCE_NOT_FOUND_REGEX = /git checkout -q (?<tag>[^\s]+).*/
         PIPENV_INSTALLATION_ERROR = "python setup.py egg_info exited with 1"
         PIPENV_INSTALLATION_ERROR_REGEX =
@@ -37,14 +27,13 @@ module Dependabot
 
         PIPENV_RANGE_WARNING = /Warning:\sPython\s[<>].* was not found/
 
-        DEPENDENCY_TYPES = %w(packages dev-packages).freeze
+        attr_reader :dependency, :dependency_files, :credentials, :repo_contents_path
 
-        attr_reader :dependency, :dependency_files, :credentials
-
-        def initialize(dependency:, dependency_files:, credentials:)
+        def initialize(dependency:, dependency_files:, credentials:, repo_contents_path:)
           @dependency               = dependency
           @dependency_files         = dependency_files
           @credentials              = credentials
+          @repo_contents_path       = repo_contents_path
         end
 
         def latest_resolvable_version(requirement: nil)
@@ -68,53 +57,18 @@ module Dependabot
           return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)
 
           @latest_resolvable_version_string[requirement] ||=
-            SharedHelpers.in_a_temporary_directory do
+            SharedHelpers.in_a_temporary_repo_directory(base_directory, repo_contents_path) do
               SharedHelpers.with_git_configured(credentials: credentials) do
-                write_temporary_dependency_files(updated_req: requirement)
+                write_temporary_dependency_files
                 install_required_python
 
-                # Shell out to Pipenv, which handles everything for us.
-                # Whilst calling `lock` avoids doing an install as part of the
-                # pipenv flow, an install is still done by pip-tools in order
-                # to resolve the dependencies. That means this is slow.
-                run_pipenv_command("pyenv exec pipenv lock")
-
-                updated_lockfile = JSON.parse(File.read("Pipfile.lock"))
-
-                fetch_version_from_parsed_lockfile(updated_lockfile)
+                pipenv_runner.run_upgrade_and_fetch_version(requirement)
               end
             rescue SharedHelpers::HelperSubprocessFailed => e
               handle_pipenv_errors(e)
             end
         end
 
-        def fetch_version_from_parsed_lockfile(updated_lockfile)
-          if dependency.requirements.any?
-            group = dependency.requirements.first[:groups].first
-            deps = updated_lockfile[group] || {}
-
-            version =
-              deps.transform_keys { |k| normalise(k) }
-                  .dig(dependency.name, "version")
-                  &.gsub(/^==/, "")
-
-            return version
-          end
-
-          Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys|
-            deps = updated_lockfile[keys.fetch(:lockfile)] || {}
-            version =
-              deps.transform_keys { |k| normalise(k) }
-                  .dig(dependency.name, "version")
-                  &.gsub(/^==/, "")
-
-            return version if version
-          end
-
-          # If the sub-dependency no longer appears in the lockfile return nil
-          nil
-        end
-
         # rubocop:disable Metrics/CyclomaticComplexity
         # rubocop:disable Metrics/PerceivedComplexity
         # rubocop:disable Metrics/AbcSize
@@ -136,6 +90,19 @@ module Dependabot
             raise DependencyFileNotResolvable, msg
           end
 
+          if error.message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
+            tag = error.message.match(GIT_REFERENCE_NOT_FOUND_REGEX).named_captures.fetch("tag")
+            # Unfortunately the error message doesn't include the package name.
+            # TODO: Talk with pipenv maintainers about exposing the package name, it used to be part of the error output
+            raise GitDependencyReferenceNotFound, "(unknown package at #{tag})"
+          end
+
+          if error.message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
+            url = error.message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX)
+                       .named_captures.fetch("url")
+            raise GitDependenciesNotReachable, url
+          end
+
           if error.message.include?("Could not find a version") || error.message.include?("ResolutionFailure")
             check_original_requirements_resolvable
           end
@@ -165,20 +132,7 @@ module Dependabot
             return if error.message.match?(/#{Regexp.quote(dependency.name)}/i)
           end
 
-          if error.message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
-            tag = error.message.match(GIT_REFERENCE_NOT_FOUND_REGEX).named_captures.fetch("tag")
-            # Unfortunately the error message doesn't include the package name.
-            # TODO: Talk with pipenv maintainers about exposing the package name, it used to be part of the error output
-            raise GitDependencyReferenceNotFound, "(unknown package at #{tag})"
-          end
-
-          if error.message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
-            url = error.message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX)
-                       .named_captures.fetch("url")
-            raise GitDependenciesNotReachable, url
-          end
-
-          raise unless error.message.include?("could not be resolved")
+          raise unless error.message.include?("ResolutionFailure")
         end
         # rubocop:enable Metrics/CyclomaticComplexity
         # rubocop:enable Metrics/PerceivedComplexity
@@ -190,10 +144,10 @@ module Dependabot
         # boolean, so that all deps for this repo will raise identical
         # errors when failing to update
         def check_original_requirements_resolvable
-          SharedHelpers.in_a_temporary_directory do
+          SharedHelpers.in_a_temporary_repo_directory(base_directory, repo_contents_path) do
             write_temporary_dependency_files(update_pipfile: false)
 
-            run_pipenv_command("pyenv exec pipenv lock")
+            pipenv_runner.run_upgrade("==#{dependency.version}")
 
             true
           rescue SharedHelpers::HelperSubprocessFailed => e
@@ -201,6 +155,10 @@ module Dependabot
           end
         end
 
+        def base_directory
+          dependency_files.first.directory
+        end
+
         def handle_pipenv_errors_resolving_original_reqs(error)
           if error.message.include?("Could not find a version") ||
              error.message.include?("package versions have conflicting dependencies")
@@ -220,10 +178,6 @@ module Dependabot
             raise DependencyFileNotResolvable, msg
           end
 
-          # NOTE: Pipenv masks the actual error, see this issue for updates:
-          # https://github.com/pypa/pipenv/issues/2791
-          # TODO: This may no longer be reproducible on latest pipenv, see linked issue,
-          # so investigate when we next bump to newer pipenv...
           handle_pipenv_installation_error(error.message) if error.message.match?(PIPENV_INSTALLATION_ERROR_REGEX)
 
           # Raise an unhandled error, as this could be a problem with
@@ -264,8 +218,7 @@ module Dependabot
           raise DependencyFileNotResolvable, msg
         end
 
-        def write_temporary_dependency_files(updated_req: nil,
-                                             update_pipfile: true)
+        def write_temporary_dependency_files(update_pipfile: true)
           dependency_files.each do |file|
             path = file.name
             FileUtils.mkdir_p(Pathname.new(path).dirname)
@@ -291,7 +244,7 @@ module Dependabot
           # Overwrite the pipfile with updated content
           File.write(
             "Pipfile",
-            pipfile_content(updated_requirement: updated_req)
+            pipfile_content
           )
         end
 
@@ -319,93 +272,27 @@ module Dependabot
           dependency_files.find { |f| f.name == config_name }
         end
 
-        def pipfile_content(updated_requirement:)
+        def pipfile_content
           content = pipfile.content
-          content = freeze_other_dependencies(content)
-          content = set_target_dependency_req(content, updated_requirement)
           content = add_private_sources(content)
           content = update_python_requirement(content)
           content
         end
 
-        def freeze_other_dependencies(pipfile_content)
-          Python::FileUpdater::PipfilePreparer
-            .new(pipfile_content: pipfile_content, lockfile: lockfile)
-            .freeze_top_level_dependencies_except([dependency])
-        end
-
         def update_python_requirement(pipfile_content)
           Python::FileUpdater::PipfilePreparer
             .new(pipfile_content: pipfile_content)
             .update_python_requirement(language_version_manager.python_major_minor)
         end
 
-        # rubocop:disable Metrics/PerceivedComplexity
-        def set_target_dependency_req(pipfile_content, updated_requirement)
-          return pipfile_content unless updated_requirement
-
-          pipfile_object = TomlRB.parse(pipfile_content)
-
-          DEPENDENCY_TYPES.each do |type|
-            names = pipfile_object[type]&.keys || []
-            pkg_name = names.find { |nm| normalise(nm) == dependency.name }
-            next unless pkg_name || subdep_type?(type)
-
-            pkg_name ||= dependency.name
-            if pipfile_object.dig(type, pkg_name).is_a?(Hash)
-              pipfile_object[type][pkg_name]["version"] = updated_requirement
-            else
-              pipfile_object[type][pkg_name] = updated_requirement
-            end
-          end
-
-          TomlRB.dump(pipfile_object)
-        end
-        # rubocop:enable Metrics/PerceivedComplexity
-
-        def subdep_type?(type)
-          return false if dependency.top_level?
-
-          lockfile_type = Python::FileParser::DEPENDENCY_GROUP_KEYS
-                          .find { |i| i.fetch(:pipfile) == type }
-                          .fetch(:lockfile)
-
-          JSON.parse(lockfile.content)
-              .fetch(lockfile_type, {})
-              .keys.any? { |k| normalise(k) == dependency.name }
-        end
-
         def add_private_sources(pipfile_content)
           Python::FileUpdater::PipfilePreparer
             .new(pipfile_content: pipfile_content)
             .replace_sources(credentials)
         end
 
-        def run_command(command, env: {}, fingerprint: nil)
-          SharedHelpers.run_shell_command(command, env: env, fingerprint: fingerprint, stderr_to_stdout: true)
-        end
-
-        def run_pipenv_command(command, env: pipenv_env_variables)
-          run_command(
-            "pyenv local #{language_version_manager.python_major_minor}",
-            fingerprint: "pyenv local <python_major_minor>"
-          )
-
-          run_command(command, env: env)
-        end
-
-        def pipenv_env_variables
-          {
-            "PIPENV_YES" => "true",       # Install new Python ver if needed
-            "PIPENV_MAX_RETRIES" => "3",  # Retry timeouts
-            "PIPENV_NOSPIN" => "1",       # Don't pollute logs with spinner
-            "PIPENV_TIMEOUT" => "600",    # Set install timeout to 10 minutes
-            "PIP_DEFAULT_TIMEOUT" => "60" # Set pip timeout to 1 minute
-          }
-        end
-
-        def normalise(name)
-          NameNormaliser.normalise(name)
+        def run_command(command)
+          SharedHelpers.run_shell_command(command, stderr_to_stdout: true)
         end
 
         def python_requirement_parser
@@ -422,6 +309,15 @@ module Dependabot
             )
         end
 
+        def pipenv_runner
+          @pipenv_runner ||=
+            PipenvRunner.new(
+              dependency: dependency,
+              lockfile: lockfile,
+              language_version_manager: language_version_manager
+            )
+        end
+
         def pipfile
           dependency_files.find { |f| f.name == "Pipfile" }
         end

data/lib/dependabot/python/update_checker/poetry_version_resolver.rb

@@ -38,12 +38,13 @@ module Dependabot
           \s+check\syour\sgit\sconfiguration
         /mx
 
-        attr_reader :dependency, :dependency_files, :credentials
+        attr_reader :dependency, :dependency_files, :credentials, :repo_contents_path
 
-        def initialize(dependency:, dependency_files:, credentials:)
+        def initialize(dependency:, dependency_files:, credentials:, repo_contents_path:)
           @dependency               = dependency
           @dependency_files         = dependency_files
           @credentials              = credentials
+          @repo_contents_path       = repo_contents_path
         end
 
         def latest_resolvable_version(requirement: nil)

data/lib/dependabot/python/update_checker.rb

@@ -191,7 +191,8 @@ module Dependabot
         {
           dependency: dependency,
           dependency_files: dependency_files,
-          credentials: credentials
+          credentials: credentials,
+          repo_contents_path: repo_contents_path
         }
       end
 
@@ -221,11 +222,11 @@ module Dependabot
         return lower_bound_req if latest_version.nil?
         return lower_bound_req unless Python::Version.correct?(latest_version)
 
-        lower_bound_req + ", <= #{latest_version}"
+        lower_bound_req + ",<=#{latest_version}"
       end
 
       def updated_version_req_lower_bound
-        return ">= #{dependency.version}" if dependency.version
+        return ">=#{dependency.version}" if dependency.version
 
         version_for_requirement =
           requirements.filter_map { |r| r[:requirement] }
@@ -235,7 +236,7 @@ module Dependabot
                       .select { |version| Gem::Version.correct?(version) }
                       .max_by { |version| Gem::Version.new(version) }
 
-        ">= #{version_for_requirement || 0}"
+        ">=#{version_for_requirement || 0}"
       end
 
       def fetch_latest_version

data/lib/dependabot/python.rb

@@ -33,3 +33,6 @@ Dependabot::Dependency.register_name_normaliser(
   "pip",
   ->(name) { Dependabot::Python::NameNormaliser.normalise(name) }
 )
+
+require "dependabot/utils"
+Dependabot::Utils.register_always_clone("pip")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-python
 version: !ruby/object:Gem::Version
-  version: 0.236.0
+  version: 0.238.0
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-26 00:00:00.000000000 Z
+date: 2023-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.236.0
+        version: 0.238.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.236.0
+        version: 0.238.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -94,20 +94,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rspec-sorbet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.9.2
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.56.0
+        version: 1.57.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.56.0
+        version: 1.57.2
 - !ruby/object:Gem::Dependency
   name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
@@ -229,6 +243,7 @@ files:
 - lib/dependabot/python/name_normaliser.rb
 - lib/dependabot/python/native_helpers.rb
 - lib/dependabot/python/pip_compile_file_matcher.rb
+- lib/dependabot/python/pipenv_runner.rb
 - lib/dependabot/python/requirement.rb
 - lib/dependabot/python/requirement_parser.rb
 - lib/dependabot/python/update_checker.rb
@@ -245,7 +260,7 @@ licenses:
 - Nonstandard
 metadata:
   bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
-  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.236.0
+  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.238.0
 post_install_message: 
 rdoc_options: []
 require_paths: