dependabot-python 0.234.0 → 0.235.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a898b6d459367fc728deb010aed7f9adbba8064ffe506547b3076d8320024642
-  data.tar.gz: 9fc76c80711472410d5919bab2ff8cca12ab161b7dfae2d23c83fe5afe3e30a2
+  metadata.gz: b267a681a1cb7f4fe9493bab088098d89b943ab77c074ee5dda1837f0a3901a2
+  data.tar.gz: f3fe927fe0afbec634e0ca2feb0ed313bfeda7a0c510457d00e864d409d277da
 SHA512:
-  metadata.gz: 5c90405c02d5c636ee64d0ecd43b9fbe504b8f6b888d22f81806dddaca8799f590fcf7b46545d06584c3b45f68ffff58f13d0e2b0699441223419eb1e4d35bb9
-  data.tar.gz: 7d10aa0fd61a0707b00bac29c8bd12ac533a49b37cfd1bb58a0830ad73eb50ad5767771889eb78236c175dc0e9cf400522395093894f715a6a80c90c523298f1
+  metadata.gz: a6e4ce8030018586812a2e87db6c7fcfacf8408982b7925a722ff503c8a6d4fd8352b9da3e6c8d527447a4931242a9146fe3a758294bd50c89675e4d77ac6769
+  data.tar.gz: 953aa95ad316e5856c55069bcebb1ec886c0482ed5bdc430ca4c616411e052cee928d6a0a9c8c3833a85b6e6ada8a91a0dc4ecccc4fca9458145d8ef67628561

data/helpers/lib/parser.py

@@ -105,6 +105,11 @@ def parse_requirements(directory):
 
                 pattern = r"-[cr] (.*) \(line \d+\)"
                 abs_path = re.search(pattern, install_req.comes_from).group(1)
+
+                # Ignore dependencies from remote constraint files
+                if not os.path.isfile(abs_path):
+                    continue
+
                 rel_path = os.path.relpath(abs_path, directory)
 
                 requirement_packages.append({

data/lib/dependabot/python/file_fetcher.rb

@@ -6,6 +6,7 @@ require "toml-rb"
 require "dependabot/file_fetchers"
 require "dependabot/file_fetchers/base"
 require "dependabot/python/language_version_manager"
+require "dependabot/python/pip_compile_file_matcher"
 require "dependabot/python/requirement_parser"
 require "dependabot/python/file_parser/pyproject_files_parser"
 require "dependabot/python/file_parser/python_requirement_parser"
@@ -75,7 +76,7 @@ module Dependabot
 
         fetched_files << setup_file if setup_file
         fetched_files << setup_cfg_file if setup_cfg_file
-        fetched_files += path_setup_files
+        fetched_files += project_files
         fetched_files << pip_conf if pip_conf
         fetched_files << python_version_file if python_version_file
 
@@ -113,8 +114,7 @@ module Dependabot
                   pipfile ||
                   pyproject
 
-        path = Pathname.new(File.join(directory, "requirements.txt"))
-                       .cleanpath.to_path
+        path = cleanpath(File.join(directory, "requirements.txt"))
         raise Dependabot::DependencyFileNotFound, path
       end
 
@@ -269,7 +269,7 @@ module Dependabot
 
         paths.flat_map do |path|
           path = File.join(current_dir, path) unless current_dir == "."
-          path = Pathname.new(path).cleanpath.to_path
+          path = cleanpath(path)
 
           next if previously_fetched_files.map(&:name).include?(path)
           next if file.name == path
@@ -293,43 +293,47 @@ module Dependabot
 
           paths.map do |path|
             path = File.join(current_dir, path) unless current_dir == "."
-            Pathname.new(path).cleanpath.to_path
+            cleanpath(path)
           end
         end.flatten.uniq
 
         constraints_paths.map { |path| fetch_file_from_host(path) }
       end
 
-      def path_setup_files
-        path_setup_files = []
-        unfetchable_files = []
+      def project_files
+        project_files = []
+        unfetchable_deps = []
 
-        path_setup_file_paths.each do |path|
-          path_setup_files += fetch_path_setup_file(path)
+        path_dependencies.each do |dep|
+          path = dep[:path]
+          project_files += fetch_project_file(path)
         rescue Dependabot::DependencyFileNotFound => e
-          unfetchable_files << e.file_path.gsub(%r{^/}, "")
+          unfetchable_deps << if sdist_or_wheel?(path)
+                                e.file_path.gsub(%r{^/}, "")
+                              else
+                                "\"#{dep[:name]}\" at #{cleanpath(File.join(directory, dep[:file]))}"
+                              end
         end
 
-        poetry_path_setup_file_paths.each do |path|
-          path_setup_files += fetch_path_setup_file(path, allow_pyproject: true)
+        poetry_path_dependencies.each do |path|
+          project_files += fetch_project_file(path)
         rescue Dependabot::DependencyFileNotFound => e
-          unfetchable_files << e.file_path.gsub(%r{^/}, "")
+          unfetchable_deps << e.file_path.gsub(%r{^/}, "")
         end
 
-        raise Dependabot::PathDependenciesNotReachable, unfetchable_files if unfetchable_files.any?
+        raise Dependabot::PathDependenciesNotReachable, unfetchable_deps if unfetchable_deps.any?
 
-        path_setup_files
+        project_files
       end
 
-      def fetch_path_setup_file(path, allow_pyproject: false)
-        path_setup_files = []
+      def fetch_project_file(path)
+        project_files = []
+
+        path = cleanpath(File.join(path, "setup.py")) unless sdist_or_wheel?(path)
 
-        unless path.end_with?(".tar.gz", ".whl", ".zip")
-          path = Pathname.new(File.join(path, "setup.py")).cleanpath.to_path
-        end
         return [] if path == "setup.py" && setup_file
 
-        path_setup_files <<
+        project_files <<
           begin
             fetch_file_from_host(
               path,
@@ -337,19 +341,20 @@ module Dependabot
             ).tap { |f| f.support_file = true }
           rescue Dependabot::DependencyFileNotFound
             # For projects with pyproject.toml attempt to fetch a pyproject.toml
-            # at the given path instead of a setup.py. We do not require a
-            # setup.py to be present, so if none can be found, simply return
-            return [] unless allow_pyproject
-
+            # at the given path instead of a setup.py.
             fetch_file_from_host(
               path.gsub("setup.py", "pyproject.toml"),
               fetch_submodules: true
             ).tap { |f| f.support_file = true }
           end
 
-        return path_setup_files unless path.end_with?(".py")
+        return project_files unless path.end_with?(".py")
 
-        path_setup_files + cfg_files_for_setup_py(path)
+        project_files + cfg_files_for_setup_py(path)
+      end
+
+      def sdist_or_wheel?(path)
+        path.end_with?(".tar.gz", ".whl", ".zip")
       end
 
       def cfg_files_for_setup_py(path)
@@ -378,60 +383,63 @@ module Dependabot
         end
       end
 
-      def path_setup_file_paths
-        requirement_txt_path_setup_file_paths +
-          requirement_in_path_setup_file_paths +
-          pipfile_path_setup_file_paths
+      def path_dependencies
+        requirement_txt_path_dependencies +
+          requirement_in_path_dependencies +
+          pipfile_path_dependencies
       end
 
-      def requirement_txt_path_setup_file_paths
+      def requirement_txt_path_dependencies
         (requirements_txt_files + child_requirement_txt_files)
-          .map { |req_file| parse_path_setup_paths(req_file) }
-          .flatten.uniq
+          .map { |req_file| parse_requirement_path_dependencies(req_file) }
+          .flatten.uniq { |dep| dep[:path] }
       end
 
-      def requirement_in_path_setup_file_paths
+      def requirement_in_path_dependencies
         requirements_in_files
-          .map { |req_file| parse_path_setup_paths(req_file) }
-          .flatten.uniq
+          .map { |req_file| parse_requirement_path_dependencies(req_file) }
+          .flatten.uniq { |dep| dep[:path] }
       end
 
-      def parse_path_setup_paths(req_file)
+      def parse_requirement_path_dependencies(req_file)
+        # If this is a pip-compile lockfile, rely on whatever path dependencies we found in the main manifest
+        return [] if pip_compile_file_matcher.lockfile_for_pip_compile_file?(req_file)
+
         uneditable_reqs =
           req_file.content
-                  .scan(/^['"]?(?:file:)?(?<path>\..*?)(?=\[|#|'|"|$)/)
-                  .flatten
-                  .map(&:strip)
-                  .reject { |p| p.include?("://") }
+                  .scan(/(?<name>^['"]?(?:file:)?(?<path>\..*?)(?=\[|#|'|"|$))/)
+                  .filter_map do |n, p|
+                    { name: n.strip, path: p.strip, file: req_file.name } unless p.include?("://")
+                  end
 
         editable_reqs =
           req_file.content
-                  .scan(/^(?:-e)\s+['"]?(?:file:)?(?<path>.*?)(?=\[|#|'|"|$)/)
-                  .flatten
-                  .map(&:strip)
-                  .reject { |p| p.include?("://") || p.include?("git@") }
+                  .scan(/(?<name>^(?:-e)\s+['"]?(?:file:)?(?<path>.*?)(?=\[|#|'|"|$))/)
+                  .filter_map do |n, p|
+                    { name: n.strip, path: p.strip, file: req_file.name } unless p.include?("://") || p.include?("git@")
+                  end
 
         uneditable_reqs + editable_reqs
       end
 
-      def pipfile_path_setup_file_paths
+      def pipfile_path_dependencies
         return [] unless pipfile
 
-        paths = []
+        deps = []
         DEPENDENCY_TYPES.each do |dep_type|
           next unless parsed_pipfile[dep_type]
 
           parsed_pipfile[dep_type].each do |_, req|
             next unless req.is_a?(Hash) && req["path"]
 
-            paths << req["path"]
+            deps << { name: req["path"], path: req["path"], file: pipfile.name }
           end
         end
 
-        paths
+        deps
       end
 
-      def poetry_path_setup_file_paths
+      def poetry_path_dependencies
         return [] unless pyproject
 
         paths = []
@@ -447,6 +455,14 @@ module Dependabot
 
         paths
       end
+
+      def cleanpath(path)
+        Pathname.new(path).cleanpath.to_path
+      end
+
+      def pip_compile_file_matcher
+        @pip_compile_file_matcher ||= PipCompileFileMatcher.new(requirements_in_files)
+      end
     end
   end
 end

data/lib/dependabot/python/file_parser/pyproject_files_parser.rb

@@ -8,7 +8,6 @@ require "dependabot/file_parsers/base/dependency_set"
 require "dependabot/python/file_parser"
 require "dependabot/python/requirement"
 require "dependabot/errors"
-require "dependabot/python/helpers"
 require "dependabot/python/name_normaliser"
 
 module Dependabot
@@ -28,7 +27,7 @@ module Dependabot
           dependency_set = Dependabot::FileParsers::Base::DependencySet.new
 
           dependency_set += pyproject_dependencies if using_poetry? || using_pep621?
-          dependency_set += lockfile_dependencies if lockfile
+          dependency_set += lockfile_dependencies if using_poetry? && lockfile
 
           dependency_set
         end
@@ -39,6 +38,16 @@ module Dependabot
 
         def pyproject_dependencies
           if using_poetry?
+            missing_keys = missing_poetry_keys
+
+            if missing_keys.any?
+              raise DependencyFileNotParseable.new(
+                pyproject.path,
+                "#{pyproject.path} is missing the following sections:\n" \
+                "  * #{missing_keys.map { |key| "tool.poetry.#{key}" }.join("\n  * ")}\n"
+              )
+            end
+
             poetry_dependencies
           else
             pep621_dependencies
@@ -53,11 +62,11 @@ module Dependabot
           dependencies = Dependabot::FileParsers::Base::DependencySet.new
 
           POETRY_DEPENDENCY_TYPES.each do |type|
-            deps_hash = parsed_pyproject.dig("tool", "poetry", type) || {}
+            deps_hash = poetry_root[type] || {}
             dependencies += parse_poetry_dependency_group(type, deps_hash)
           end
 
-          groups = parsed_pyproject.dig("tool", "poetry", "group") || {}
+          groups = poetry_root["group"] || {}
           groups.each do |group, group_spec|
             dependencies += parse_poetry_dependency_group(group, group_spec["dependencies"])
           end
@@ -138,7 +147,11 @@ module Dependabot
         end
 
         def using_poetry?
-          !parsed_pyproject.dig("tool", "poetry").nil?
+          !poetry_root.nil?
+        end
+
+        def missing_poetry_keys
+          %w(name version description authors).reject { |key| poetry_root.key?(key) }
         end
 
         def using_pep621?
@@ -146,6 +159,10 @@ module Dependabot
             !parsed_pyproject.dig("project", "optional-dependencies").nil?
         end
 
+        def poetry_root
+          parsed_pyproject.dig("tool", "poetry")
+        end
+
         def using_pdm?
           using_pep621? && pdm_lock
         end
@@ -187,7 +204,7 @@ module Dependabot
             File.write(lockfile.name, lockfile.content)
 
             begin
-              output = Helpers.run_poetry_command("pyenv exec poetry show --only main")
+              output = SharedHelpers.run_shell_command("pyenv exec poetry show --only main")
 
               output.split("\n").map { |line| line.split.first }
             rescue SharedHelpers::HelperSubprocessFailed

data/lib/dependabot/python/file_parser/python_requirement_parser.rb

@@ -103,21 +103,7 @@ module Dependabot
         end
 
         def run_command(command, env: {})
-          start = Time.now
-          command = SharedHelpers.escape_command(command)
-          stdout, process = Open3.capture2e(env, command)
-          time_taken = Time.now - start
-
-          return stdout if process.success?
-
-          raise SharedHelpers::HelperSubprocessFailed.new(
-            message: stdout,
-            error_context: {
-              command: command,
-              time_taken: time_taken,
-              process_exit_value: process.to_s
-            }
-          )
+          SharedHelpers.run_shell_command(command, env: env, stderr_to_stdout: true)
         end
 
         def requirement_class

data/lib/dependabot/python/file_parser.rb

@@ -10,6 +10,7 @@ require "dependabot/python/requirement"
 require "dependabot/errors"
 require "dependabot/python/native_helpers"
 require "dependabot/python/name_normaliser"
+require "dependabot/python/pip_compile_file_matcher"
 
 module Dependabot
   module Python
@@ -74,21 +75,29 @@ module Dependabot
           # probably blocked. Ignore it.
           next if blocking_marker?(dep)
 
+          name = dep["name"]
+          file = dep["file"]
+          version = dep["version"]
+          original_file = get_original_file(file)
+
           requirements =
-            if lockfile_for_pip_compile_file?(dep["file"]) then []
+            if original_file && pip_compile_file_matcher.lockfile_for_pip_compile_file?(original_file) then []
             else
               [{
                 requirement: dep["requirement"],
-                file: Pathname.new(dep["file"]).cleanpath.to_path,
+                file: Pathname.new(file).cleanpath.to_path,
                 source: nil,
-                groups: group_from_filename(dep["file"])
+                groups: group_from_filename(file)
               }]
             end
 
+          # PyYAML < 6.0 will cause `pip-compile` to fail due to incompatiblity with Cython 3. Workaround it.
+          SharedHelpers.run_shell_command("pyenv exec pip install cython<3.0") if old_pyyaml?(name, version)
+
           dependencies <<
             Dependency.new(
-              name: normalised_name(dep["name"], dep["extras"]),
-              version: dep["version"]&.include?("*") ? nil : dep["version"],
+              name: normalised_name(name, dep["extras"]),
+              version: version&.include?("*") ? nil : version,
               requirements: requirements,
               package_manager: "pip"
             )
@@ -96,6 +105,13 @@ module Dependabot
         dependencies
       end
 
+      def old_pyyaml?(name, version)
+        major_version = version&.split(".")&.first
+        return false unless major_version
+
+        name == "pyyaml" && major_version < "6"
+      end
+
       def group_from_filename(filename)
         if filename.include?("dev") then ["dev-dependencies"]
         else
@@ -118,17 +134,6 @@ module Dependabot
           .dependency_set
       end
 
-      def lockfile_for_pip_compile_file?(filename)
-        return false unless pip_compile_files.any?
-        return false unless filename.end_with?(".txt")
-
-        file = dependency_files.find { |f| f.name == filename }
-        return true if file&.content&.match?(output_file_regex(filename))
-
-        basename = filename.gsub(/\.txt$/, "")
-        pip_compile_files.any? { |f| f.name == basename + ".in" }
-      end
-
       def parsed_requirement_files
         SharedHelpers.in_a_temporary_directory do
           write_temporary_dependency_files
@@ -201,10 +206,6 @@ module Dependabot
         @pipfile_lock ||= get_original_file("Pipfile.lock")
       end
 
-      def output_file_regex(filename)
-        "--output-file[=\s]+#{Regexp.escape(filename)}(?:\s|$)"
-      end
-
       def pyproject
         @pyproject ||= get_original_file("pyproject.toml")
       end
@@ -225,6 +226,10 @@ module Dependabot
         @pip_compile_files ||=
           dependency_files.select { |f| f.name.end_with?(".in") }
       end
+
+      def pip_compile_file_matcher
+        @pip_compile_file_matcher ||= PipCompileFileMatcher.new(pip_compile_files)
+      end
     end
   end
 end

data/lib/dependabot/python/file_updater/pip_compile_file_updater.rb

@@ -27,6 +27,8 @@ module Dependabot
         WARNINGS = /\s*# WARNING:.*\Z/m
         UNSAFE_NOTE = /\s*# The following packages are considered to be unsafe.*\Z/m
         RESOLVER_REGEX = /(?<=--resolver=)(\w+)/
+        NATIVE_COMPILATION_ERROR =
+          "pip._internal.exceptions.InstallationSubprocessError: Getting requirements to build wheel exited with 1"
 
         attr_reader :dependencies, :dependency_files, :credentials
 
@@ -34,6 +36,7 @@ module Dependabot
           @dependencies = dependencies
           @dependency_files = dependency_files
           @credentials = credentials
+          @build_isolation = true
         end
 
         def updated_dependency_files
@@ -67,28 +70,7 @@ module Dependabot
             language_version_manager.install_required_python
 
             filenames_to_compile.each do |filename|
-              # Shell out to pip-compile, generate a new set of requirements.
-              # This is slow, as pip-compile needs to do installs.
-              options = pip_compile_options(filename)
-              options_fingerprint = pip_compile_options_fingerprint(options)
-
-              name_part = "pyenv exec pip-compile " \
-                          "#{options} -P " \
-                          "#{dependency.name}"
-              fingerprint_name_part = "pyenv exec pip-compile " \
-                                      "#{options_fingerprint} -P " \
-                                      "<dependency_name>"
-
-              version_part = "#{dependency.version} #{filename}"
-              fingerprint_version_part = "<dependency_version> <filename>"
-
-              # Don't escape pyenv `dep-name==version` syntax
-              run_pip_compile_command(
-                "#{SharedHelpers.escape_command(name_part)}==" \
-                "#{SharedHelpers.escape_command(version_part)}",
-                allow_unsafe_shell_command: true,
-                fingerprint: "#{fingerprint_name_part}==#{fingerprint_version_part}"
-              )
+              compile_file(filename)
             end
 
             # Remove any .python-version file before parsing the reqs
@@ -108,6 +90,44 @@ module Dependabot
           end
         end
 
+        def compile_file(filename)
+          # Shell out to pip-compile, generate a new set of requirements.
+          # This is slow, as pip-compile needs to do installs.
+          options = pip_compile_options(filename)
+          options_fingerprint = pip_compile_options_fingerprint(options)
+
+          name_part = "pyenv exec pip-compile " \
+                      "#{options} -P " \
+                      "#{dependency.name}"
+          fingerprint_name_part = "pyenv exec pip-compile " \
+                                  "#{options_fingerprint} -P " \
+                                  "<dependency_name>"
+
+          version_part = "#{dependency.version} #{filename}"
+          fingerprint_version_part = "<dependency_version> <filename>"
+
+          # Don't escape pyenv `dep-name==version` syntax
+          run_pip_compile_command(
+            "#{SharedHelpers.escape_command(name_part)}==" \
+            "#{SharedHelpers.escape_command(version_part)}",
+            allow_unsafe_shell_command: true,
+            fingerprint: "#{fingerprint_name_part}==#{fingerprint_version_part}"
+          )
+        rescue SharedHelpers::HelperSubprocessFailed => e
+          retry_count ||= 0
+          retry_count += 1
+          if compilation_error?(e) && retry_count <= 1
+            @build_isolation = false
+            retry
+          end
+
+          raise
+        end
+
+        def compilation_error?(error)
+          error.message.include?(NATIVE_COMPILATION_ERROR)
+        end
+
         def update_manifest_files
           dependency_files.filter_map do |file|
             next unless file.name.end_with?(".in")
@@ -146,30 +166,21 @@ module Dependabot
         end
 
         def run_command(cmd, env: python_env, allow_unsafe_shell_command: false, fingerprint:)
-          start = Time.now
-          command = if allow_unsafe_shell_command
-                      cmd
-                    else
-                      SharedHelpers.escape_command(cmd)
-                    end
-          stdout, process = Open3.capture2e(env, command)
-          time_taken = Time.now - start
-
-          return stdout if process.success?
+          SharedHelpers.run_shell_command(
+            cmd,
+            env: env,
+            allow_unsafe_shell_command: allow_unsafe_shell_command,
+            fingerprint: fingerprint,
+            stderr_to_stdout: true
+          )
+        rescue SharedHelpers::HelperSubprocessFailed => e
+          stdout = e.message
 
           if stdout.match?(INCOMPATIBLE_VERSIONS_REGEX)
             raise DependencyFileNotResolvable, stdout.match(INCOMPATIBLE_VERSIONS_REGEX)
           end
 
-          raise SharedHelpers::HelperSubprocessFailed.new(
-            message: stdout,
-            error_context: {
-              command: command,
-              fingerprint: fingerprint,
-              time_taken: time_taken,
-              process_exit_value: process.to_s
-            }
-          )
+          raise
         end
 
         def run_pip_compile_command(command, allow_unsafe_shell_command: false, fingerprint:)
@@ -412,7 +423,7 @@ module Dependabot
         end
 
         def pip_compile_options(filename)
-          options = ["--build-isolation"]
+          options = @build_isolation ? ["--build-isolation"] : ["--no-build-isolation"]
           options += pip_compile_index_options
 
           if (requirements_file = compiled_file_for_filename(filename))

data/lib/dependabot/python/file_updater/pipfile_file_updater.rb

@@ -245,28 +245,16 @@ module Dependabot
           File.write("dev-req.txt", dev_req_content)
         end
 
-        def run_command(command, env: {})
-          start = Time.now
-          command = SharedHelpers.escape_command(command)
-          stdout, _, process = Open3.capture3(env, command)
-          time_taken = Time.now - start
-
-          # Raise an error with the output from the shell session if Pipenv
-          # returns a non-zero status
-          return stdout if process.success?
-
-          raise SharedHelpers::HelperSubprocessFailed.new(
-            message: stdout,
-            error_context: {
-              command: command,
-              time_taken: time_taken,
-              process_exit_value: process.to_s
-            }
-          )
+        def run_command(command, env: {}, fingerprint: nil)
+          SharedHelpers.run_shell_command(command, env: env, fingerprint: fingerprint)
         end
 
         def run_pipenv_command(command, env: pipenv_env_variables)
-          run_command("pyenv local #{language_version_manager.python_major_minor}")
+          run_command(
+            "pyenv local #{language_version_manager.python_major_minor}",
+            fingerprint: "pyenv local <python_major_minor>"
+          )
+
           run_command(command, env: env)
         end
 

data/lib/dependabot/python/file_updater/poetry_file_updater.rb

@@ -10,7 +10,6 @@ require "dependabot/python/version"
 require "dependabot/python/requirement"
 require "dependabot/python/file_parser/python_requirement_parser"
 require "dependabot/python/file_updater"
-require "dependabot/python/helpers"
 require "dependabot/python/native_helpers"
 require "dependabot/python/name_normaliser"
 
@@ -61,34 +60,37 @@ module Dependabot
         end
 
         def updated_pyproject_content
-          dependencies
-            .select { |dep| requirement_changed?(pyproject, dep) }
-            .reduce(pyproject.content.dup) do |content, dep|
-              updated_requirement =
-                dep.requirements.find { |r| r[:file] == pyproject.name }
-                   .fetch(:requirement)
-
-              old_req =
-                dep.previous_requirements
-                   .find { |r| r[:file] == pyproject.name }
-                   .fetch(:requirement)
-
-              declaration_regex = declaration_regex(dep)
-              updated_content = if content.match?(declaration_regex)
-                                  content.gsub(declaration_regex(dep)) do |match|
-                                    match.gsub(old_req, updated_requirement)
-                                  end
-                                else
-                                  content.gsub(table_declaration_regex(dep)) do |match|
-                                    match.gsub(/(\s*version\s*=\s*["'])#{Regexp.escape(old_req)}/,
-                                               '\1' + updated_requirement)
-                                  end
-                                end
-
-              raise "Content did not change!" if content == updated_content
-
-              updated_content
+          content = pyproject.content
+          return content unless requirement_changed?(pyproject, dependency)
+
+          updated_content = content.dup
+
+          dependency.requirements.zip(dependency.previous_requirements).each do |new_r, old_r|
+            next unless new_r[:file] == pyproject.name && old_r[:file] == pyproject.name
+
+            updated_content = replace_dep(dependency, updated_content, new_r, old_r)
+          end
+
+          raise "Content did not change!" if content == updated_content
+
+          updated_content
+        end
+
+        def replace_dep(dep, content, new_r, old_r)
+          new_req = new_r[:requirement]
+          old_req = old_r[:requirement]
+
+          declaration_regex = declaration_regex(dep, old_r)
+          if content.match?(declaration_regex)
+            content.gsub(declaration_regex) do |match|
+              match.gsub(old_req, new_req)
             end
+          else
+            content.gsub(table_declaration_regex(dep, new_r)) do |match|
+              match.gsub(/(\s*version\s*=\s*["'])#{Regexp.escape(old_req)}/,
+                         '\1' + new_req)
+            end
+          end
         end
 
         def updated_lockfile_content
@@ -204,7 +206,7 @@ module Dependabot
         end
 
         def run_poetry_command(command, fingerprint: nil)
-          Helpers.run_poetry_command(command, fingerprint: fingerprint)
+          SharedHelpers.run_shell_command(command, fingerprint: fingerprint)
         end
 
         def write_temporary_dependency_files(pyproject_content)
@@ -241,12 +243,12 @@ module Dependabot
           end
         end
 
-        def declaration_regex(dep)
-          /(?:^\s*|["'])#{escape(dep)}["']?\s*=.*$/i
+        def declaration_regex(dep, old_req)
+          /#{old_req[:groups].first}(?:\.dependencies)?\]\s*\n.*?(?:^\s*|["'])#{escape(dep)}["']?\s*=.*$/mi
         end
 
-        def table_declaration_regex(dep)
-          /tool\.poetry\.[^\n]+\.#{escape(dep)}\]\n.*?\s*version\s* =.*?\n/m
+        def table_declaration_regex(dep, old_req)
+          /tool\.poetry\.#{old_req[:groups].first}\.#{escape(dep)}\]\n.*?\s*version\s* =.*?\n/m
         end
 
         def escape(dep)

data/lib/dependabot/python/file_updater/requirement_file_updater.rb

@@ -48,10 +48,11 @@ module Dependabot
         end
 
         def updated_requirement_or_setup_file_content(new_req, old_req)
-          content = get_original_file(new_req.fetch(:file)).content
+          original_file = get_original_file(new_req.fetch(:file))
+          raise "Could not find a dependency file for #{new_req}" unless original_file
 
           RequirementReplacer.new(
-            content: content,
+            content: original_file.content,
             dependency_name: dependency.name,
             old_requirement: old_req.fetch(:requirement),
             new_requirement: new_req.fetch(:requirement),

data/lib/dependabot/python/pip_compile_file_matcher.rb

@@ -0,0 +1,32 @@
+# typed: true
+# frozen_string_literal: true
+
+module Dependabot
+  module Python
+    class PipCompileFileMatcher
+      def initialize(requirements_in_files)
+        @requirements_in_files = requirements_in_files
+      end
+
+      def lockfile_for_pip_compile_file?(file)
+        return false unless requirements_in_files.any?
+
+        name = file.name
+        return false unless name.end_with?(".txt")
+
+        return true if file.content.match?(output_file_regex(name))
+
+        basename = name.gsub(/\.txt$/, "")
+        requirements_in_files.any? { |f| f.name == basename + ".in" }
+      end
+
+      private
+
+      attr_reader :requirements_in_files
+
+      def output_file_regex(filename)
+        "--output-file[=\s]+#{Regexp.escape(filename)}(?:\s|$)"
+      end
+    end
+  end
+end

data/lib/dependabot/python/requirement_parser.rb

@@ -9,7 +9,7 @@ module Dependabot
       COMPARISON = /===|==|>=|<=|<|>|~=|!=/
       VERSION = /([1-9][0-9]*!)?[0-9]+[a-zA-Z0-9\-_.*]*(\+[0-9a-zA-Z]+(\.[0-9a-zA-Z]+)*)?/
 
-      REQUIREMENT = /(?<comparison>#{COMPARISON})\s*\\?\s*(?<version>#{VERSION})/
+      REQUIREMENT = /(?<comparison>#{COMPARISON})\s*\\?\s*v?(?<version>#{VERSION})/
       HASH = /--hash=(?<algorithm>.*?):(?<hash>.*?)(?=\s|\\|$)/
       REQUIREMENTS = /#{REQUIREMENT}(\s*,\s*\\?\s*#{REQUIREMENT})*/
       HASHES = /#{HASH}(\s*\\?\s*#{HASH})*/

data/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb

@@ -27,7 +27,7 @@ module Dependabot
         GIT_DEPENDENCY_UNREACHABLE_REGEX = /git clone --filter=blob:none --quiet (?<url>[^\s]+).* /
         GIT_REFERENCE_NOT_FOUND_REGEX = /Did not find branch or tag '(?<tag>[^\n"]+)'/m
         NATIVE_COMPILATION_ERROR =
-          "pip._internal.exceptions.InstallationSubprocessError: Command errored out with exit status 1:"
+          "pip._internal.exceptions.InstallationSubprocessError: Getting requirements to build wheel exited with 1"
         # See https://packaging.python.org/en/latest/tutorials/packaging-projects/#configuring-metadata
         PYTHON_PACKAGE_NAME_REGEX = /[A-Za-z0-9_\-]+/
         RESOLUTION_IMPOSSIBLE_ERROR = "ResolutionImpossible"
@@ -43,17 +43,21 @@ module Dependabot
         end
 
         def latest_resolvable_version(requirement: nil)
+          @latest_resolvable_version_string ||= {}
+          return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)
+
           version_string =
             fetch_latest_resolvable_version_string(requirement: requirement)
 
-          version_string.nil? ? nil : Python::Version.new(version_string)
+          @latest_resolvable_version_string[requirement] ||=
+            version_string.nil? ? nil : Python::Version.new(version_string)
         end
 
         def resolvable?(version:)
           @resolvable ||= {}
           return @resolvable[version] if @resolvable.key?(version)
 
-          @resolvable[version] = if fetch_latest_resolvable_version_string(requirement: "==#{version}")
+          @resolvable[version] = if latest_resolvable_version(requirement: "==#{version}")
                                    true
                                  else
                                    false
@@ -63,57 +67,59 @@ module Dependabot
         private
 
         def fetch_latest_resolvable_version_string(requirement:)
-          @latest_resolvable_version_string ||= {}
-          return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)
+          SharedHelpers.in_a_temporary_directory do
+            SharedHelpers.with_git_configured(credentials: credentials) do
+              write_temporary_dependency_files(updated_req: requirement)
+              language_version_manager.install_required_python
 
-          @latest_resolvable_version_string[requirement] ||=
-            SharedHelpers.in_a_temporary_directory do
-              SharedHelpers.with_git_configured(credentials: credentials) do
-                write_temporary_dependency_files(updated_req: requirement)
-                language_version_manager.install_required_python
-
-                filenames_to_compile.each do |filename|
-                  # Shell out to pip-compile.
-                  # This is slow, as pip-compile needs to do installs.
-                  options = pip_compile_options(filename)
-                  options_fingerprint = pip_compile_options_fingerprint(options)
-
-                  run_pip_compile_command(
-                    "pyenv exec pip-compile -v #{options} -P #{dependency.name} #{filename}",
-                    fingerprint: "pyenv exec pip-compile -v #{options_fingerprint} -P <dependency_name> <filename>"
-                  )
-
-                  next if dependency.top_level?
-
-                  # Run pip-compile a second time for transient dependencies
-                  # to make sure we do not update dependencies that are
-                  # superfluous. pip-compile does not detect these when
-                  # updating a specific dependency with the -P option.
-                  # Running pip-compile a second time will automatically remove
-                  # superfluous dependencies. Dependabot then marks those with
-                  # update_not_possible.
-                  write_original_manifest_files
-                  run_pip_compile_command(
-                    "pyenv exec pip-compile #{options} #{filename}",
-                    fingerprint: "pyenv exec pip-compile #{options_fingerprint} <filename>"
-                  )
-                end
-
-                # Remove any .python-version file before parsing the reqs
-                FileUtils.remove_entry(".python-version", true)
-
-                parse_updated_files
-              end
-            rescue SharedHelpers::HelperSubprocessFailed => e
-              retry_count ||= 0
-              retry_count += 1
-              if compilation_error?(e) && retry_count <= 1
-                @build_isolation = false
-                retry
+              filenames_to_compile.each do |filename|
+                return nil unless compile_file(filename)
               end
 
-              handle_pip_compile_errors(e)
+              # Remove any .python-version file before parsing the reqs
+              FileUtils.remove_entry(".python-version", true)
+
+              parse_updated_files
             end
+          end
+        end
+
+        def compile_file(filename)
+          # Shell out to pip-compile.
+          # This is slow, as pip-compile needs to do installs.
+          options = pip_compile_options(filename)
+          options_fingerprint = pip_compile_options_fingerprint(options)
+
+          run_pip_compile_command(
+            "pyenv exec pip-compile -v #{options} -P #{dependency.name} #{filename}",
+            fingerprint: "pyenv exec pip-compile -v #{options_fingerprint} -P <dependency_name> <filename>"
+          )
+
+          return true if dependency.top_level?
+
+          # Run pip-compile a second time for transient dependencies
+          # to make sure we do not update dependencies that are
+          # superfluous. pip-compile does not detect these when
+          # updating a specific dependency with the -P option.
+          # Running pip-compile a second time will automatically remove
+          # superfluous dependencies. Dependabot then marks those with
+          # update_not_possible.
+          write_original_manifest_files
+          run_pip_compile_command(
+            "pyenv exec pip-compile #{options} #{filename}",
+            fingerprint: "pyenv exec pip-compile #{options_fingerprint} <filename>"
+          )
+
+          true
+        rescue SharedHelpers::HelperSubprocessFailed => e
+          retry_count ||= 0
+          retry_count += 1
+          if compilation_error?(e) && retry_count <= 1
+            @build_isolation = false
+            retry
+          end
+
+          handle_pip_compile_errors(e.message)
         end
 
         def compilation_error?(error)
@@ -122,8 +128,8 @@ module Dependabot
 
         # rubocop:disable Metrics/AbcSize
         # rubocop:disable Metrics/PerceivedComplexity
-        def handle_pip_compile_errors(error)
-          if error.message.include?(RESOLUTION_IMPOSSIBLE_ERROR)
+        def handle_pip_compile_errors(message)
+          if message.include?(RESOLUTION_IMPOSSIBLE_ERROR)
             check_original_requirements_resolvable
             # If the original requirements are resolvable but we get an
             # incompatibility error after unlocking then it's likely to be
@@ -131,14 +137,14 @@ module Dependabot
             return nil
           end
 
-          if error.message.include?("UnsupportedConstraint")
+          if message.include?("UnsupportedConstraint")
             # If there's an unsupported constraint, check if it existed
             # previously (and raise if it did)
             check_original_requirements_resolvable
           end
 
-          if (error.message.include?('Command "python setup.py egg_info') ||
-              error.message.include?(
+          if (message.include?('Command "python setup.py egg_info') ||
+              message.include?(
                 "exit status 1: python setup.py egg_info"
               )) &&
              check_original_requirements_resolvable
@@ -147,16 +153,16 @@ module Dependabot
             return
           end
 
-          if error.message.include?(RESOLUTION_IMPOSSIBLE_ERROR) &&
-             !error.message.match?(/#{Regexp.quote(dependency.name)}/i)
+          if message.include?(RESOLUTION_IMPOSSIBLE_ERROR) &&
+             !message.match?(/#{Regexp.quote(dependency.name)}/i)
             # Sometimes pip-tools gets confused and can't work around
             # sub-dependency incompatibilities. Ignore those cases.
             return nil
           end
 
-          if error.message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
-            tag = error.message.match(GIT_REFERENCE_NOT_FOUND_REGEX).named_captures.fetch("tag")
-            constraints_section = error.message.split("Finding the best candidates:").first
+          if message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
+            tag = message.match(GIT_REFERENCE_NOT_FOUND_REGEX).named_captures.fetch("tag")
+            constraints_section = message.split("Finding the best candidates:").first
             egg_regex = /#{Regexp.escape(tag)}#egg=(#{PYTHON_PACKAGE_NAME_REGEX})/
             name_match = constraints_section.scan(egg_regex)
 
@@ -166,15 +172,15 @@ module Dependabot
             raise GitDependencyReferenceNotFound, "(unknown package at #{tag})"
           end
 
-          if error.message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
-            url = error.message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX)
-                       .named_captures.fetch("url")
+          if message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
+            url = message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX)
+                         .named_captures.fetch("url")
             raise GitDependenciesNotReachable, url
           end
 
-          raise Dependabot::OutOfDisk if error.message.end_with?("[Errno 28] No space left on device")
+          raise Dependabot::OutOfDisk if message.end_with?("[Errno 28] No space left on device")
 
-          raise Dependabot::OutOfMemory if error.message.end_with?("MemoryError")
+          raise Dependabot::OutOfMemory if message.end_with?("MemoryError")
 
           raise
         end
@@ -217,22 +223,7 @@ module Dependabot
         end
 
         def run_command(command, env: python_env, fingerprint:)
-          start = Time.now
-          command = SharedHelpers.escape_command(command)
-          stdout, process = Open3.capture2e(env, command)
-          time_taken = Time.now - start
-
-          return stdout if process.success?
-
-          raise SharedHelpers::HelperSubprocessFailed.new(
-            message: stdout,
-            error_context: {
-              command: command,
-              fingerprint: fingerprint,
-              time_taken: time_taken,
-              process_exit_value: process.to_s
-            }
-          )
+          SharedHelpers.run_shell_command(command, env: env, fingerprint: fingerprint, stderr_to_stdout: true)
         end
 
         def pip_compile_options_fingerprint(options)

data/lib/dependabot/python/update_checker/pipenv_version_resolver.rb

@@ -381,26 +381,16 @@ module Dependabot
             .replace_sources(credentials)
         end
 
-        def run_command(command, env: {})
-          start = Time.now
-          command = SharedHelpers.escape_command(command)
-          stdout, process = Open3.capture2e(env, command)
-          time_taken = Time.now - start
-
-          return stdout if process.success?
-
-          raise SharedHelpers::HelperSubprocessFailed.new(
-            message: stdout,
-            error_context: {
-              command: command,
-              time_taken: time_taken,
-              process_exit_value: process.to_s
-            }
-          )
+        def run_command(command, env: {}, fingerprint: nil)
+          SharedHelpers.run_shell_command(command, env: env, fingerprint: fingerprint, stderr_to_stdout: true)
         end
 
         def run_pipenv_command(command, env: pipenv_env_variables)
-          run_command("pyenv local #{language_version_manager.python_major_minor}")
+          run_command(
+            "pyenv local #{language_version_manager.python_major_minor}",
+            fingerprint: "pyenv local <python_major_minor>"
+          )
+
           run_command(command, env: env)
         end
 

data/lib/dependabot/python/update_checker/poetry_version_resolver.rb

@@ -14,7 +14,6 @@ require "dependabot/python/file_updater/pyproject_preparer"
 require "dependabot/python/update_checker"
 require "dependabot/python/version"
 require "dependabot/python/requirement"
-require "dependabot/python/helpers"
 require "dependabot/python/native_helpers"
 require "dependabot/python/authed_url_builder"
 require "dependabot/python/name_normaliser"
@@ -309,7 +308,7 @@ module Dependabot
         end
 
         def run_poetry_command(command, fingerprint: nil)
-          Helpers.run_poetry_command(command, fingerprint: fingerprint)
+          SharedHelpers.run_shell_command(command, fingerprint: fingerprint)
         end
 
         def normalise(name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-python
 version: !ruby/object:Gem::Version
-  version: 0.234.0
+  version: 0.235.0
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-12 00:00:00.000000000 Z
+date: 2023-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.234.0
+        version: 0.235.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.234.0
+        version: 0.235.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -224,11 +224,11 @@ files:
 - lib/dependabot/python/file_updater/requirement_file_updater.rb
 - lib/dependabot/python/file_updater/requirement_replacer.rb
 - lib/dependabot/python/file_updater/setup_file_sanitizer.rb
-- lib/dependabot/python/helpers.rb
 - lib/dependabot/python/language_version_manager.rb
 - lib/dependabot/python/metadata_finder.rb
 - lib/dependabot/python/name_normaliser.rb
 - lib/dependabot/python/native_helpers.rb
+- lib/dependabot/python/pip_compile_file_matcher.rb
 - lib/dependabot/python/requirement.rb
 - lib/dependabot/python/requirement_parser.rb
 - lib/dependabot/python/update_checker.rb
@@ -245,7 +245,7 @@ licenses:
 - Nonstandard
 metadata:
   bug_tracker_uri: https://github.com/dependabot/dependabot-core/issues
-  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.234.0
+  changelog_uri: https://github.com/dependabot/dependabot-core/releases/tag/v0.235.0
 post_install_message: 
 rdoc_options: []
 require_paths:

data/lib/dependabot/python/helpers.rb

@@ -1,35 +0,0 @@
-# typed: true
-# frozen_string_literal: true
-
-require "time"
-require "open3"
-
-require "dependabot/errors"
-require "dependabot/shared_helpers"
-
-module Dependabot
-  module Python
-    module Helpers
-      def self.run_poetry_command(command, fingerprint: nil)
-        start = Time.now
-        command = SharedHelpers.escape_command(command)
-        stdout, stderr, process = Open3.capture3(command)
-        time_taken = Time.now - start
-
-        # Raise an error with the output from the shell session if Poetry
-        # returns a non-zero status
-        return stdout if process.success?
-
-        raise SharedHelpers::HelperSubprocessFailed.new(
-          message: stderr,
-          error_context: {
-            command: command,
-            fingerprint: fingerprint,
-            time_taken: time_taken,
-            process_exit_value: process.to_s
-          }
-        )
-      end
-    end
-  end
-end