dependabot-python 0.211.0 → 0.213.0

data/lib/dependabot/python/update_checker/poetry_version_resolver.rb

@@ -3,6 +3,7 @@
 require "excon"
 require "toml-rb"
 require "open3"
+require "uri"
 require "dependabot/dependency"
 require "dependabot/errors"
 require "dependabot/shared_helpers"
@@ -23,18 +24,30 @@ module Dependabot
       # This class does version resolution for pyproject.toml files.
       class PoetryVersionResolver
         GIT_REFERENCE_NOT_FOUND_REGEX = /
-          'git'.*pypoetry-git-(?<name>.+?).{8}',
+          (?:'git'.*pypoetry-git-(?<name>.+?).{8}',
           'checkout',
           '(?<tag>.+?)'
-        /x.freeze
+          |
+          Failed to checkout
+          (?<tag>.+?)
+          (?<url>.+?).git at '(?<tag>.+?)'
+          |
+          ...Failedtoclone
+          (?<url>.+?).gitat'(?<tag>.+?)',
+          verifyrefexistsonremote)
+        /x # TODO: remove the first clause and | when py3.6 support is EoL
         GIT_DEPENDENCY_UNREACHABLE_REGEX = /
-            '\['git',
-            \s+'clone',
-            \s+'--recurse-submodules',
-            \s+'(--)?',
-            \s+'(?<url>.+?)'.*
-            \s+exit\s+status\s+128
-          /mx.freeze
+          (?:'\['git',
+          \s+'clone',
+          \s+'--recurse-submodules',
+          \s+'(--)?',
+          \s+'(?<url>.+?)'.*
+          \s+exit\s+status\s+128
+          |
+          \s+Failed\sto\sclone
+          \s+(?<url>.+?),
+          \s+check\syour\sgit\sconfiguration)
+        /mx # TODO: remove the first clause and | when py3.6 support is EoL
 
         attr_reader :dependency, :dependency_files, :credentials
 
@@ -61,7 +74,8 @@ module Dependabot
                                    false
                                  end
         rescue SharedHelpers::HelperSubprocessFailed => e
-          raise unless e.message.include?("SolverProblemError")
+          raise unless e.message.include?("SolverProblemError") || # TODO: Remove once py3.6 is EoL
+                       e.message.include?("version solving failed.")
 
           @resolvable[version] = false
         end
@@ -78,13 +92,11 @@ module Dependabot
                 write_temporary_dependency_files(updated_req: requirement)
                 add_auth_env_vars
 
-                if python_version && !pre_installed_python?(python_version)
-                  run_poetry_command("pyenv install -s #{python_version}")
-                  run_poetry_command("pyenv exec pip install --upgrade pip")
-                  run_poetry_command(
-                    "pyenv exec pip install -r "\
-                    "#{NativeHelpers.python_requirements_path}"
-                  )
+                Helpers.install_required_python(python_version)
+
+                # use system git instead of the pure Python dulwich
+                unless python_version&.start_with?("3.6")
+                  run_poetry_command("pyenv exec poetry config experimental.system-git-client true")
                 end
 
                 # Shell out to Poetry, which handles everything for us.
@@ -118,8 +130,13 @@ module Dependabot
         def handle_poetry_errors(error)
           if error.message.gsub(/\s/, "").match?(GIT_REFERENCE_NOT_FOUND_REGEX)
             message = error.message.gsub(/\s/, "")
-            name = message.match(GIT_REFERENCE_NOT_FOUND_REGEX).
-                   named_captures.fetch("name")
+            match = message.match(GIT_REFERENCE_NOT_FOUND_REGEX)
+            name = if (url = match.named_captures.fetch("url"))
+                     File.basename(URI.parse(url).path)
+                   else
+                     message.match(GIT_REFERENCE_NOT_FOUND_REGEX).
+                       named_captures.fetch("name")
+                   end
             raise GitDependencyReferenceNotFound, name
           end
 
@@ -130,7 +147,8 @@ module Dependabot
           end
 
           raise unless error.message.include?("SolverProblemError") ||
-                       error.message.include?("PackageNotFound")
+                       error.message.include?("PackageNotFound") ||
+                       error.message.include?("version solving failed.")
 
           check_original_requirements_resolvable
 
@@ -161,7 +179,8 @@ module Dependabot
               @original_reqs_resolvable = true
             rescue SharedHelpers::HelperSubprocessFailed => e
               raise unless e.message.include?("SolverProblemError") ||
-                           e.message.include?("PackageNotFound")
+                           e.message.include?("PackageNotFound") ||
+                           e.message.include?("version solving failed.")
 
               msg = clean_error_message(e.message)
               raise DependencyFileNotResolvable, msg
@@ -214,9 +233,9 @@ module Dependabot
           end
           return version if version
 
-          msg = "Dependabot detected the following Python requirements "\
-                "for your project: '#{requirements}'.\n\nCurrently, the "\
-                "following Python versions are supported in Dependabot: "\
+          msg = "Dependabot detected the following Python requirements " \
+                "for your project: '#{requirements}'.\n\nCurrently, the " \
+                "following Python versions are supported in Dependabot: " \
                 "#{PythonVersions::SUPPORTED_VERSIONS.join(', ')}."
           raise DependencyFileNotResolvable, msg
         end
@@ -265,7 +284,7 @@ module Dependabot
           pyproject_object = TomlRB.parse(pyproject_content)
           poetry_object = pyproject_object.dig("tool", "poetry")
 
-          Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |type|
+          Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |type|
             names = poetry_object[type]&.keys || []
             pkg_name = names.find { |nm| normalise(nm) == dependency.name }
             next unless pkg_name
@@ -318,7 +337,7 @@ module Dependabot
           stdout, process = Open3.capture2e(command)
           time_taken = Time.now - start
 
-          # Raise an error with the output from the shell session if Pipenv
+          # Raise an error with the output from the shell session if poetry
           # returns a non-zero status
           return if process.success?
 

data/lib/dependabot/python/update_checker/requirements_updater.rb

@@ -9,8 +9,8 @@ module Dependabot
   module Python
     class UpdateChecker
       class RequirementsUpdater
-        PYPROJECT_OR_SEPARATOR = /(?<=[a-zA-Z0-9*])\s*\|+/.freeze
-        PYPROJECT_SEPARATOR = /#{PYPROJECT_OR_SEPARATOR}|,/.freeze
+        PYPROJECT_OR_SEPARATOR = /(?<=[a-zA-Z0-9*])\s*\|+/
+        PYPROJECT_SEPARATOR = /#{PYPROJECT_OR_SEPARATOR}|,/
 
         class UnfixableRequirement < StandardError; end
 
@@ -175,11 +175,25 @@ module Dependabot
         end
         # rubocop:enable Metrics/PerceivedComplexity
 
-        # rubocop:disable Metrics/PerceivedComplexity
         def updated_requirement(req)
           return req unless latest_resolvable_version
           return req unless req.fetch(:requirement)
 
+          case update_strategy
+          when :bump_versions
+            update_requirement(req)
+          when :bump_versions_if_necessary
+            update_requirement_if_needed(req)
+          end
+        end
+
+        def update_requirement_if_needed(req)
+          return req if new_version_satisfies?(req)
+
+          update_requirement(req)
+        end
+
+        def update_requirement(req)
           requirement_strings = req[:requirement].split(",").map(&:strip)
 
           new_requirement =
@@ -197,7 +211,6 @@ module Dependabot
         rescue UnfixableRequirement
           req.merge(requirement: :unfixable)
         end
-        # rubocop:enable Metrics/PerceivedComplexity
 
         def new_version_satisfies?(req)
           requirement_class.
@@ -260,7 +273,7 @@ module Dependabot
         # Updates the version in a constraint to be the given version
         def bump_version(req_string, version_to_be_permitted)
           old_version = req_string.
-                        match(/(#{RequirementParser::VERSION})/).
+                        match(/(#{RequirementParser::VERSION})/o).
                         captures.first
 
           req_string.sub(

data/lib/dependabot/python/update_checker.rb

@@ -26,7 +26,7 @@ module Dependabot
         https://pypi.python.org/simple/
         https://pypi.org/simple/
       ).freeze
-      VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/.freeze
+      VERSION_REGEX = /[0-9]+(?:\.[A-Za-z0-9\-_]+)*/
 
       def latest_version
         @latest_version ||= fetch_latest_version
@@ -89,7 +89,7 @@ module Dependabot
 
       def updated_requirements
         RequirementsUpdater.new(
-          requirements: dependency.requirements,
+          requirements: requirements,
           latest_resolvable_version: preferred_resolvable_version&.to_s,
           update_strategy: requirements_update_strategy,
           has_lockfile: !(pipfile_lock || poetry_lock || pyproject_lock).nil?
@@ -100,8 +100,8 @@ module Dependabot
         # If passed in as an option (in the base class) honour that option
         return @requirements_update_strategy.to_sym if @requirements_update_strategy
 
-        # Otherwise, check if this is a poetry library or not
-        poetry_library? ? :widen_ranges : :bump_versions
+        # Otherwise, check if this is a library or not
+        library? ? :widen_ranges : :bump_versions
       end
 
       private
@@ -115,6 +115,17 @@ module Dependabot
         raise NotImplementedError
       end
 
+      def preferred_version_resolvable_with_unlock?
+        # Our requirements file updater doesn't currently support widening
+        # ranges, so avoid updating this dependency if widening ranges has been
+        # required and the dependency is present on a requirements file.
+        # Otherwise, we will crash later on. TODO: Consider what the correct
+        # behavior is in these cases.
+        return false if requirements_update_strategy == :widen_ranges && updating_requirements_file?
+
+        super
+      end
+
       def fetch_lowest_resolvable_security_fix_version
         fix_version = lowest_security_fix_version
         return latest_resolvable_version if fix_version.nil?
@@ -132,10 +143,8 @@ module Dependabot
         resolver.resolvable?(version: fix_version) ? fix_version : nil
       end
 
-      # rubocop:disable Metrics/PerceivedComplexity
       def resolver_type
-        reqs = dependency.requirements
-        req_files = reqs.map { |r| r.fetch(:file) }
+        reqs = requirements
 
         # If there are no requirements then this is a sub-dependency. It
         # must come from one of Pipenv, Poetry or pip-tools, and can't come
@@ -144,9 +153,9 @@ module Dependabot
 
         # Otherwise, this is a top-level dependency, and we can figure out
         # which resolver to use based on the filename of its requirements
-        return :pipenv if req_files.any? { |f| f == "Pipfile" }
-        return :poetry if req_files.any? { |f| f == "pyproject.toml" }
-        return :pip_compile if req_files.any? { |f| f.end_with?(".in") }
+        return :pipenv if updating_pipfile?
+        return pyproject_resolver if updating_pyproject?
+        return :pip_compile if updating_in_file?
 
         if dependency.version && !exact_requirement?(reqs)
           subdependency_resolver
@@ -154,7 +163,6 @@ module Dependabot
           :requirements
         end
       end
-      # rubocop:enable Metrics/PerceivedComplexity
 
       def subdependency_resolver
         return :pipenv if pipfile_lock
@@ -164,6 +172,12 @@ module Dependabot
         raise "Claimed to be a sub-dependency, but no lockfile exists!"
       end
 
+      def pyproject_resolver
+        return :poetry if poetry_based?
+
+        :requirements
+      end
+
       def exact_requirement?(reqs)
         reqs = reqs.map { |r| r.fetch(:requirement) }
         reqs = reqs.compact
@@ -204,16 +218,14 @@ module Dependabot
       end
 
       def current_requirement_string
-        reqs = dependency.requirements
+        reqs = requirements
         return if reqs.none?
 
-        requirement =
-          case resolver_type
-          when :pipenv then reqs.find { |r| r[:file] == "Pipfile" }
-          when :poetry then reqs.find { |r| r[:file] == "pyproject.toml" }
-          when :pip_compile then reqs.find { |r| r[:file].end_with?(".in") }
-          when :requirements then reqs.find { |r| r[:file].end_with?(".txt") }
-          end
+        requirement = reqs.find do |r|
+          file = r[:file]
+
+          file == "Pipfile" || file == "pyproject.toml" || file.end_with?(".in") || file.end_with?(".txt")
+        end
 
         requirement&.fetch(:requirement)
       end
@@ -238,7 +250,7 @@ module Dependabot
         return ">= #{dependency.version}" if dependency.version
 
         version_for_requirement =
-          dependency.requirements.map { |r| r[:requirement] }.compact.
+          requirements.filter_map { |r| r[:requirement] }.
           reject { |req_string| req_string.start_with?("<") }.
           select { |req_string| req_string.match?(VERSION_REGEX) }.
           map { |req_string| req_string.match(VERSION_REGEX) }.
@@ -263,26 +275,53 @@ module Dependabot
         )
       end
 
-      def poetry_library?
-        return false unless pyproject
+      def poetry_based?
+        updating_pyproject? && !poetry_details.nil?
+      end
+
+      def library?
+        return unless updating_pyproject?
 
         # Hit PyPi and check whether there are details for a library with a
         # matching name and description
-        details = TomlRB.parse(pyproject.content).dig("tool", "poetry")
-        return false unless details
-
         index_response = Dependabot::RegistryClient.get(
-          url: "https://pypi.org/pypi/#{normalised_name(details['name'])}/json/"
+          url: "https://pypi.org/pypi/#{normalised_name(library_details['name'])}/json/"
         )
 
         return false unless index_response.status == 200
 
         pypi_info = JSON.parse(index_response.body)["info"] || {}
-        pypi_info["summary"] == details["description"]
+        pypi_info["summary"] == library_details["description"]
+      rescue Excon::Error::Timeout
+        false
       rescue URI::InvalidURIError
         false
       end
 
+      def updating_pipfile?
+        requirement_files.any?("Pipfile")
+      end
+
+      def updating_pyproject?
+        requirement_files.any?("pyproject.toml")
+      end
+
+      def updating_in_file?
+        requirement_files.any? { |f| f.end_with?(".in") }
+      end
+
+      def updating_requirements_file?
+        requirement_files.any? { |f| f =~ /\.txt$|\.in$/ }
+      end
+
+      def requirement_files
+        requirements.map { |r| r.fetch(:file) }
+      end
+
+      def requirements
+        dependency.requirements
+      end
+
       def normalised_name(name)
         NameNormaliser.normalise(name)
       end
@@ -307,6 +346,22 @@ module Dependabot
         dependency_files.find { |f| f.name == "poetry.lock" }
       end
 
+      def library_details
+        @library_details ||= poetry_details || standard_details
+      end
+
+      def poetry_details
+        @poetry_details ||= toml_content.dig("tool", "poetry")
+      end
+
+      def standard_details
+        @standard_details ||= toml_content["project"]
+      end
+
+      def toml_content
+        @toml_content ||= TomlRB.parse(pyproject.content)
+      end
+
       def pip_compile_files
         dependency_files.select { |f| f.name.end_with?(".in") }
       end

data/lib/dependabot/python/version.rb

@@ -16,9 +16,9 @@ module Dependabot
 
       # See https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions
       VERSION_PATTERN = 'v?([1-9][0-9]*!)?[0-9]+[0-9a-zA-Z]*(?>\.[0-9a-zA-Z]+)*' \
-                        '(-[0-9A-Za-z-]+(\.[0-9a-zA-Z-]+)*)?' \
+                        '(-[0-9A-Za-z]+(\.[0-9a-zA-Z]+)*)?' \
                         '(\+[0-9a-zA-Z]+(\.[0-9a-zA-Z]+)*)?'
-      ANCHORED_VERSION_PATTERN = /\A\s*(#{VERSION_PATTERN})?\s*\z/.freeze
+      ANCHORED_VERSION_PATTERN = /\A\s*(#{VERSION_PATTERN})?\s*\z/
 
       def self.correct?(version)
         return false if version.nil?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-python
 version: !ruby/object:Gem::Version
-  version: 0.211.0
+  version: 0.213.0
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-23 00:00:00.000000000 Z
+date: 2022-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,42 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.211.0
+        version: 0.213.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.211.0
-- !ruby/object:Gem::Dependency
-  name: debase
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.2.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.2.3
-- !ruby/object:Gem::Dependency
-  name: debase-ruby_core_source
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.10.16
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.10.16
+        version: 0.213.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.11.1
+        version: 3.13.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.11.1
+        version: 3.13.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,28 +114,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.35.1
+        version: 1.37.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.35.1
+        version: 1.37.1
 - !ruby/object:Gem::Dependency
-  name: ruby-debug-ide
+  name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.3
+        version: 1.15.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.3
+        version: 1.15.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -252,7 +224,7 @@ files:
 - lib/dependabot/python/file_fetcher.rb
 - lib/dependabot/python/file_parser.rb
 - lib/dependabot/python/file_parser/pipfile_files_parser.rb
-- lib/dependabot/python/file_parser/poetry_files_parser.rb
+- lib/dependabot/python/file_parser/pyproject_files_parser.rb
 - lib/dependabot/python/file_parser/python_requirement_parser.rb
 - lib/dependabot/python/file_parser/setup_file_parser.rb
 - lib/dependabot/python/file_updater.rb
@@ -265,6 +237,7 @@ files:
 - lib/dependabot/python/file_updater/requirement_file_updater.rb
 - lib/dependabot/python/file_updater/requirement_replacer.rb
 - lib/dependabot/python/file_updater/setup_file_sanitizer.rb
+- lib/dependabot/python/helpers.rb
 - lib/dependabot/python/metadata_finder.rb
 - lib/dependabot/python/name_normaliser.rb
 - lib/dependabot/python/native_helpers.rb
@@ -292,14 +265,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.1.0
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Python support for dependabot