dependabot-python 0.215.0 → 0.216.0

data/lib/dependabot/python/python_versions.rb

@@ -4,7 +4,7 @@ module Dependabot
   module Python
     module PythonVersions
       PRE_INSTALLED_PYTHON_VERSIONS = %w(
-        3.11.0
+        3.11.3
       ).freeze
 
       # Due to an OpenSSL issue we can only install the following versions in
@@ -13,11 +13,11 @@ module Dependabot
       #
       # WARNING: 3.9.3 is purposefully omitted as it was recalled: https://www.python.org/downloads/release/python-393/
       SUPPORTED_VERSIONS = %w(
-        3.11.0
-        3.10.8 3.10.7 3.10.6 3.10.5 3.10.4 3.10.3 3.10.2 3.10.1 3.10.0
-        3.9.15 3.9.14 3.9.13 3.9.12 3.9.11 3.9.10 3.9.9 3.9.8 3.9.7 3.9.6 3.9.5 3.9.4 3.9.2 3.9.1 3.9.0
-        3.8.15 3.8.14 3.8.13 3.8.12 3.8.11 3.8.10 3.8.9 3.8.8 3.8.7 3.8.6 3.8.5 3.8.4 3.8.3 3.8.2 3.8.1 3.8.0
-        3.7.15 3.7.14 3.7.13 3.7.12 3.7.11 3.7.10 3.7.9 3.7.8 3.7.7 3.7.6 3.7.5 3.7.4 3.7.3 3.7.2 3.7.1 3.7.0
+        3.11.3 3.11.2 3.11.1 3.11.0
+        3.10.11 3.10.10 3.10.9 3.10.8 3.10.7 3.10.6 3.10.5 3.10.4 3.10.3 3.10.2 3.10.1 3.10.0
+        3.9.16 3.9.15 3.9.14 3.9.13 3.9.12 3.9.11 3.9.10 3.9.9 3.9.8 3.9.7 3.9.6 3.9.5 3.9.4 3.9.2 3.9.1 3.9.0
+        3.8.16 3.8.15 3.8.14 3.8.13 3.8.12 3.8.11 3.8.10 3.8.9 3.8.8 3.8.7 3.8.6 3.8.5 3.8.4 3.8.3 3.8.2 3.8.1 3.8.0
+        3.7.16 3.7.15 3.7.14 3.7.13 3.7.12 3.7.11 3.7.10 3.7.9 3.7.8 3.7.7 3.7.6 3.7.5 3.7.4 3.7.3 3.7.2 3.7.1 3.7.0
         3.6.15 3.6.14 3.6.13 3.6.12 3.6.11 3.6.10 3.6.9 3.6.8 3.6.7 3.6.6 3.6.5 3.6.4 3.6.3 3.6.2 3.6.1 3.6.0
         3.5.10 3.5.8 3.5.7 3.5.6 3.5.5 3.5.4 3.5.3
       ).freeze

data/lib/dependabot/python/requirement_parser.rb

@@ -9,7 +9,7 @@ module Dependabot
       VERSION = /([1-9][0-9]*!)?[0-9]+[a-zA-Z0-9\-_.*]*(\+[0-9a-zA-Z]+(\.[0-9a-zA-Z]+)*)?/
 
       REQUIREMENT = /(?<comparison>#{COMPARISON})\s*\\?\s*(?<version>#{VERSION})/
-      HASH = /--hash=(?<algorithm>.*?):(?<hash>.*?)(?=\s|$)/
+      HASH = /--hash=(?<algorithm>.*?):(?<hash>.*?)(?=\s|\\|$)/
       REQUIREMENTS = /#{REQUIREMENT}(\s*,\s*\\?\s*#{REQUIREMENT})*/
       HASHES = /#{HASH}(\s*\\?\s*#{HASH})*/
       MARKER_OP = /\s*(#{COMPARISON}|(\s*in)|(\s*not\s*in))/

data/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb

@@ -11,7 +11,7 @@ require "dependabot/python/file_updater/requirement_replacer"
 require "dependabot/python/file_updater/setup_file_sanitizer"
 require "dependabot/python/version"
 require "dependabot/shared_helpers"
-require "dependabot/python/helpers"
+require "dependabot/python/language_version_manager"
 require "dependabot/python/native_helpers"
 require "dependabot/python/python_versions"
 require "dependabot/python/name_normaliser"
@@ -71,7 +71,7 @@ module Dependabot
             SharedHelpers.in_a_temporary_directory do
               SharedHelpers.with_git_configured(credentials: credentials) do
                 write_temporary_dependency_files(updated_req: requirement)
-                Helpers.install_required_python(python_version)
+                language_version_manager.install_required_python
 
                 filenames_to_compile.each do |filename|
                   # Shell out to pip-compile.
@@ -173,6 +173,10 @@ module Dependabot
             raise GitDependenciesNotReachable, url
           end
 
+          raise Dependabot::OutOfDisk if error.message.end_with?("[Errno 28] No space left on device")
+
+          raise Dependabot::OutOfMemory if error.message.end_with?("MemoryError")
+
           raise
         end
         # rubocop:enable Metrics/AbcSize
@@ -233,7 +237,7 @@ module Dependabot
         end
 
         def new_resolver_supported?
-          python_version >= Python::Version.new("3.7")
+          language_version_manager.python_version >= Python::Version.new("3.7")
         end
 
         def pip_compile_options_fingerprint(options)
@@ -275,7 +279,7 @@ module Dependabot
 
         def run_pip_compile_command(command, fingerprint:)
           run_command(
-            "pyenv local #{Helpers.python_major_minor(python_version)}",
+            "pyenv local #{language_version_manager.python_major_minor}",
             fingerprint: "pyenv local <python_major_minor>"
           )
 
@@ -322,7 +326,7 @@ module Dependabot
           end
 
           # Overwrite the .python-version with updated content
-          File.write(".python-version", Helpers.python_major_minor(python_version))
+          File.write(".python-version", language_version_manager.python_major_minor)
 
           setup_files.each do |file|
             path = file.name
@@ -434,9 +438,9 @@ module Dependabot
           while (remaining_filenames = filenames - ordered_filenames).any?
             ordered_filenames +=
               remaining_filenames.
-              select do |fn|
+              reject do |fn|
                 unupdated_reqs = requirement_map[fn] - ordered_filenames
-                (unupdated_reqs & filenames).empty?
+                unupdated_reqs.intersect?(filenames)
               end
           end
 
@@ -479,41 +483,6 @@ module Dependabot
           ).parse.find { |d| d.name == dependency.name }&.version
         end
 
-        def python_version
-          @python_version ||=
-            user_specified_python_version ||
-            python_version_matching_imputed_requirements ||
-            PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
-        end
-
-        def user_specified_python_version
-          return unless python_requirement_parser.user_specified_requirements.any?
-
-          user_specified_requirements =
-            python_requirement_parser.user_specified_requirements.
-            map { |r| Python::Requirement.requirements_array(r) }
-          python_version_matching(user_specified_requirements)
-        end
-
-        def python_version_matching_imputed_requirements
-          compiled_file_python_requirement_markers =
-            python_requirement_parser.imputed_requirements.map do |r|
-              Dependabot::Python::Requirement.new(r)
-            end
-          python_version_matching(compiled_file_python_requirement_markers)
-        end
-
-        def python_version_matching(requirements)
-          PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.find do |version_string|
-            version = Python::Version.new(version_string)
-            requirements.all? do |req|
-              next req.any? { |r| r.satisfied_by?(version) } if req.is_a?(Array)
-
-              req.satisfied_by?(version)
-            end
-          end
-        end
-
         def python_requirement_parser
           @python_requirement_parser ||=
             FileParser::PythonRequirementParser.new(
@@ -521,8 +490,11 @@ module Dependabot
             )
         end
 
-        def pre_installed_python?(version)
-          PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version)
+        def language_version_manager
+          @language_version_manager ||=
+            LanguageVersionManager.new(
+              python_requirement_parser: python_requirement_parser
+            )
         end
 
         def setup_files

data/lib/dependabot/python/update_checker/pip_version_resolver.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "dependabot/python/language_version_manager"
 require "dependabot/python/update_checker"
 require "dependabot/python/update_checker/latest_version_finder"
 require "dependabot/python/file_parser/python_requirement_parser"
@@ -20,17 +21,17 @@ module Dependabot
         end
 
         def latest_resolvable_version
-          latest_version_finder.latest_version(python_version: python_version)
+          latest_version_finder.latest_version(python_version: language_version_manager.python_version)
         end
 
         def latest_resolvable_version_with_no_unlock
           latest_version_finder.
-            latest_version_with_no_unlock(python_version: python_version)
+            latest_version_with_no_unlock(python_version: language_version_manager.python_version)
         end
 
         def lowest_resolvable_security_fix_version
           latest_version_finder.
-            lowest_security_fix_version(python_version: python_version)
+            lowest_security_fix_version(python_version: language_version_manager.python_version)
         end
 
         private
@@ -49,45 +50,18 @@ module Dependabot
           )
         end
 
-        def python_version
-          @python_version ||=
-            user_specified_python_version ||
-            python_version_matching_imputed_requirements ||
-            PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
-        end
-
-        def user_specified_python_version
-          return unless python_requirement_parser.user_specified_requirements.any?
-
-          user_specified_requirements =
-            python_requirement_parser.user_specified_requirements.
-            map { |r| Python::Requirement.requirements_array(r) }
-          python_version_matching(user_specified_requirements)
-        end
-
-        def python_version_matching_imputed_requirements
-          compiled_file_python_requirement_markers =
-            python_requirement_parser.imputed_requirements.map do |r|
-              Dependabot::Python::Requirement.new(r)
-            end
-          python_version_matching(compiled_file_python_requirement_markers)
-        end
-
-        def python_version_matching(requirements)
-          PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.find do |version_string|
-            version = Python::Version.new(version_string)
-            requirements.all? do |req|
-              next req.any? { |r| r.satisfied_by?(version) } if req.is_a?(Array)
-
-              req.satisfied_by?(version)
-            end
-          end
-        end
-
         def python_requirement_parser
           @python_requirement_parser ||=
-            FileParser::PythonRequirementParser.
-            new(dependency_files: dependency_files)
+            FileParser::PythonRequirementParser.new(
+              dependency_files: dependency_files
+            )
+        end
+
+        def language_version_manager
+          @language_version_manager ||=
+            LanguageVersionManager.new(
+              python_requirement_parser: python_requirement_parser
+            )
         end
       end
     end

data/lib/dependabot/python/update_checker/pipenv_version_resolver.rb

@@ -175,7 +175,7 @@ module Dependabot
           end
 
           if error.message.include?("UnsupportedPythonVersion") &&
-             user_specified_python_requirement
+             language_version_manager.user_specified_python_version
             check_original_requirements_resolvable
 
             # The latest version of the dependency we're updating to needs a
@@ -231,7 +231,7 @@ module Dependabot
           end
 
           if error.message.include?("UnsupportedPythonVersion") &&
-             user_specified_python_requirement
+             language_version_manager.user_specified_python_version
             msg = clean_error_message(error.message).
                   lines.take_while { |l| !l.start_with?("File") }.join.strip
             raise if msg.empty?
@@ -239,8 +239,10 @@ module Dependabot
             raise DependencyFileNotResolvable, msg
           end
 
-          # NOTE: Pipenv masks the actualy error, see this issue for updates:
+          # NOTE: Pipenv masks the actual error, see this issue for updates:
           # https://github.com/pypa/pipenv/issues/2791
+          # TODO: This may no longer be reproducible on latest pipenv, see linked issue,
+          # so investigate when we next bump to newer pipenv...
           handle_pipenv_installation_error(error.message) if error.message.match?(PIPENV_INSTALLATION_ERROR_REGEX)
 
           # Raise an unhandled error, as this could be a problem with
@@ -290,7 +292,7 @@ module Dependabot
           end
 
           # Overwrite the .python-version with updated content
-          File.write(".python-version", Helpers.python_major_minor(python_version))
+          File.write(".python-version", language_version_manager.python_major_minor)
 
           setup_files.each do |file|
             path = file.name
@@ -320,7 +322,7 @@ module Dependabot
             nil
           end
 
-          Helpers.install_required_python(python_version)
+          language_version_manager.install_required_python
         end
 
         def sanitized_setup_file_content(file)
@@ -354,7 +356,7 @@ module Dependabot
         def update_python_requirement(pipfile_content)
           Python::FileUpdater::PipfilePreparer.
             new(pipfile_content: pipfile_content).
-            update_python_requirement(Helpers.python_major_minor(python_version))
+            update_python_requirement(language_version_manager.python_major_minor)
         end
 
         # rubocop:disable Metrics/PerceivedComplexity
@@ -398,57 +400,6 @@ module Dependabot
             replace_sources(credentials)
         end
 
-        def python_version
-          @python_version ||= python_version_from_supported_versions
-        end
-
-        def python_version_from_supported_versions
-          requirement_string =
-            if @using_python_two then "2.7.*"
-            elsif user_specified_python_requirement
-              parts = user_specified_python_requirement.split(".")
-              parts.fill("*", (parts.length)..2).join(".")
-            else
-              PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
-            end
-
-          # Ideally, the requirement is satisfied by a Python version we support
-          requirement =
-            Python::Requirement.requirements_array(requirement_string).first
-          version =
-            PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.
-            find { |v| requirement.satisfied_by?(Python::Version.new(v)) }
-          return version if version
-
-          # If not, and changing the patch version would fix things, we do that
-          # as the patch version is unlikely to affect resolution
-          requirement =
-            Python::Requirement.new(requirement_string.gsub(/\.\d+$/, ".*"))
-          version =
-            PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.
-            find { |v| requirement.satisfied_by?(Python::Version.new(v)) }
-          return version if version
-
-          # Otherwise we have to raise, giving details of the Python versions
-          # that Dependabot supports
-          msg = "Dependabot detected the following Python requirement " \
-                "for your project: '#{requirement_string}'.\n\nCurrently, the " \
-                "following Python versions are supported in Dependabot: " \
-                "#{PythonVersions::SUPPORTED_VERSIONS.join(', ')}."
-          raise DependencyFileNotResolvable, msg
-        end
-
-        def user_specified_python_requirement
-          python_requirement_parser.user_specified_requirements.first
-        end
-
-        def python_requirement_parser
-          @python_requirement_parser ||=
-            FileParser::PythonRequirementParser.new(
-              dependency_files: dependency_files
-            )
-        end
-
         def run_command(command, env: {})
           start = Time.now
           command = SharedHelpers.escape_command(command)
@@ -468,7 +419,7 @@ module Dependabot
         end
 
         def run_pipenv_command(command, env: pipenv_env_variables)
-          run_command("pyenv local #{Helpers.python_major_minor(python_version)}")
+          run_command("pyenv local #{language_version_manager.python_major_minor}")
           run_command(command, env: env)
         end
 
@@ -486,6 +437,20 @@ module Dependabot
           NameNormaliser.normalise(name)
         end
 
+        def python_requirement_parser
+          @python_requirement_parser ||=
+            FileParser::PythonRequirementParser.new(
+              dependency_files: dependency_files
+            )
+        end
+
+        def language_version_manager
+          @language_version_manager ||=
+            LanguageVersionManager.new(
+              python_requirement_parser: python_requirement_parser
+            )
+        end
+
         def pipfile
           dependency_files.find { |f| f.name == "Pipfile" }
         end

data/lib/dependabot/python/update_checker/poetry_version_resolver.rb

@@ -92,10 +92,10 @@ module Dependabot
                 write_temporary_dependency_files(updated_req: requirement)
                 add_auth_env_vars
 
-                Helpers.install_required_python(python_version)
+                language_version_manager.install_required_python
 
                 # use system git instead of the pure Python dulwich
-                unless python_version&.start_with?("3.6")
+                unless language_version_manager.python_version&.start_with?("3.6")
                   run_poetry_command("pyenv exec poetry config experimental.system-git-client true")
                 end
 
@@ -205,7 +205,7 @@ module Dependabot
           end
 
           # Overwrite the .python-version with updated content
-          File.write(".python-version", Helpers.python_major_minor(python_version)) if python_version
+          File.write(".python-version", language_version_manager.python_major_minor)
 
           # Overwrite the pyproject with updated content
           if update_pyproject
@@ -224,39 +224,10 @@ module Dependabot
             add_auth_env_vars(credentials)
         end
 
-        def python_version
-          requirements = python_requirement_parser.user_specified_requirements
-          requirements = requirements.
-                         map { |r| Python::Requirement.requirements_array(r) }
-
-          version = PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.find do |v|
-            requirements.all? do |reqs|
-              reqs.any? { |r| r.satisfied_by?(Python::Version.new(v)) }
-            end
-          end
-          return version if version
-
-          msg = "Dependabot detected the following Python requirements " \
-                "for your project: '#{requirements}'.\n\nCurrently, the " \
-                "following Python versions are supported in Dependabot: " \
-                "#{PythonVersions::SUPPORTED_VERSIONS.join(', ')}."
-          raise DependencyFileNotResolvable, msg
-        end
-
-        def python_requirement_parser
-          @python_requirement_parser ||=
-            FileParser::PythonRequirementParser.new(
-              dependency_files: dependency_files
-            )
-        end
-
-        def pre_installed_python?(version)
-          PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version)
-        end
-
         def updated_pyproject_content(updated_requirement:)
           content = pyproject.content
           content = sanitize_pyproject_content(content)
+          content = update_python_requirement(content)
           content = freeze_other_dependencies(content)
           content = set_target_dependency_req(content, updated_requirement)
           content
@@ -265,6 +236,7 @@ module Dependabot
         def sanitized_pyproject_content
           content = pyproject.content
           content = sanitize_pyproject_content(content)
+          content = update_python_requirement(content)
           content
         end
 
@@ -274,13 +246,18 @@ module Dependabot
             sanitize
         end
 
+        def update_python_requirement(pyproject_content)
+          Python::FileUpdater::PyprojectPreparer.
+            new(pyproject_content: pyproject_content).
+            update_python_requirement(language_version_manager.python_major_minor)
+        end
+
         def freeze_other_dependencies(pyproject_content)
           Python::FileUpdater::PyprojectPreparer.
             new(pyproject_content: pyproject_content, lockfile: lockfile).
             freeze_top_level_dependencies_except([dependency])
         end
 
-        # rubocop:disable Metrics/PerceivedComplexity
         def set_target_dependency_req(pyproject_content, updated_requirement)
           return pyproject_content unless updated_requirement
 
@@ -288,15 +265,15 @@ module Dependabot
           poetry_object = pyproject_object.dig("tool", "poetry")
 
           Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |type|
-            names = poetry_object[type]&.keys || []
-            pkg_name = names.find { |nm| normalise(nm) == dependency.name }
-            next unless pkg_name
-
-            if poetry_object.dig(type, pkg_name).is_a?(Hash)
-              poetry_object[type][pkg_name]["version"] = updated_requirement
-            else
-              poetry_object[type][pkg_name] = updated_requirement
-            end
+            dependencies = poetry_object[type]
+            next unless dependencies
+
+            update_dependency_requirement(dependencies, updated_requirement)
+          end
+
+          groups = poetry_object["group"]&.values || []
+          groups.each do |group_spec|
+            update_dependency_requirement(group_spec["dependencies"], updated_requirement)
           end
 
           # If this is a sub-dependency, add the new requirement
@@ -307,7 +284,18 @@ module Dependabot
 
           TomlRB.dump(pyproject_object)
         end
-        # rubocop:enable Metrics/PerceivedComplexity
+
+        def update_dependency_requirement(toml_node, requirement)
+          names = toml_node.keys
+          pkg_name = names.find { |nm| normalise(nm) == dependency.name }
+          return unless pkg_name
+
+          if toml_node[pkg_name].is_a?(Hash)
+            toml_node[pkg_name]["version"] = requirement
+          else
+            toml_node[pkg_name] = requirement
+          end
+        end
 
         def subdep_type
           category =
@@ -318,6 +306,20 @@ module Dependabot
           category == "dev" ? "dev-dependencies" : "dependencies"
         end
 
+        def python_requirement_parser
+          @python_requirement_parser ||=
+            FileParser::PythonRequirementParser.new(
+              dependency_files: dependency_files
+            )
+        end
+
+        def language_version_manager
+          @language_version_manager ||=
+            LanguageVersionManager.new(
+              python_requirement_parser: python_requirement_parser
+            )
+        end
+
         def pyproject
           dependency_files.find { |f| f.name == "pyproject.toml" }
         end

data/lib/dependabot/python/update_checker/requirements_updater.rb

@@ -88,12 +88,19 @@ module Dependabot
           case update_strategy
           when :widen_ranges then widen_pyproject_requirement(req)
           when :bump_versions then update_pyproject_version(req)
+          when :bump_versions_if_necessary then update_pyproject_version_if_needed(req)
           else raise "Unexpected update strategy: #{update_strategy}"
           end
         rescue UnfixableRequirement
           req.merge(requirement: :unfixable)
         end
 
+        def update_pyproject_version_if_needed(req)
+          return req if new_version_satisfies?(req)
+
+          update_pyproject_version(req)
+        end
+
         def update_pyproject_version(req)
           requirement_strings = req[:requirement].split(",").map(&:strip)
 
@@ -180,10 +187,14 @@ module Dependabot
           return req unless req.fetch(:requirement)
 
           case update_strategy
+          when :widen_ranges
+            widen_requirement(req)
           when :bump_versions
             update_requirement(req)
           when :bump_versions_if_necessary
             update_requirement_if_needed(req)
+          else
+            raise "Unexpected update strategy: #{update_strategy}"
           end
         end
 
@@ -212,6 +223,14 @@ module Dependabot
           req.merge(requirement: :unfixable)
         end
 
+        def widen_requirement(req)
+          return req if new_version_satisfies?(req)
+
+          new_requirement = widen_requirement_range(req[:requirement])
+
+          req.merge(requirement: new_requirement)
+        end
+
         def new_version_satisfies?(req)
           requirement_class.
             requirements_array(req.fetch(:requirement)).
@@ -256,8 +275,10 @@ module Dependabot
             next r.to_s if r.satisfied_by?(latest_resolvable_version)
 
             case op = r.requirements.first.first
-            when "<", "<="
-              "<" + update_greatest_version(r.to_s, latest_resolvable_version)
+            when "<"
+              "<" + update_greatest_version(r.requirements.first.last, latest_resolvable_version)
+            when "<="
+              "<=" + latest_resolvable_version.to_s
             when "!=", ">", ">="
               raise UnfixableRequirement
             else
@@ -329,14 +350,12 @@ module Dependabot
           end
         end
 
-        # Updates the version in a "<" or "<=" constraint to allow the given
-        # version
-        def update_greatest_version(req_string, version_to_be_permitted)
+        # Updates the version in a "<" constraint to allow the given version
+        def update_greatest_version(version, version_to_be_permitted)
           if version_to_be_permitted.is_a?(String)
             version_to_be_permitted =
               Python::Version.new(version_to_be_permitted)
           end
-          version = Python::Version.new(req_string.gsub(/<=?/, ""))
           version = version.release if version.prerelease?
 
           index_to_update = [