dependabot-python 0.214.0 → 0.216.0

data/lib/dependabot/python/language_version_manager.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "dependabot/logger"
+require "dependabot/python/version"
+
+module Dependabot
+  module Python
+    class LanguageVersionManager
+      def initialize(python_requirement_parser:)
+        @python_requirement_parser = python_requirement_parser
+      end
+
+      def install_required_python
+        # The leading space is important in the version check
+        return if SharedHelpers.run_shell_command("pyenv versions").include?(" #{python_major_minor}.")
+
+        if File.exist?("/usr/local/.pyenv/#{python_major_minor}.tar.gz")
+          SharedHelpers.run_shell_command(
+            "tar xzf /usr/local/.pyenv/#{python_major_minor}.tar.gz -C /usr/local/.pyenv/"
+          )
+          return if SharedHelpers.run_shell_command("pyenv versions").
+                    include?(" #{python_major_minor}.")
+        end
+
+        Dependabot.logger.info("Installing required Python #{python_version}.")
+        start = Time.now
+        SharedHelpers.run_shell_command("pyenv install -s #{python_version}")
+        SharedHelpers.run_shell_command("pyenv exec pip install --upgrade pip")
+        SharedHelpers.run_shell_command("pyenv exec pip install -r" \
+                                        "#{NativeHelpers.python_requirements_path}")
+        time_taken = Time.now - start
+        Dependabot.logger.info("Installing Python #{python_version} took #{time_taken}s.")
+      end
+
+      def python_major_minor
+        @python ||= Python::Version.new(python_version)
+        "#{@python.segments[0]}.#{@python.segments[1]}"
+      end
+
+      def python_version
+        @python_version ||= python_version_from_supported_versions
+      end
+
+      def python_requirement_string
+        if user_specified_python_version
+          if user_specified_python_version.start_with?(/\d/)
+            parts = user_specified_python_version.split(".")
+            parts.fill("*", (parts.length)..2).join(".")
+          else
+            user_specified_python_version
+          end
+        elsif python_version_matching_imputed_requirements
+          python_version_matching_imputed_requirements
+        else
+          PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
+        end
+      end
+
+      def python_version_from_supported_versions
+        requirement_string = python_requirement_string
+
+        # Ideally, the requirement is satisfied by a Python version we support
+        requirement =
+          Python::Requirement.requirements_array(requirement_string).first
+        version =
+          PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.
+          find { |v| requirement.satisfied_by?(Python::Version.new(v)) }
+        return version if version
+
+        # If not, and we're dealing with a simple version string
+        # and changing the patch version would fix things, we do that
+        # as the patch version is unlikely to affect resolution
+        if requirement_string.start_with?(/\d/)
+          requirement =
+            Python::Requirement.new(requirement_string.gsub(/\.\d+$/, ".*"))
+          version =
+            PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.
+            find { |v| requirement.satisfied_by?(Python::Version.new(v)) }
+          return version if version
+        end
+
+        # Otherwise we have to raise, giving details of the Python versions
+        # that Dependabot supports
+        msg = "Dependabot detected the following Python requirement " \
+              "for your project: '#{requirement_string}'.\n\nCurrently, the " \
+              "following Python versions are supported in Dependabot: " \
+              "#{PythonVersions::SUPPORTED_VERSIONS.join(', ')}."
+        raise DependencyFileNotResolvable, msg
+      end
+
+      def user_specified_python_version
+        @python_requirement_parser.user_specified_requirements.first
+      end
+
+      def python_version_matching_imputed_requirements
+        compiled_file_python_requirement_markers =
+          @python_requirement_parser.imputed_requirements.map do |r|
+            Dependabot::Python::Requirement.new(r)
+          end
+        python_version_matching(compiled_file_python_requirement_markers)
+      end
+
+      def python_version_matching(requirements)
+        PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.find do |version_string|
+          version = Python::Version.new(version_string)
+          requirements.all? do |req|
+            next req.any? { |r| r.satisfied_by?(version) } if req.is_a?(Array)
+
+            req.satisfied_by?(version)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/python_versions.rb

@@ -4,7 +4,7 @@ module Dependabot
   module Python
     module PythonVersions
       PRE_INSTALLED_PYTHON_VERSIONS = %w(
-        3.11.0
+        3.11.3
       ).freeze
 
       # Due to an OpenSSL issue we can only install the following versions in
@@ -13,11 +13,11 @@ module Dependabot
       #
       # WARNING: 3.9.3 is purposefully omitted as it was recalled: https://www.python.org/downloads/release/python-393/
       SUPPORTED_VERSIONS = %w(
-        3.11.0
-        3.10.8 3.10.7 3.10.6 3.10.5 3.10.4 3.10.3 3.10.2 3.10.1 3.10.0
-        3.9.15 3.9.14 3.9.13 3.9.12 3.9.11 3.9.10 3.9.9 3.9.8 3.9.7 3.9.6 3.9.5 3.9.4 3.9.2 3.9.1 3.9.0
-        3.8.15 3.8.14 3.8.13 3.8.12 3.8.11 3.8.10 3.8.9 3.8.8 3.8.7 3.8.6 3.8.5 3.8.4 3.8.3 3.8.2 3.8.1 3.8.0
-        3.7.15 3.7.14 3.7.13 3.7.12 3.7.11 3.7.10 3.7.9 3.7.8 3.7.7 3.7.6 3.7.5 3.7.4 3.7.3 3.7.2 3.7.1 3.7.0
+        3.11.3 3.11.2 3.11.1 3.11.0
+        3.10.11 3.10.10 3.10.9 3.10.8 3.10.7 3.10.6 3.10.5 3.10.4 3.10.3 3.10.2 3.10.1 3.10.0
+        3.9.16 3.9.15 3.9.14 3.9.13 3.9.12 3.9.11 3.9.10 3.9.9 3.9.8 3.9.7 3.9.6 3.9.5 3.9.4 3.9.2 3.9.1 3.9.0
+        3.8.16 3.8.15 3.8.14 3.8.13 3.8.12 3.8.11 3.8.10 3.8.9 3.8.8 3.8.7 3.8.6 3.8.5 3.8.4 3.8.3 3.8.2 3.8.1 3.8.0
+        3.7.16 3.7.15 3.7.14 3.7.13 3.7.12 3.7.11 3.7.10 3.7.9 3.7.8 3.7.7 3.7.6 3.7.5 3.7.4 3.7.3 3.7.2 3.7.1 3.7.0
         3.6.15 3.6.14 3.6.13 3.6.12 3.6.11 3.6.10 3.6.9 3.6.8 3.6.7 3.6.6 3.6.5 3.6.4 3.6.3 3.6.2 3.6.1 3.6.0
         3.5.10 3.5.8 3.5.7 3.5.6 3.5.5 3.5.4 3.5.3
       ).freeze

data/lib/dependabot/python/requirement_parser.rb

@@ -9,7 +9,7 @@ module Dependabot
       VERSION = /([1-9][0-9]*!)?[0-9]+[a-zA-Z0-9\-_.*]*(\+[0-9a-zA-Z]+(\.[0-9a-zA-Z]+)*)?/
 
       REQUIREMENT = /(?<comparison>#{COMPARISON})\s*\\?\s*(?<version>#{VERSION})/
-      HASH = /--hash=(?<algorithm>.*?):(?<hash>.*?)(?=\s|$)/
+      HASH = /--hash=(?<algorithm>.*?):(?<hash>.*?)(?=\s|\\|$)/
       REQUIREMENTS = /#{REQUIREMENT}(\s*,\s*\\?\s*#{REQUIREMENT})*/
       HASHES = /#{HASH}(\s*\\?\s*#{HASH})*/
       MARKER_OP = /\s*(#{COMPARISON}|(\s*in)|(\s*not\s*in))/

data/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb

@@ -11,7 +11,7 @@ require "dependabot/python/file_updater/requirement_replacer"
 require "dependabot/python/file_updater/setup_file_sanitizer"
 require "dependabot/python/version"
 require "dependabot/shared_helpers"
-require "dependabot/python/helpers"
+require "dependabot/python/language_version_manager"
 require "dependabot/python/native_helpers"
 require "dependabot/python/python_versions"
 require "dependabot/python/name_normaliser"
@@ -71,13 +71,17 @@ module Dependabot
             SharedHelpers.in_a_temporary_directory do
               SharedHelpers.with_git_configured(credentials: credentials) do
                 write_temporary_dependency_files(updated_req: requirement)
-                Helpers.install_required_python(python_version)
+                language_version_manager.install_required_python
 
                 filenames_to_compile.each do |filename|
                   # Shell out to pip-compile.
                   # This is slow, as pip-compile needs to do installs.
+                  options = pip_compile_options(filename)
+                  options_fingerprint = pip_compile_options_fingerprint(options)
+
                   run_pip_compile_command(
-                    "pyenv exec pip-compile -v #{pip_compile_options(filename)} -P #{dependency.name} #{filename}"
+                    "pyenv exec pip-compile -v #{options} -P #{dependency.name} #{filename}",
+                    fingerprint: "pyenv exec pip-compile -v #{options_fingerprint} -P <dependency_name> <filename>"
                   )
 
                   next if dependency.top_level?
@@ -91,7 +95,8 @@ module Dependabot
                   # update_not_possible.
                   write_original_manifest_files
                   run_pip_compile_command(
-                    "pyenv exec pip-compile #{pip_compile_options(filename)} #{filename}"
+                    "pyenv exec pip-compile #{options} #{filename}",
+                    fingerprint: "pyenv exec pip-compile #{options_fingerprint} <filename>"
                   )
                 end
 
@@ -168,6 +173,10 @@ module Dependabot
             raise GitDependenciesNotReachable, url
           end
 
+          raise Dependabot::OutOfDisk if error.message.end_with?("[Errno 28] No space left on device")
+
+          raise Dependabot::OutOfMemory if error.message.end_with?("MemoryError")
+
           raise
         end
         # rubocop:enable Metrics/AbcSize
@@ -183,8 +192,12 @@ module Dependabot
               write_temporary_dependency_files(update_requirement: false)
 
               filenames_to_compile.each do |filename|
+                options = pip_compile_options(filename)
+                options_fingerprint = pip_compile_options_fingerprint(options)
+
                 run_pip_compile_command(
-                  "pyenv exec pip-compile #{pip_compile_options(filename)} #{filename}"
+                  "pyenv exec pip-compile #{options} #{filename}",
+                  fingerprint: "pyenv exec pip-compile #{options_fingerprint} <filename>"
                 )
               end
 
@@ -204,7 +217,7 @@ module Dependabot
           end
         end
 
-        def run_command(command, env: python_env)
+        def run_command(command, env: python_env, fingerprint:)
           start = Time.now
           command = SharedHelpers.escape_command(command)
           stdout, process = Open3.capture2e(env, command)
@@ -216,6 +229,7 @@ module Dependabot
             message: stdout,
             error_context: {
               command: command,
+              fingerprint: fingerprint,
               time_taken: time_taken,
               process_exit_value: process.to_s
             }
@@ -223,7 +237,17 @@ module Dependabot
         end
 
         def new_resolver_supported?
-          python_version >= Python::Version.new("3.7")
+          language_version_manager.python_version >= Python::Version.new("3.7")
+        end
+
+        def pip_compile_options_fingerprint(options)
+          options.sub(
+            /--output-file=\S+/, "--output-file=<output_file>"
+          ).sub(
+            /--index-url=\S+/, "--index-url=<index_url>"
+          ).sub(
+            /--extra-index-url=\S+/, "--extra-index-url=<extra_index_url>"
+          )
         end
 
         def pip_compile_options(filename)
@@ -253,9 +277,13 @@ module Dependabot
             end
         end
 
-        def run_pip_compile_command(command)
-          run_command("pyenv local #{Helpers.python_major_minor(python_version)}")
-          run_command(command)
+        def run_pip_compile_command(command, fingerprint:)
+          run_command(
+            "pyenv local #{language_version_manager.python_major_minor}",
+            fingerprint: "pyenv local <python_major_minor>"
+          )
+
+          run_command(command, fingerprint: fingerprint)
         end
 
         def python_env
@@ -298,7 +326,7 @@ module Dependabot
           end
 
           # Overwrite the .python-version with updated content
-          File.write(".python-version", Helpers.python_major_minor(python_version))
+          File.write(".python-version", language_version_manager.python_major_minor)
 
           setup_files.each do |file|
             path = file.name
@@ -410,9 +438,9 @@ module Dependabot
           while (remaining_filenames = filenames - ordered_filenames).any?
             ordered_filenames +=
               remaining_filenames.
-              select do |fn|
+              reject do |fn|
                 unupdated_reqs = requirement_map[fn] - ordered_filenames
-                (unupdated_reqs & filenames).empty?
+                unupdated_reqs.intersect?(filenames)
               end
           end
 
@@ -455,41 +483,6 @@ module Dependabot
           ).parse.find { |d| d.name == dependency.name }&.version
         end
 
-        def python_version
-          @python_version ||=
-            user_specified_python_version ||
-            python_version_matching_imputed_requirements ||
-            PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
-        end
-
-        def user_specified_python_version
-          return unless python_requirement_parser.user_specified_requirements.any?
-
-          user_specified_requirements =
-            python_requirement_parser.user_specified_requirements.
-            map { |r| Python::Requirement.requirements_array(r) }
-          python_version_matching(user_specified_requirements)
-        end
-
-        def python_version_matching_imputed_requirements
-          compiled_file_python_requirement_markers =
-            python_requirement_parser.imputed_requirements.map do |r|
-              Dependabot::Python::Requirement.new(r)
-            end
-          python_version_matching(compiled_file_python_requirement_markers)
-        end
-
-        def python_version_matching(requirements)
-          PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.find do |version_string|
-            version = Python::Version.new(version_string)
-            requirements.all? do |req|
-              next req.any? { |r| r.satisfied_by?(version) } if req.is_a?(Array)
-
-              req.satisfied_by?(version)
-            end
-          end
-        end
-
         def python_requirement_parser
           @python_requirement_parser ||=
             FileParser::PythonRequirementParser.new(
@@ -497,8 +490,11 @@ module Dependabot
             )
         end
 
-        def pre_installed_python?(version)
-          PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version)
+        def language_version_manager
+          @language_version_manager ||=
+            LanguageVersionManager.new(
+              python_requirement_parser: python_requirement_parser
+            )
         end
 
         def setup_files

data/lib/dependabot/python/update_checker/pip_version_resolver.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "dependabot/python/language_version_manager"
 require "dependabot/python/update_checker"
 require "dependabot/python/update_checker/latest_version_finder"
 require "dependabot/python/file_parser/python_requirement_parser"
@@ -20,17 +21,17 @@ module Dependabot
         end
 
         def latest_resolvable_version
-          latest_version_finder.latest_version(python_version: python_version)
+          latest_version_finder.latest_version(python_version: language_version_manager.python_version)
         end
 
         def latest_resolvable_version_with_no_unlock
           latest_version_finder.
-            latest_version_with_no_unlock(python_version: python_version)
+            latest_version_with_no_unlock(python_version: language_version_manager.python_version)
         end
 
         def lowest_resolvable_security_fix_version
           latest_version_finder.
-            lowest_security_fix_version(python_version: python_version)
+            lowest_security_fix_version(python_version: language_version_manager.python_version)
         end
 
         private
@@ -49,45 +50,18 @@ module Dependabot
           )
         end
 
-        def python_version
-          @python_version ||=
-            user_specified_python_version ||
-            python_version_matching_imputed_requirements ||
-            PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
-        end
-
-        def user_specified_python_version
-          return unless python_requirement_parser.user_specified_requirements.any?
-
-          user_specified_requirements =
-            python_requirement_parser.user_specified_requirements.
-            map { |r| Python::Requirement.requirements_array(r) }
-          python_version_matching(user_specified_requirements)
-        end
-
-        def python_version_matching_imputed_requirements
-          compiled_file_python_requirement_markers =
-            python_requirement_parser.imputed_requirements.map do |r|
-              Dependabot::Python::Requirement.new(r)
-            end
-          python_version_matching(compiled_file_python_requirement_markers)
-        end
-
-        def python_version_matching(requirements)
-          PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.find do |version_string|
-            version = Python::Version.new(version_string)
-            requirements.all? do |req|
-              next req.any? { |r| r.satisfied_by?(version) } if req.is_a?(Array)
-
-              req.satisfied_by?(version)
-            end
-          end
-        end
-
         def python_requirement_parser
           @python_requirement_parser ||=
-            FileParser::PythonRequirementParser.
-            new(dependency_files: dependency_files)
+            FileParser::PythonRequirementParser.new(
+              dependency_files: dependency_files
+            )
+        end
+
+        def language_version_manager
+          @language_version_manager ||=
+            LanguageVersionManager.new(
+              python_requirement_parser: python_requirement_parser
+            )
         end
       end
     end

data/lib/dependabot/python/update_checker/pipenv_version_resolver.rb

@@ -175,7 +175,7 @@ module Dependabot
           end
 
           if error.message.include?("UnsupportedPythonVersion") &&
-             user_specified_python_requirement
+             language_version_manager.user_specified_python_version
             check_original_requirements_resolvable
 
             # The latest version of the dependency we're updating to needs a
@@ -231,7 +231,7 @@ module Dependabot
           end
 
           if error.message.include?("UnsupportedPythonVersion") &&
-             user_specified_python_requirement
+             language_version_manager.user_specified_python_version
             msg = clean_error_message(error.message).
                   lines.take_while { |l| !l.start_with?("File") }.join.strip
             raise if msg.empty?
@@ -239,8 +239,10 @@ module Dependabot
             raise DependencyFileNotResolvable, msg
           end
 
-          # NOTE: Pipenv masks the actualy error, see this issue for updates:
+          # NOTE: Pipenv masks the actual error, see this issue for updates:
           # https://github.com/pypa/pipenv/issues/2791
+          # TODO: This may no longer be reproducible on latest pipenv, see linked issue,
+          # so investigate when we next bump to newer pipenv...
           handle_pipenv_installation_error(error.message) if error.message.match?(PIPENV_INSTALLATION_ERROR_REGEX)
 
           # Raise an unhandled error, as this could be a problem with
@@ -290,7 +292,7 @@ module Dependabot
           end
 
           # Overwrite the .python-version with updated content
-          File.write(".python-version", Helpers.python_major_minor(python_version))
+          File.write(".python-version", language_version_manager.python_major_minor)
 
           setup_files.each do |file|
             path = file.name
@@ -320,7 +322,7 @@ module Dependabot
             nil
           end
 
-          Helpers.install_required_python(python_version)
+          language_version_manager.install_required_python
         end
 
         def sanitized_setup_file_content(file)
@@ -354,7 +356,7 @@ module Dependabot
         def update_python_requirement(pipfile_content)
           Python::FileUpdater::PipfilePreparer.
             new(pipfile_content: pipfile_content).
-            update_python_requirement(Helpers.python_major_minor(python_version))
+            update_python_requirement(language_version_manager.python_major_minor)
         end
 
         # rubocop:disable Metrics/PerceivedComplexity
@@ -398,57 +400,6 @@ module Dependabot
             replace_sources(credentials)
         end
 
-        def python_version
-          @python_version ||= python_version_from_supported_versions
-        end
-
-        def python_version_from_supported_versions
-          requirement_string =
-            if @using_python_two then "2.7.*"
-            elsif user_specified_python_requirement
-              parts = user_specified_python_requirement.split(".")
-              parts.fill("*", (parts.length)..2).join(".")
-            else
-              PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
-            end
-
-          # Ideally, the requirement is satisfied by a Python version we support
-          requirement =
-            Python::Requirement.requirements_array(requirement_string).first
-          version =
-            PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.
-            find { |v| requirement.satisfied_by?(Python::Version.new(v)) }
-          return version if version
-
-          # If not, and changing the patch version would fix things, we do that
-          # as the patch version is unlikely to affect resolution
-          requirement =
-            Python::Requirement.new(requirement_string.gsub(/\.\d+$/, ".*"))
-          version =
-            PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.
-            find { |v| requirement.satisfied_by?(Python::Version.new(v)) }
-          return version if version
-
-          # Otherwise we have to raise, giving details of the Python versions
-          # that Dependabot supports
-          msg = "Dependabot detected the following Python requirement " \
-                "for your project: '#{requirement_string}'.\n\nCurrently, the " \
-                "following Python versions are supported in Dependabot: " \
-                "#{PythonVersions::SUPPORTED_VERSIONS.join(', ')}."
-          raise DependencyFileNotResolvable, msg
-        end
-
-        def user_specified_python_requirement
-          python_requirement_parser.user_specified_requirements.first
-        end
-
-        def python_requirement_parser
-          @python_requirement_parser ||=
-            FileParser::PythonRequirementParser.new(
-              dependency_files: dependency_files
-            )
-        end
-
         def run_command(command, env: {})
           start = Time.now
           command = SharedHelpers.escape_command(command)
@@ -468,7 +419,7 @@ module Dependabot
         end
 
         def run_pipenv_command(command, env: pipenv_env_variables)
-          run_command("pyenv local #{Helpers.python_major_minor(python_version)}")
+          run_command("pyenv local #{language_version_manager.python_major_minor}")
           run_command(command, env: env)
         end
 
@@ -486,6 +437,20 @@ module Dependabot
           NameNormaliser.normalise(name)
         end
 
+        def python_requirement_parser
+          @python_requirement_parser ||=
+            FileParser::PythonRequirementParser.new(
+              dependency_files: dependency_files
+            )
+        end
+
+        def language_version_manager
+          @language_version_manager ||=
+            LanguageVersionManager.new(
+              python_requirement_parser: python_requirement_parser
+            )
+        end
+
         def pipfile
           dependency_files.find { |f| f.name == "Pipfile" }
         end