dependabot-uv 0.299.1

data/lib/dependabot/uv/file_parser/setup_file_parser.rb

@@ -0,0 +1,193 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "dependabot/dependency"
+require "dependabot/errors"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/shared_helpers"
+require "dependabot/uv/file_parser"
+require "dependabot/uv/native_helpers"
+require "dependabot/uv/name_normaliser"
+require "sorbet-runtime"
+
+module Dependabot
+  module Uv
+    class FileParser
+      class SetupFileParser
+        extend T::Sig
+        INSTALL_REQUIRES_REGEX = /install_requires\s*=\s*\[/m
+        SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*\[/m
+        TESTS_REQUIRE_REGEX = /tests_require\s*=\s*\[/m
+        EXTRAS_REQUIRE_REGEX = /extras_require\s*=\s*\{/m
+
+        CLOSING_BRACKET = T.let({ "[" => "]", "{" => "}" }.freeze, T.any(T.untyped, T.untyped))
+
+        sig { params(dependency_files: T::Array[Dependabot::DependencyFile]).void }
+        def initialize(dependency_files:)
+          @dependency_files = dependency_files
+        end
+
+        sig { returns(Dependabot::FileParsers::Base::DependencySet) }
+        def dependency_set
+          dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+          parsed_setup_file.each do |dep|
+            # If a requirement has a `<` or `<=` marker then updating it is
+            # probably blocked. Ignore it.
+            next if dep["markers"].include?("<")
+
+            # If the requirement is our inserted version, ignore it
+            # (we wouldn't be able to update it)
+            next if dep["version"] == "0.0.1+dependabot"
+
+            dependencies <<
+              Dependency.new(
+                name: normalised_name(dep["name"], dep["extras"]),
+                version: dep["version"]&.include?("*") ? nil : dep["version"],
+                requirements: [{
+                  requirement: dep["requirement"],
+                  file: Pathname.new(dep["file"]).cleanpath.to_path,
+                  source: nil,
+                  groups: [dep["requirement_type"]]
+                }],
+                package_manager: "uv"
+              )
+          end
+          dependencies
+        end
+
+        private
+
+        sig { returns(T::Array[Dependabot::DependencyFile]) }
+        attr_reader :dependency_files
+
+        sig { returns(T.untyped) }
+        def parsed_setup_file
+          SharedHelpers.in_a_temporary_directory do
+            write_temporary_dependency_files
+
+            requirements = SharedHelpers.run_helper_subprocess(
+              command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
+              function: "parse_setup",
+              args: [Dir.pwd]
+            )
+
+            check_requirements(requirements)
+            requirements
+          end
+        rescue SharedHelpers::HelperSubprocessFailed => e
+          raise Dependabot::DependencyFileNotEvaluatable, e.message if e.message.start_with?("InstallationError")
+
+          return [] unless setup_file
+
+          parsed_sanitized_setup_file
+        end
+
+        sig { returns(T.nilable(T.any(T::Hash[String, T.untyped], String, T::Array[T::Hash[String, T.untyped]]))) }
+        def parsed_sanitized_setup_file
+          SharedHelpers.in_a_temporary_directory do
+            write_sanitized_setup_file
+
+            requirements = SharedHelpers.run_helper_subprocess(
+              command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
+              function: "parse_setup",
+              args: [Dir.pwd]
+            )
+
+            check_requirements(requirements)
+            requirements
+          end
+        rescue SharedHelpers::HelperSubprocessFailed
+          # Assume there are no dependencies in setup.py files that fail to
+          # parse. This isn't ideal, and we should continue to improve
+          # parsing, but there are a *lot* of things that can go wrong at
+          # the moment!
+          []
+        end
+
+        sig { params(requirements: T.untyped).returns(T.untyped) }
+        def check_requirements(requirements)
+          requirements&.each do |dep|
+            next unless dep["requirement"]
+
+            Uv::Requirement.new(dep["requirement"].split(","))
+          rescue Gem::Requirement::BadRequirementError => e
+            raise Dependabot::DependencyFileNotEvaluatable, e.message
+          end
+        end
+
+        sig { void }
+        def write_temporary_dependency_files
+          dependency_files
+            .reject { |f| f.name == ".python-version" }
+            .each do |file|
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, file.content)
+            end
+        end
+
+        # Write a setup.py with only entries for the requires fields.
+        #
+        # This sanitization is far from perfect (it will fail if any of the
+        # entries are dynamic), but it is an alternative approach to the one
+        # used in parser.py which sometimes succeeds when that has failed.
+        sig { void }
+        def write_sanitized_setup_file
+          install_requires = get_regexed_req_array(INSTALL_REQUIRES_REGEX)
+          setup_requires = get_regexed_req_array(SETUP_REQUIRES_REGEX)
+          tests_require = get_regexed_req_array(TESTS_REQUIRE_REGEX)
+          extras_require = get_regexed_req_dict(EXTRAS_REQUIRE_REGEX)
+
+          tmp = "from setuptools import setup\n\n" \
+                "setup(name=\"sanitized-package\",version=\"0.0.1\","
+
+          tmp += "install_requires=#{install_requires}," if install_requires
+          tmp += "setup_requires=#{setup_requires}," if setup_requires
+          tmp += "tests_require=#{tests_require}," if tests_require
+          tmp += "extras_require=#{extras_require}," if extras_require
+          tmp += ")"
+
+          File.write("setup.py", tmp)
+        end
+
+        sig { params(regex: Regexp).returns(T.nilable(String)) }
+        def get_regexed_req_array(regex)
+          return unless (mch = setup_file.content.match(regex))
+
+          "[#{mch.post_match[0..closing_bracket_index(mch.post_match, '[')]}"
+        end
+
+        sig { params(regex: Regexp).returns(T.nilable(String)) }
+        def get_regexed_req_dict(regex)
+          return unless (mch = setup_file.content.match(regex))
+
+          "{#{mch.post_match[0..closing_bracket_index(mch.post_match, '{')]}"
+        end
+
+        sig { params(string: String, bracket: String).returns(Integer) }
+        def closing_bracket_index(string, bracket)
+          closes_required = 1
+
+          string.chars.each_with_index do |char, index|
+            closes_required += 1 if char == bracket
+            closes_required -= 1 if char == CLOSING_BRACKET.fetch(bracket)
+            return index if closes_required.zero?
+          end
+
+          0
+        end
+
+        sig { params(name: String, extras: T::Array[String]).returns(String) }
+        def normalised_name(name, extras)
+          NameNormaliser.normalise_including_extras(name, extras)
+        end
+
+        sig { returns(T.untyped) }
+        def setup_file
+          dependency_files.find { |f| f.name == "setup.py" }
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/uv/file_parser.rb

@@ -0,0 +1,437 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "dependabot/dependency"
+require "dependabot/file_parsers"
+require "dependabot/file_parsers/base"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/shared_helpers"
+require "dependabot/uv/requirement"
+require "dependabot/errors"
+require "dependabot/uv/language"
+require "dependabot/uv/native_helpers"
+require "dependabot/uv/name_normaliser"
+require "dependabot/uv/pip_compile_file_matcher"
+require "dependabot/uv/language_version_manager"
+require "dependabot/uv/package_manager"
+
+module Dependabot
+  module Uv
+    class FileParser < Dependabot::FileParsers::Base
+      extend T::Sig
+      require_relative "file_parser/pipfile_files_parser"
+      require_relative "file_parser/pyproject_files_parser"
+      require_relative "file_parser/setup_file_parser"
+      require_relative "file_parser/python_requirement_parser"
+
+      DEPENDENCY_GROUP_KEYS = T.let([
+        {
+          pipfile: "packages",
+          lockfile: "default"
+        },
+        {
+          pipfile: "dev-packages",
+          lockfile: "develop"
+        }
+      ].freeze, T::Array[T::Hash[Symbol, String]])
+      REQUIREMENT_FILE_EVALUATION_ERRORS = %w(
+        InstallationError RequirementsFileParseError InvalidMarker
+        InvalidRequirement ValueError RecursionError
+      ).freeze
+
+      # we use this placeholder version in case we are not able to detect any
+      # PIP version from shell, we are ensuring that the actual update is not blocked
+      # in any way if any metric collection exception start happening
+      UNDETECTED_PACKAGE_MANAGER_VERSION = "0.0"
+
+      sig { override.returns(T::Array[Dependabot::Dependency]) }
+      def parse
+        # TODO: setup.py from external dependencies is evaluated. Provide guards before removing this.
+        raise Dependabot::UnexpectedExternalCode if @reject_external_code
+
+        dependency_set = DependencySet.new
+
+        dependency_set += pyproject_file_dependencies if pyproject
+        dependency_set += requirement_dependencies if requirement_files.any?
+
+        dependency_set.dependencies
+      end
+
+      sig { override.returns(Ecosystem) }
+      def ecosystem
+        @ecosystem ||= T.let(
+          Ecosystem.new(
+            name: ECOSYSTEM,
+            package_manager: package_manager,
+            language: language
+          ),
+          T.nilable(Ecosystem)
+        )
+      end
+
+      private
+
+      sig { returns(Dependabot::Uv::LanguageVersionManager) }
+      def language_version_manager
+        @language_version_manager ||= T.let(LanguageVersionManager.new(python_requirement_parser:
+                                        python_requirement_parser), T.nilable(LanguageVersionManager))
+      end
+
+      sig { returns(Dependabot::Uv::FileParser::PythonRequirementParser) }
+      def python_requirement_parser
+        @python_requirement_parser ||= T.let(FileParser::PythonRequirementParser.new(dependency_files:
+                                         dependency_files), T.nilable(FileParser::PythonRequirementParser))
+      end
+
+      sig { returns(Ecosystem::VersionManager) }
+      def package_manager
+        if Dependabot::Experiments.enabled?(:enable_file_parser_python_local)
+          Dependabot.logger.info("Detected package manager : #{detected_package_manager.name}")
+        end
+
+        @package_manager ||= T.let(detected_package_manager, T.nilable(Dependabot::Ecosystem::VersionManager))
+      end
+
+      sig { returns(Ecosystem::VersionManager) }
+      def detected_package_manager
+        setup_python_environment if Dependabot::Experiments.enabled?(:enable_file_parser_python_local)
+
+        PackageManager.new(T.must(detect_pipcompile_version))
+      end
+
+      # Detects the version of pip-compile. If the version cannot be detected, it returns nil
+      sig { returns(T.nilable(String)) }
+      def detect_pipcompile_version
+        if pipcompile_in_file
+          package_manager = PackageManager::NAME
+
+          version = package_manager_version(package_manager)
+                    .to_s.split("version ").last&.split(")")&.first
+
+          log_if_version_malformed(package_manager, version)
+
+          # makes sure we have correct version format returned
+          version if version&.match?(/^\d+(?:\.\d+)*$/)
+        end
+      rescue StandardError
+        nil
+      end
+
+      sig { params(package_manager: String).returns(T.any(String, T.untyped)) }
+      def package_manager_version(package_manager)
+        version_info = SharedHelpers.run_shell_command("pyenv exec #{package_manager} --version")
+        Dependabot.logger.info("Package manager #{package_manager}, Info : #{version_info}")
+
+        version_info.match(/\d+(?:\.\d+)*/)&.to_s
+      rescue StandardError => e
+        Dependabot.logger.error(e.message)
+        nil
+      end
+
+      # setup python local setup on file parser stage
+      sig { returns(T.nilable(String)) }
+      def setup_python_environment
+        language_version_manager.install_required_python
+
+        SharedHelpers.run_shell_command("pyenv local #{language_version_manager.python_major_minor}")
+      rescue StandardError => e
+        Dependabot.logger.error(e.message)
+        nil
+      end
+
+      sig { params(package_manager: String, version: String).returns(T::Boolean) }
+      def log_if_version_malformed(package_manager, version)
+        # logs warning if malformed version is found
+        if version.match?(/^\d+(?:\.\d+)*$/)
+          true
+        else
+          Dependabot.logger.warn("Detected #{package_manager} with malformed version #{version}")
+          false
+        end
+      end
+
+      sig { returns(String) }
+      def python_raw_version
+        if Dependabot::Experiments.enabled?(:enable_file_parser_python_local)
+          Dependabot.logger.info("Detected python version: #{language_version_manager.python_version}")
+          Dependabot.logger.info("Detected python major minor version: #{language_version_manager.python_major_minor}")
+        end
+
+        language_version_manager.python_version
+      end
+
+      sig { returns(String) }
+      def python_command_version
+        language_version_manager.installed_version
+      end
+
+      sig { returns(T.nilable(Ecosystem::VersionManager)) }
+      def language
+        Language.new(
+          detected_version: python_raw_version,
+          raw_version: python_command_version
+        )
+      end
+
+      sig { returns(T::Array[Dependabot::DependencyFile]) }
+      def requirement_files
+        dependency_files.select { |f| f.name.end_with?(".txt", ".in") }
+      end
+
+      sig { returns(DependencySet) }
+      def pipenv_dependencies
+        @pipenv_dependencies ||= T.let(PipfileFilesParser.new(dependency_files:
+                                    dependency_files).dependency_set, T.nilable(DependencySet))
+      end
+
+      sig { returns(DependencySet) }
+      def pyproject_file_dependencies
+        @pyproject_file_dependencies ||= T.let(PyprojectFilesParser.new(dependency_files:
+                                          dependency_files).dependency_set, T.nilable(DependencySet))
+      end
+
+      sig { returns(DependencySet) }
+      def requirement_dependencies
+        dependencies = DependencySet.new
+        parsed_requirement_files.each do |dep|
+          # If a requirement has a `<`, `<=` or '==' marker then updating it is
+          # probably blocked. Ignore it.
+          next if blocking_marker?(dep)
+
+          name = dep["name"]
+          file = dep["file"]
+          version = dep["version"]
+          original_file = get_original_file(file)
+
+          requirements =
+            if original_file && pip_compile_file_matcher.lockfile_for_pip_compile_file?(original_file) then []
+            else
+              [{
+                requirement: dep["requirement"],
+                file: Pathname.new(file).cleanpath.to_path,
+                source: nil,
+                groups: group_from_filename(file)
+              }]
+            end
+
+          # PyYAML < 6.0 will cause `pip-compile` to fail due to incompatibility with Cython 3. Workaround it.
+          SharedHelpers.run_shell_command("pyenv exec pip install cython<3.0") if old_pyyaml?(name, version)
+
+          dependencies <<
+            Dependency.new(
+              name: normalised_name(name, dep["extras"]),
+              version: version&.include?("*") ? nil : version,
+              requirements: requirements,
+              package_manager: "uv"
+            )
+        end
+        dependencies
+      end
+
+      sig { params(name: T.nilable(String), version: T.nilable(String)).returns(T::Boolean) }
+      def old_pyyaml?(name, version)
+        major_version = version&.split(".")&.first
+        return false unless major_version
+
+        name == "pyyaml" && major_version < "6"
+      end
+
+      sig { params(filename: String).returns(T::Array[String]) }
+      def group_from_filename(filename)
+        if filename.include?("dev") then ["dev-dependencies"]
+        else
+          ["dependencies"]
+        end
+      end
+
+      sig { params(dep: T.untyped).returns(T::Boolean) }
+      def blocking_marker?(dep)
+        return false if dep["markers"] == "None"
+
+        marker = dep["markers"]
+        version = python_raw_version
+
+        if marker.include?("python_version")
+          !marker_satisfied?(marker, version)
+        else
+          return true if dep["markers"].include?("<")
+          return false if dep["markers"].include?(">")
+          return false if dep["requirement"].nil?
+
+          dep["requirement"].include?("<")
+        end
+      end
+
+      sig do
+        params(marker: T.untyped, python_version: T.any(String, Integer, Gem::Version)).returns(T::Boolean)
+      end
+      def marker_satisfied?(marker, python_version)
+        conditions = marker.split(/\s+(and|or)\s+/)
+
+        # Explicitly define the type of result as T::Boolean
+        result = T.let(evaluate_condition(conditions.shift, python_version), T::Boolean)
+
+        until conditions.empty?
+          operator = conditions.shift
+          next_condition = conditions.shift
+          next_result = evaluate_condition(next_condition, python_version)
+
+          result = if operator == "and"
+                     result && next_result
+                   else
+                     result || next_result
+                   end
+        end
+
+        result
+      end
+
+      sig do
+        params(condition: T.untyped,
+               python_version: T.any(String, Integer, Gem::Version)).returns(T::Boolean)
+      end
+      def evaluate_condition(condition, python_version)
+        operator, version = condition.match(/([<>=!]=?)\s*"?([\d.]+)"?/)&.captures
+
+        case operator
+        when "<"
+          Dependabot::Uv::Version.new(python_version) < Dependabot::Uv::Version.new(version)
+        when "<="
+          Dependabot::Uv::Version.new(python_version) <= Dependabot::Uv::Version.new(version)
+        when ">"
+          Dependabot::Uv::Version.new(python_version) > Dependabot::Uv::Version.new(version)
+        when ">="
+          Dependabot::Uv::Version.new(python_version) >= Dependabot::Uv::Version.new(version)
+        when "=="
+          Dependabot::Uv::Version.new(python_version) == Dependabot::Uv::Version.new(version)
+        else
+          false
+        end
+      end
+
+      sig { returns(DependencySet) }
+      def setup_file_dependencies
+        @setup_file_dependencies ||= T.let(SetupFileParser.new(dependency_files: dependency_files)
+                                    .dependency_set, T.nilable(DependencySet))
+      end
+
+      sig { returns(T.untyped) }
+      def parsed_requirement_files
+        SharedHelpers.in_a_temporary_directory do
+          write_temporary_dependency_files
+
+          requirements = SharedHelpers.run_helper_subprocess(
+            command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
+            function: "parse_requirements",
+            args: [Dir.pwd]
+          )
+
+          check_requirements(requirements)
+          requirements
+        end
+      rescue SharedHelpers::HelperSubprocessFailed => e
+        evaluation_errors = REQUIREMENT_FILE_EVALUATION_ERRORS
+        raise unless e.message.start_with?(*evaluation_errors)
+
+        raise Dependabot::DependencyFileNotEvaluatable, e.message
+      end
+
+      sig { params(requirements: T.untyped).returns(T.untyped) }
+      def check_requirements(requirements)
+        requirements.each do |dep|
+          next unless dep["requirement"]
+
+          Uv::Requirement.new(dep["requirement"].split(","))
+        rescue Gem::Requirement::BadRequirementError => e
+          raise Dependabot::DependencyFileNotEvaluatable, e.message
+        end
+      end
+
+      sig { returns(T::Boolean) }
+      def pipcompile_in_file
+        requirement_files.any? { |f| f.name.end_with?(PackageManager::MANIFEST_FILENAME) }
+      end
+
+      sig { returns(T::Array[Dependabot::DependencyFile]) }
+      def write_temporary_dependency_files
+        dependency_files
+          .reject { |f| f.name == ".python-version" }
+          .each do |file|
+            path = file.name
+            FileUtils.mkdir_p(Pathname.new(path).dirname)
+            File.write(path, remove_imports(file))
+          end
+      end
+
+      sig { params(file: T.untyped).returns(T.untyped) }
+      def remove_imports(file)
+        return file.content if file.path.end_with?(".tar.gz", ".whl", ".zip")
+
+        file.content.lines
+            .reject { |l| l.match?(/^['"]?(?<path>\..*?)(?=\[|#|'|"|$)/) }
+            .reject { |l| l.match?(/^(?:-e)\s+['"]?(?<path>.*?)(?=\[|#|'|"|$)/) }
+            .join
+      end
+
+      sig { params(name: String, extras: T::Array[String]).returns(String) }
+      def normalised_name(name, extras = [])
+        NameNormaliser.normalise_including_extras(name, extras)
+      end
+
+      sig { override.returns(T.untyped) }
+      def check_required_files
+        filenames = dependency_files.map(&:name)
+        return if filenames.any? { |name| name.end_with?(".txt", ".in") }
+        return if pipfile
+        return if pyproject
+        return if setup_file
+        return if setup_cfg_file
+
+        raise "Missing required files!"
+      end
+
+      sig { returns(T.nilable(Dependabot::DependencyFile)) }
+      def pipfile
+        @pipfile ||= T.let(get_original_file("Pipfile"), T.nilable(Dependabot::DependencyFile))
+      end
+
+      sig { returns(T.nilable(Dependabot::DependencyFile)) }
+      def pipfile_lock
+        @pipfile_lock ||= T.let(get_original_file("Pipfile.lock"), T.nilable(Dependabot::DependencyFile))
+      end
+
+      sig { returns(T.nilable(Dependabot::DependencyFile)) }
+      def pyproject
+        @pyproject ||= T.let(get_original_file("pyproject.toml"), T.nilable(Dependabot::DependencyFile))
+      end
+
+      sig { returns(T.nilable(Dependabot::DependencyFile)) }
+      def poetry_lock
+        @poetry_lock ||= T.let(get_original_file("poetry.lock"), T.nilable(Dependabot::DependencyFile))
+      end
+
+      sig { returns(T.nilable(Dependabot::DependencyFile)) }
+      def setup_file
+        @setup_file ||= T.let(get_original_file("setup.py"), T.nilable(Dependabot::DependencyFile))
+      end
+
+      sig { returns(T.nilable(Dependabot::DependencyFile)) }
+      def setup_cfg_file
+        @setup_cfg_file ||= T.let(get_original_file("setup.cfg"), T.nilable(Dependabot::DependencyFile))
+      end
+
+      sig { returns(T::Array[Dependabot::Uv::Requirement]) }
+      def pip_compile_files
+        @pip_compile_files ||= T.let(dependency_files.select { |f| f.name.end_with?(".in") }, T.untyped)
+      end
+
+      sig { returns(Dependabot::Uv::PipCompileFileMatcher) }
+      def pip_compile_file_matcher
+        @pip_compile_file_matcher ||= T.let(PipCompileFileMatcher.new(pip_compile_files),
+                                            T.nilable(Dependabot::Uv::PipCompileFileMatcher))
+      end
+    end
+  end
+end
+
+Dependabot::FileParsers.register("uv", Dependabot::Uv::FileParser)