dependabot-uv 0.299.1

data/lib/dependabot/uv/file_fetcher.rb

@@ -0,0 +1,328 @@
+# typed: true
+# frozen_string_literal: true
+
+require "toml-rb"
+require "sorbet-runtime"
+
+require "dependabot/file_fetchers"
+require "dependabot/file_fetchers/base"
+require "dependabot/uv/language_version_manager"
+require "dependabot/uv/pip_compile_file_matcher"
+require "dependabot/uv/requirement_parser"
+require "dependabot/uv/file_parser/pyproject_files_parser"
+require "dependabot/uv/file_parser/python_requirement_parser"
+require "dependabot/errors"
+
+module Dependabot
+  module Uv
+    class FileFetcher < Dependabot::FileFetchers::Base
+      extend T::Sig
+      extend T::Helpers
+
+      CHILD_REQUIREMENT_REGEX = /^-r\s?(?<path>.*\.(?:txt|in))/
+      CONSTRAINT_REGEX = /^-c\s?(?<path>.*\.(?:txt|in))/
+      DEPENDENCY_TYPES = %w(packages dev-packages).freeze
+
+      def self.required_files_in?(filenames)
+        return true if filenames.any? { |name| name.end_with?(".txt", ".in") }
+
+        # If there is a directory of requirements return true
+        return true if filenames.include?("requirements")
+
+        # If this repo is using pyproject.toml return true
+        filenames.include?("pyproject.toml")
+      end
+
+      def self.required_files_message
+        "Repo must contain a requirements.txt, requirements.in, or pyproject.toml" \
+      end
+
+      def ecosystem_versions
+        # Hmm... it's weird that this calls file parser methods, but here we are in the file fetcher... for all
+        # ecosystems our goal is to extract the user specified versions, so we'll need to do file parsing... so should
+        # we move this `ecosystem_versions` metrics method to run in the file parser for all ecosystems? Downside is if
+        # file parsing blows up, this metric isn't emitted, but reality is we have to parse anyway... as we want to know
+        # the user-specified range of versions, not the version Dependabot chose to run.
+        python_requirement_parser = FileParser::PythonRequirementParser.new(dependency_files: files)
+        language_version_manager = LanguageVersionManager.new(python_requirement_parser: python_requirement_parser)
+        Dependabot.logger.info("Dependabot is using Python version '#{language_version_manager.python_major_minor}'.")
+        {
+          languages: {
+            python: {
+              # TODO: alternatively this could use `python_requirement_parser.user_specified_requirements` which
+              # returns an array... which we could flip to return a hash of manifest name => version
+              # string and then check for min/max versions... today it simply defaults to
+              # array.first which seems rather arbitrary.
+              "raw" => language_version_manager.user_specified_python_version || "unknown",
+              "max" => language_version_manager.python_major_minor || "unknown"
+            }
+          }
+        }
+      end
+
+      sig { override.returns(T::Array[DependencyFile]) }
+      def fetch_files
+        fetched_files = []
+
+        fetched_files += pyproject_files
+
+        fetched_files += requirements_in_files
+        fetched_files += requirement_files if requirements_txt_files.any?
+
+        fetched_files += project_files
+        fetched_files << python_version_file if python_version_file
+
+        uniq_files(fetched_files)
+      end
+
+      private
+
+      def uniq_files(fetched_files)
+        uniq_files = fetched_files.reject(&:support_file?).uniq
+        uniq_files += fetched_files
+                      .reject { |f| uniq_files.map(&:name).include?(f.name) }
+      end
+
+      def pyproject_files
+        [pyproject].compact
+      end
+
+      def requirement_files
+        [
+          *requirements_txt_files,
+          *child_requirement_txt_files,
+          *constraints_files
+        ]
+      end
+
+      def python_version_file
+        return @python_version_file if defined?(@python_version_file)
+
+        @python_version_file = fetch_support_file(".python-version")
+
+        return @python_version_file if @python_version_file
+        return if [".", "/"].include?(directory)
+
+        # Check the top-level for a .python-version file, too
+        reverse_path = Pathname.new(directory[0]).relative_path_from(directory)
+        @python_version_file =
+          fetch_support_file(File.join(reverse_path, ".python-version"))
+          &.tap { |f| f.name = ".python-version" }
+      end
+
+      def pyproject
+        return @pyproject if defined?(@pyproject)
+
+        @pyproject = fetch_file_if_present("pyproject.toml")
+      end
+
+      def requirements_txt_files
+        req_txt_and_in_files.select { |f| f.name.end_with?(".txt") }
+      end
+
+      def requirements_in_files
+        req_txt_and_in_files.select { |f| f.name.end_with?(".in") } +
+          child_requirement_in_files
+      end
+
+      def parsed_pyproject
+        raise "No pyproject.toml" unless pyproject
+
+        @parsed_pyproject ||= TomlRB.parse(pyproject.content)
+      rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
+        raise Dependabot::DependencyFileNotParseable, pyproject.path
+      end
+
+      def req_txt_and_in_files
+        return @req_txt_and_in_files if @req_txt_and_in_files
+
+        @req_txt_and_in_files = []
+
+        repo_contents
+          .select { |f| f.type == "file" }
+          .select { |f| f.name.end_with?(".txt", ".in") }
+          .reject { |f| f.size > 500_000 }
+          .map { |f| fetch_file_from_host(f.name) }
+          .select { |f| requirements_file?(f) }
+          .each { |f| @req_txt_and_in_files << f }
+
+        repo_contents
+          .select { |f| f.type == "dir" }
+          .each { |f| @req_txt_and_in_files += req_files_for_dir(f) }
+
+        @req_txt_and_in_files
+      end
+
+      def req_files_for_dir(requirements_dir)
+        dir = directory.gsub(%r{(^/|/$)}, "")
+        relative_reqs_dir =
+          requirements_dir.path.gsub(%r{^/?#{Regexp.escape(dir)}/?}, "")
+
+        repo_contents(dir: relative_reqs_dir)
+          .select { |f| f.type == "file" }
+          .select { |f| f.name.end_with?(".txt", ".in") }
+          .reject { |f| f.size > 500_000 }
+          .map { |f| fetch_file_from_host("#{relative_reqs_dir}/#{f.name}") }
+          .select { |f| requirements_file?(f) }
+      end
+
+      def child_requirement_txt_files
+        child_requirement_files.select { |f| f.name.end_with?(".txt") }
+      end
+
+      def child_requirement_in_files
+        child_requirement_files.select { |f| f.name.end_with?(".in") }
+      end
+
+      def child_requirement_files
+        @child_requirement_files ||=
+          begin
+            fetched_files = req_txt_and_in_files.dup
+            req_txt_and_in_files.flat_map do |requirement_file|
+              child_files = fetch_child_requirement_files(
+                file: requirement_file,
+                previously_fetched_files: fetched_files
+              )
+
+              fetched_files += child_files
+              child_files
+            end
+          end
+      end
+
+      def fetch_child_requirement_files(file:, previously_fetched_files:)
+        paths = file.content.scan(CHILD_REQUIREMENT_REGEX).flatten
+        current_dir = File.dirname(file.name)
+
+        paths.flat_map do |path|
+          path = File.join(current_dir, path) unless current_dir == "."
+          path = cleanpath(path)
+
+          next if previously_fetched_files.map(&:name).include?(path)
+          next if file.name == path
+
+          fetched_file = fetch_file_from_host(path)
+          grandchild_requirement_files = fetch_child_requirement_files(
+            file: fetched_file,
+            previously_fetched_files: previously_fetched_files + [file]
+          )
+          [fetched_file, *grandchild_requirement_files]
+        end.compact
+      end
+
+      def constraints_files
+        all_requirement_files = requirements_txt_files +
+                                child_requirement_txt_files
+
+        constraints_paths = all_requirement_files.map do |req_file|
+          current_dir = File.dirname(req_file.name)
+          paths = req_file.content.scan(CONSTRAINT_REGEX).flatten
+
+          paths.map do |path|
+            path = File.join(current_dir, path) unless current_dir == "."
+            cleanpath(path)
+          end
+        end.flatten.uniq
+
+        constraints_paths.map { |path| fetch_file_from_host(path) }
+      end
+
+      def project_files
+        project_files = T.let([], T::Array[Dependabot::DependencyFile])
+        unfetchable_deps = []
+
+        path_dependencies.each do |dep|
+          path = dep[:path]
+          project_files += fetch_project_file(path)
+        rescue Dependabot::DependencyFileNotFound
+          unfetchable_deps << "\"#{dep[:name]}\" at #{cleanpath(File.join(directory, dep[:file]))}"
+        end
+
+        raise Dependabot::PathDependenciesNotReachable, unfetchable_deps if unfetchable_deps.any?
+
+        project_files
+      end
+
+      def fetch_project_file(path)
+        project_files = []
+
+        path = cleanpath(File.join(path, "pyproject.toml")) unless sdist_or_wheel?(path)
+
+        return [] if path == "pyproject.toml" && pyproject
+
+        project_files << fetch_file_from_host(
+          path,
+          fetch_submodules: true
+        ).tap { |f| f.support_file = true }
+
+        project_files
+      end
+
+      def sdist_or_wheel?(path)
+        path.end_with?(".tar.gz", ".whl", ".zip")
+      end
+
+      def requirements_file?(file)
+        return false unless file.content.valid_encoding?
+        return true if file.name.match?(/requirements/x)
+
+        file.content.lines.all? do |line|
+          next true if line.strip.empty?
+          next true if line.strip.start_with?("#", "-r ", "-c ", "-e ", "--")
+
+          line.match?(RequirementParser::VALID_REQ_TXT_REQUIREMENT)
+        end
+      end
+
+      def path_dependencies
+        [
+          *requirement_txt_path_dependencies,
+          *requirement_in_path_dependencies
+        ]
+      end
+
+      def requirement_txt_path_dependencies
+        (requirements_txt_files + child_requirement_txt_files)
+          .map { |req_file| parse_requirement_path_dependencies(req_file) }
+          .flatten.uniq { |dep| dep[:path] }
+      end
+
+      def requirement_in_path_dependencies
+        requirements_in_files
+          .map { |req_file| parse_requirement_path_dependencies(req_file) }
+          .flatten.uniq { |dep| dep[:path] }
+      end
+
+      def parse_requirement_path_dependencies(req_file)
+        # If this is a pip-compile lockfile, rely on whatever path dependencies we found in the main manifest
+        return [] if pip_compile_file_matcher.lockfile_for_pip_compile_file?(req_file)
+
+        uneditable_reqs =
+          req_file.content
+                  .scan(/(?<name>^['"]?(?:file:)?(?<path>\..*?)(?=\[|#|'|"|$))/)
+                  .filter_map do |n, p|
+                    { name: n.strip, path: p.strip, file: req_file.name } unless p.include?("://")
+                  end
+
+        editable_reqs =
+          req_file.content
+                  .scan(/(?<name>^(?:-e)\s+['"]?(?:file:)?(?<path>.*?)(?=\[|#|'|"|$))/)
+                  .filter_map do |n, p|
+                    { name: n.strip, path: p.strip, file: req_file.name } unless p.include?("://") || p.include?("git@")
+                  end
+
+        uneditable_reqs + editable_reqs
+      end
+
+      def cleanpath(path)
+        Pathname.new(path).cleanpath.to_path
+      end
+
+      def pip_compile_file_matcher
+        @pip_compile_file_matcher ||= PipCompileFileMatcher.new(requirements_in_files)
+      end
+    end
+  end
+end
+
+Dependabot::FileFetchers.register("uv", Dependabot::Uv::FileFetcher)

data/lib/dependabot/uv/file_parser/pipfile_files_parser.rb

@@ -0,0 +1,192 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/dependency"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/uv/file_parser"
+require "dependabot/errors"
+require "dependabot/uv/name_normaliser"
+
+module Dependabot
+  module Uv
+    class FileParser
+      class PipfileFilesParser
+        extend T::Sig
+        DEPENDENCY_GROUP_KEYS = T.let([
+          {
+            pipfile: "packages",
+            lockfile: "default"
+          },
+          {
+            pipfile: "dev-packages",
+            lockfile: "develop"
+          }
+        ].freeze, T::Array[T::Hash[Symbol, String]])
+
+        sig { params(dependency_files: T::Array[Dependabot::DependencyFile]).void }
+        def initialize(dependency_files:)
+          @dependency_files = dependency_files
+        end
+
+        sig { returns(Dependabot::FileParsers::Base::DependencySet) }
+        def dependency_set
+          dependency_set = Dependabot::FileParsers::Base::DependencySet.new
+
+          dependency_set += pipfile_dependencies
+          dependency_set += pipfile_lock_dependencies
+
+          dependency_set
+        end
+
+        private
+
+        sig { returns(T::Array[Dependabot::DependencyFile]) }
+        attr_reader :dependency_files
+
+        sig { returns(Dependabot::FileParsers::Base::DependencySet) }
+        def pipfile_dependencies
+          dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+          DEPENDENCY_GROUP_KEYS.each do |keys|
+            next unless parsed_pipfile[T.must(keys[:pipfile])]
+
+            parsed_pipfile[T.must(keys[:pipfile])].map do |dep_name, req|
+              group = keys[:lockfile]
+              next unless specifies_version?(req)
+              next if git_or_path_requirement?(req)
+              next if pipfile_lock && !dependency_version(dep_name, req, T.must(group))
+
+              # Empty requirements are not allowed in Dependabot::Dependency and
+              # equivalent to "*" (latest available version)
+              req = "*" if req == ""
+
+              dependencies <<
+                Dependency.new(
+                  name: normalised_name(dep_name),
+                  version: dependency_version(dep_name, req, T.must(group)),
+                  requirements: [{
+                    requirement: req.is_a?(String) ? req : req["version"],
+                    file: T.must(pipfile).name,
+                    source: nil,
+                    groups: [group]
+                  }],
+                  package_manager: "uv",
+                  metadata: { original_name: dep_name }
+                )
+            end
+          end
+
+          dependencies
+        end
+
+        # Create a DependencySet where each element has no requirement. Any
+        # requirements will be added when combining the DependencySet with
+        # other DependencySets.
+        sig { returns(Dependabot::FileParsers::Base::DependencySet) }
+        def pipfile_lock_dependencies
+          dependencies = Dependabot::FileParsers::Base::DependencySet.new
+          return dependencies unless pipfile_lock
+
+          DEPENDENCY_GROUP_KEYS.map { |h| h.fetch(:lockfile) }.each do |key|
+            next unless parsed_pipfile_lock[key]
+
+            parsed_pipfile_lock[key].each do |dep_name, details|
+              version = case details
+                        when String then details
+                        when Hash then details["version"]
+                        end
+              next unless version
+              next if git_or_path_requirement?(details)
+
+              dependencies <<
+                Dependency.new(
+                  name: dep_name,
+                  version: version&.gsub(/^===?/, ""),
+                  requirements: [],
+                  package_manager: "uv",
+                  subdependency_metadata: [{ production: key != "develop" }]
+                )
+            end
+          end
+
+          dependencies
+        end
+
+        sig do
+          params(dep_name: String, requirement: T.any(String, T::Hash[String, T.untyped]),
+                 group: String).returns(T.nilable(String))
+        end
+        def dependency_version(dep_name, requirement, group)
+          req = version_from_hash_or_string(requirement)
+
+          if pipfile_lock
+            details = parsed_pipfile_lock
+                      .dig(group, normalised_name(dep_name))
+
+            version = version_from_hash_or_string(details)
+            version&.gsub(/^===?/, "")
+          elsif T.must(req).start_with?("==") && !T.must(req).include?("*")
+            T.must(req).strip.gsub(/^===?/, "")
+          end
+        end
+
+        sig do
+          params(obj: T.any(String, NilClass, T::Array[String], T::Hash[String, T.untyped])).returns(T.nilable(String))
+        end
+        def version_from_hash_or_string(obj)
+          case obj
+          when String then obj.strip
+          when Hash then obj["version"]
+          end
+        end
+
+        sig { params(req: T.any(String, T::Hash[String, T.untyped])).returns(T.any(T::Boolean, NilClass, String)) }
+        def specifies_version?(req)
+          return true if req.is_a?(String)
+
+          req["version"]
+        end
+
+        sig { params(req: T.any(String, T::Hash[String, T.untyped])).returns(T::Boolean) }
+        def git_or_path_requirement?(req)
+          return false unless req.is_a?(Hash)
+
+          %w(git path).any? { |k| req.key?(k) }
+        end
+
+        sig { params(name: String, extras: T::Array[String]).returns(String) }
+        def normalised_name(name, extras = [])
+          NameNormaliser.normalise_including_extras(name, extras)
+        end
+
+        sig { returns(T::Hash[String, T.untyped]) }
+        def parsed_pipfile
+          @parsed_pipfile ||= T.let(TomlRB.parse(T.must(pipfile).content), T.nilable(T::Hash[String, T.untyped]))
+        rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
+          raise Dependabot::DependencyFileNotParseable, T.must(pipfile).path
+        end
+
+        sig { returns(T::Hash[String, T.untyped]) }
+        def parsed_pipfile_lock
+          @parsed_pipfile_lock ||= T.let(JSON.parse(T.must(T.must(pipfile_lock).content)),
+                                         T.nilable(T::Hash[String, T.untyped]))
+        rescue JSON::ParserError
+          raise Dependabot::DependencyFileNotParseable, T.must(pipfile_lock).path
+        end
+
+        sig { returns(T.nilable(Dependabot::DependencyFile)) }
+        def pipfile
+          @pipfile ||= T.let(dependency_files.find { |f| f.name == "Pipfile" }, T.nilable(Dependabot::DependencyFile))
+        end
+
+        sig { returns(T.nilable(Dependabot::DependencyFile)) }
+        def pipfile_lock
+          @pipfile_lock ||= T.let(dependency_files.find { |f| f.name == "Pipfile.lock" },
+                                  T.nilable(Dependabot::DependencyFile))
+        end
+      end
+    end
+  end
+end