dependabot-core 0.76.1

data/lib/dependabot/file_parsers/python/pip.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/dependency"
+require "dependabot/file_parsers/base"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/shared_helpers"
+require "dependabot/utils/python/requirement"
+require "dependabot/errors"
+
+module Dependabot
+  module FileParsers
+    module Python
+      class Pip < Dependabot::FileParsers::Base
+        require_relative "pip/pipfile_files_parser"
+        require_relative "pip/poetry_files_parser"
+        require_relative "pip/setup_file_parser"
+
+        POETRY_DEPENDENCY_TYPES =
+          %w(tool.poetry.dependencies tool.poetry.dev-dependencies).freeze
+        DEPENDENCY_GROUP_KEYS = [
+          {
+            pipfile: "packages",
+            lockfile: "default"
+          },
+          {
+            pipfile: "dev-packages",
+            lockfile: "develop"
+          }
+        ].freeze
+        REQUIREMENT_FILE_EVALUATION_ERRORS = %w(
+          InstallationError RequirementsFileParseError InvalidMarker
+          InvalidRequirement
+        ).freeze
+
+        def parse
+          dependency_set = DependencySet.new
+
+          dependency_set += pipenv_dependencies if pipfile
+          dependency_set += poetry_dependencies if using_poetry?
+          dependency_set += requirement_dependencies if requirement_files.any?
+          dependency_set += setup_file_dependencies if setup_file
+
+          dependency_set.dependencies
+        end
+
+        private
+
+        def requirement_files
+          dependency_files.select { |f| f.name.end_with?(".txt", ".in") }
+        end
+
+        def pipenv_dependencies
+          @pipenv_dependencies ||=
+            PipfileFilesParser.
+            new(dependency_files: dependency_files).
+            dependency_set
+        end
+
+        def poetry_dependencies
+          @poetry_dependencies ||=
+            PoetryFilesParser.
+            new(dependency_files: dependency_files).
+            dependency_set
+        end
+
+        def requirement_dependencies
+          dependencies = DependencySet.new
+          parsed_requirement_files.each do |dep|
+            # This isn't ideal, but currently the FileUpdater won't update
+            # deps that appear in a requirements.txt and Pipfile / Pipfile.lock
+            # and *aren't* a straight lockfile for the Pipfile
+            next if included_in_pipenv_deps?(normalised_name(dep["name"]))
+
+            # If a requirement has a `<` or `<=` marker then updating it is
+            # probably blocked. Ignore it.
+            next if dep["markers"].include?("<")
+
+            requirements =
+              if lockfile_for_pip_compile_file?(dep["file"]) then []
+              else
+                [{
+                  requirement: dep["requirement"],
+                  file: Pathname.new(dep["file"]).cleanpath.to_path,
+                  source: nil,
+                  groups: []
+                }]
+              end
+
+            dependencies <<
+              Dependency.new(
+                name: normalised_name(dep["name"]),
+                version: dep["version"]&.include?("*") ? nil : dep["version"],
+                requirements: requirements,
+                package_manager: "pip"
+              )
+          end
+          dependencies
+        end
+
+        def included_in_pipenv_deps?(dep_name)
+          return false unless pipfile
+
+          pipenv_dependencies.dependencies.map(&:name).include?(dep_name)
+        end
+
+        def setup_file_dependencies
+          @setup_file_dependencies ||=
+            SetupFileParser.
+            new(dependency_files: dependency_files).
+            dependency_set
+        end
+
+        def lockfile_for_pip_compile_file?(filename)
+          return false unless pip_compile_files.any?
+          return false unless filename.end_with?(".txt")
+
+          basename = filename.gsub(/\.txt$/, "")
+          pip_compile_files.any? { |f| f.name == basename + ".in" }
+        end
+
+        def parsed_requirement_files
+          SharedHelpers.in_a_temporary_directory do
+            write_temporary_dependency_files
+
+            requirements = SharedHelpers.run_helper_subprocess(
+              command: "pyenv exec python #{python_helper_path}",
+              function: "parse_requirements",
+              args: [Dir.pwd]
+            )
+
+            check_requirements(requirements)
+            requirements
+          end
+        rescue SharedHelpers::HelperSubprocessFailed => error
+          evaluation_errors = REQUIREMENT_FILE_EVALUATION_ERRORS
+          raise unless error.message.start_with?(*evaluation_errors)
+
+          raise Dependabot::DependencyFileNotEvaluatable, error.message
+        end
+
+        def check_requirements(requirements)
+          requirements.each do |dep|
+            next unless dep["requirement"]
+
+            Utils::Python::Requirement.new(dep["requirement"].split(","))
+          rescue Gem::Requirement::BadRequirementError => error
+            raise Dependabot::DependencyFileNotEvaluatable, error.message
+          end
+        end
+
+        def write_temporary_dependency_files
+          dependency_files.
+            reject { |f| f.name == ".python-version" }.
+            each do |file|
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, file.content)
+            end
+        end
+
+        def python_helper_path
+          project_root = File.join(File.dirname(__FILE__), "../../../..")
+          File.join(project_root, "helpers/python/run.py")
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalised_name(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def check_required_files
+          filenames = dependency_files.map(&:name)
+          return if filenames.any? { |name| name.end_with?(".txt", ".in") }
+          return if pipfile
+          return if pyproject
+          return if setup_file
+
+          raise "No requirements.txt or setup.py!"
+        end
+
+        def pipfile
+          @pipfile ||= get_original_file("Pipfile")
+        end
+
+        def pipfile_lock
+          @pipfile_lock ||= get_original_file("Pipfile.lock")
+        end
+
+        def using_poetry?
+          return false unless pyproject
+          return true if poetry_lock || pyproject_lock
+
+          !TomlRB.parse(pyproject.content).dig("tool", "poetry").nil?
+        rescue TomlRB::ParseError
+          raise Dependabot::DependencyFileNotParseable, pyproject.path
+        end
+
+        def pyproject
+          @pyproject ||= get_original_file("pyproject.toml")
+        end
+
+        def pyproject_lock
+          @pyproject_lock ||= get_original_file("pyproject.lock")
+        end
+
+        def poetry_lock
+          @poetry_lock ||= get_original_file("poetry.lock")
+        end
+
+        def setup_file
+          @setup_file ||= get_original_file("setup.py")
+        end
+
+        def pip_compile_files
+          @pip_compile_files ||=
+            dependency_files.select { |f| f.name.end_with?(".in") }
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/file_parsers/python/pip/pipfile_files_parser.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/dependency"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/file_parsers/python/pip"
+require "dependabot/errors"
+
+module Dependabot
+  module FileParsers
+    module Python
+      class Pip
+        class PipfileFilesParser
+          DEPENDENCY_GROUP_KEYS = [
+            {
+              pipfile: "packages",
+              lockfile: "default"
+            },
+            {
+              pipfile: "dev-packages",
+              lockfile: "develop"
+            }
+          ].freeze
+
+          def initialize(dependency_files:)
+            @dependency_files = dependency_files
+          end
+
+          def dependency_set
+            dependency_set = Dependabot::FileParsers::Base::DependencySet.new
+
+            dependency_set += pipfile_dependencies
+            dependency_set += pipfile_lock_dependencies
+
+            dependency_set
+          end
+
+          private
+
+          attr_reader :dependency_files
+
+          def pipfile_dependencies
+            dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+            DEPENDENCY_GROUP_KEYS.each do |keys|
+              next unless parsed_pipfile[keys[:pipfile]]
+
+              parsed_pipfile[keys[:pipfile]].map do |dep_name, req|
+                group = keys[:lockfile]
+                next unless req.is_a?(String) || req["version"]
+                if pipfile_lock && !dependency_version(dep_name, req, group)
+                  next
+                end
+
+                dependencies <<
+                  Dependency.new(
+                    name: normalised_name(dep_name),
+                    version: dependency_version(dep_name, req, group),
+                    requirements: [{
+                      requirement: req.is_a?(String) ? req : req["version"],
+                      file: pipfile.name,
+                      source: nil,
+                      groups: [group]
+                    }],
+                    package_manager: "pip"
+                  )
+              end
+            end
+
+            dependencies
+          end
+
+          # Create a DependencySet where each element has no requirement. Any
+          # requirements will be added when combining the DependencySet with
+          # other DependencySets.
+          def pipfile_lock_dependencies
+            dependencies = Dependabot::FileParsers::Base::DependencySet.new
+            return dependencies unless pipfile_lock
+
+            DEPENDENCY_GROUP_KEYS.map { |h| h.fetch(:lockfile) }.each do |key|
+              next unless parsed_pipfile_lock[key]
+
+              parsed_pipfile_lock[key].each do |dep_name, details|
+                version = case details
+                          when String then details
+                          when Hash then details["version"]
+                          end
+                next unless version
+
+                dependencies <<
+                  Dependency.new(
+                    name: dep_name,
+                    version: version&.gsub(/^===?/, ""),
+                    requirements: [],
+                    package_manager: "pip"
+                  )
+              end
+            end
+
+            dependencies
+          end
+
+          def dependency_version(dep_name, requirement, group)
+            req = version_from_hash_or_string(requirement)
+
+            if pipfile_lock
+              details = parsed_pipfile_lock.
+                        dig(group, normalised_name(dep_name))
+
+              version = version_from_hash_or_string(details)
+              version&.gsub(/^===?/, "")
+            elsif req.start_with?("==") && !req.include?("*")
+              req.strip.gsub(/^===?/, "")
+            end
+          end
+
+          def version_from_hash_or_string(obj)
+            case obj
+            when String then obj.strip
+            when Hash then obj["version"]
+            end
+          end
+
+          # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+          def normalised_name(name)
+            name.downcase.gsub(/[-_.]+/, "-")
+          end
+
+          def parsed_pipfile
+            @parsed_pipfile ||= TomlRB.parse(pipfile.content)
+          rescue TomlRB::ParseError
+            raise Dependabot::DependencyFileNotParseable, pipfile.path
+          end
+
+          def parsed_pipfile_lock
+            @parsed_pipfile_lock ||= JSON.parse(pipfile_lock.content)
+          rescue JSON::ParserError
+            raise Dependabot::DependencyFileNotParseable, pipfile_lock.path
+          end
+
+          def pipfile
+            @pipfile ||= dependency_files.find { |f| f.name == "Pipfile" }
+          end
+
+          def pipfile_lock
+            @pipfile_lock ||=
+              dependency_files.find { |f| f.name == "Pipfile.lock" }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/file_parsers/python/pip/poetry_files_parser.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/dependency"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/file_parsers/python/pip"
+require "dependabot/errors"
+
+module Dependabot
+  module FileParsers
+    module Python
+      class Pip
+        class PoetryFilesParser
+          POETRY_DEPENDENCY_TYPES = %w(dependencies dev-dependencies).freeze
+
+          def initialize(dependency_files:)
+            @dependency_files = dependency_files
+          end
+
+          def dependency_set
+            dependency_set = Dependabot::FileParsers::Base::DependencySet.new
+
+            dependency_set += pyproject_dependencies
+            dependency_set += lockfile_dependencies if lockfile
+
+            dependency_set
+          end
+
+          private
+
+          attr_reader :dependency_files
+
+          def pyproject_dependencies
+            dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+            POETRY_DEPENDENCY_TYPES.each do |type|
+              deps_hash = parsed_pyproject.dig("tool", "poetry", type) || {}
+
+              deps_hash.each do |name, req|
+                next if normalised_name(name) == "python"
+                next if req.is_a?(Hash) && req.key?("git")
+
+                dependencies <<
+                  Dependency.new(
+                    name: normalised_name(name),
+                    version: version_from_lockfile(name),
+                    requirements: [{
+                      requirement: req.is_a?(String) ? req : req["version"],
+                      file: pyproject.name,
+                      source: nil,
+                      groups: [type]
+                    }],
+                    package_manager: "pip"
+                  )
+              end
+            end
+
+            dependencies
+          end
+
+          # Create a DependencySet where each element has no requirement. Any
+          # requirements will be added when combining the DependencySet with
+          # other DependencySets.
+          def lockfile_dependencies
+            dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+            parsed_lockfile.fetch("package", []).each do |details|
+              next if details.dig("source", "type") == "git"
+
+              dependencies <<
+                Dependency.new(
+                  name: details.fetch("name"),
+                  version: details.fetch("version"),
+                  requirements: [],
+                  package_manager: "pip"
+                )
+            end
+
+            dependencies
+          end
+
+          def version_from_lockfile(dep_name)
+            return unless parsed_lockfile
+
+            parsed_lockfile.fetch("package", []).
+              find { |p| p.fetch("name") == normalised_name(dep_name) }&.
+              fetch("verison", nil)
+          end
+
+          # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+          def normalised_name(name)
+            name.downcase.gsub(/[-_.]+/, "-")
+          end
+
+          def parsed_pyproject
+            @parsed_pyproject ||= TomlRB.parse(pyproject.content)
+          rescue TomlRB::ParseError
+            raise Dependabot::DependencyFileNotParseable, pyproject.path
+          end
+
+          def parsed_pyproject_lock
+            @parsed_pyproject_lock ||= TomlRB.parse(pyproject_lock.content)
+          rescue TomlRB::ParseError
+            raise Dependabot::DependencyFileNotParseable, pyproject_lock.path
+          end
+
+          def parsed_poetry_lock
+            @parsed_poetry_lock ||= TomlRB.parse(poetry_lock.content)
+          rescue TomlRB::ParseError
+            raise Dependabot::DependencyFileNotParseable, poetry_lock.path
+          end
+
+          def pyproject
+            @pyproject ||=
+              dependency_files.find { |f| f.name == "pyproject.toml" }
+          end
+
+          def lockfile
+            poetry_lock || pyproject_lock
+          end
+
+          def parsed_lockfile
+            return parsed_poetry_lock if poetry_lock
+            return parsed_pyproject_lock if pyproject_lock
+          end
+
+          def pyproject_lock
+            @pyproject_lock ||=
+              dependency_files.find { |f| f.name == "pyproject.lock" }
+          end
+
+          def poetry_lock
+            @poetry_lock ||=
+              dependency_files.find { |f| f.name == "poetry.lock" }
+          end
+        end
+      end
+    end
+  end
+end