dependabot-python 0.79.0

data/lib/dependabot/python/file_parser.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/dependency"
+require "dependabot/file_parsers"
+require "dependabot/file_parsers/base"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/shared_helpers"
+require "dependabot/python/requirement"
+require "dependabot/errors"
+require "dependabot/python/native_helpers"
+
+module Dependabot
+  module Python
+    class FileParser < Dependabot::FileParsers::Base
+      require_relative "file_parser/pipfile_files_parser"
+      require_relative "file_parser/poetry_files_parser"
+      require_relative "file_parser/setup_file_parser"
+
+      POETRY_DEPENDENCY_TYPES =
+        %w(tool.poetry.dependencies tool.poetry.dev-dependencies).freeze
+      DEPENDENCY_GROUP_KEYS = [
+        {
+          pipfile: "packages",
+          lockfile: "default"
+        },
+        {
+          pipfile: "dev-packages",
+          lockfile: "develop"
+        }
+      ].freeze
+      REQUIREMENT_FILE_EVALUATION_ERRORS = %w(
+        InstallationError RequirementsFileParseError InvalidMarker
+        InvalidRequirement
+      ).freeze
+
+      def parse
+        dependency_set = DependencySet.new
+
+        dependency_set += pipenv_dependencies if pipfile
+        dependency_set += poetry_dependencies if using_poetry?
+        dependency_set += requirement_dependencies if requirement_files.any?
+        dependency_set += setup_file_dependencies if setup_file
+
+        dependency_set.dependencies
+      end
+
+      private
+
+      def requirement_files
+        dependency_files.select { |f| f.name.end_with?(".txt", ".in") }
+      end
+
+      def pipenv_dependencies
+        @pipenv_dependencies ||=
+          PipfileFilesParser.
+          new(dependency_files: dependency_files).
+          dependency_set
+      end
+
+      def poetry_dependencies
+        @poetry_dependencies ||=
+          PoetryFilesParser.
+          new(dependency_files: dependency_files).
+          dependency_set
+      end
+
+      def requirement_dependencies
+        dependencies = DependencySet.new
+        parsed_requirement_files.each do |dep|
+          # This isn't ideal, but currently the FileUpdater won't update
+          # deps that appear in a requirements.txt and Pipfile / Pipfile.lock
+          # and *aren't* a straight lockfile for the Pipfile
+          next if included_in_pipenv_deps?(normalised_name(dep["name"]))
+
+          # If a requirement has a `<` or `<=` marker then updating it is
+          # probably blocked. Ignore it.
+          next if dep["markers"].include?("<")
+
+          requirements =
+            if lockfile_for_pip_compile_file?(dep["file"]) then []
+            else
+              [{
+                requirement: dep["requirement"],
+                file: Pathname.new(dep["file"]).cleanpath.to_path,
+                source: nil,
+                groups: []
+              }]
+            end
+
+          dependencies <<
+            Dependency.new(
+              name: normalised_name(dep["name"]),
+              version: dep["version"]&.include?("*") ? nil : dep["version"],
+              requirements: requirements,
+              package_manager: "pip"
+            )
+        end
+        dependencies
+      end
+
+      def included_in_pipenv_deps?(dep_name)
+        return false unless pipfile
+
+        pipenv_dependencies.dependencies.map(&:name).include?(dep_name)
+      end
+
+      def setup_file_dependencies
+        @setup_file_dependencies ||=
+          SetupFileParser.
+          new(dependency_files: dependency_files).
+          dependency_set
+      end
+
+      def lockfile_for_pip_compile_file?(filename)
+        return false unless pip_compile_files.any?
+        return false unless filename.end_with?(".txt")
+
+        basename = filename.gsub(/\.txt$/, "")
+        pip_compile_files.any? { |f| f.name == basename + ".in" }
+      end
+
+      def parsed_requirement_files
+        SharedHelpers.in_a_temporary_directory do
+          write_temporary_dependency_files
+
+          requirements = SharedHelpers.run_helper_subprocess(
+            command: "pyenv exec python #{NativeHelpers.python_helper_path}",
+            function: "parse_requirements",
+            args: [Dir.pwd]
+          )
+
+          check_requirements(requirements)
+          requirements
+        end
+      rescue SharedHelpers::HelperSubprocessFailed => error
+        evaluation_errors = REQUIREMENT_FILE_EVALUATION_ERRORS
+        raise unless error.message.start_with?(*evaluation_errors)
+
+        raise Dependabot::DependencyFileNotEvaluatable, error.message
+      end
+
+      def check_requirements(requirements)
+        requirements.each do |dep|
+          next unless dep["requirement"]
+
+          Python::Requirement.new(dep["requirement"].split(","))
+        rescue Gem::Requirement::BadRequirementError => error
+          raise Dependabot::DependencyFileNotEvaluatable, error.message
+        end
+      end
+
+      def write_temporary_dependency_files
+        dependency_files.
+          reject { |f| f.name == ".python-version" }.
+          each do |file|
+            path = file.name
+            FileUtils.mkdir_p(Pathname.new(path).dirname)
+            File.write(path, file.content)
+          end
+      end
+
+      # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+      def normalised_name(name)
+        name.downcase.gsub(/[-_.]+/, "-")
+      end
+
+      def check_required_files
+        filenames = dependency_files.map(&:name)
+        return if filenames.any? { |name| name.end_with?(".txt", ".in") }
+        return if pipfile
+        return if pyproject
+        return if setup_file
+
+        raise "No requirements.txt or setup.py!"
+      end
+
+      def pipfile
+        @pipfile ||= get_original_file("Pipfile")
+      end
+
+      def pipfile_lock
+        @pipfile_lock ||= get_original_file("Pipfile.lock")
+      end
+
+      def using_poetry?
+        return false unless pyproject
+        return true if poetry_lock || pyproject_lock
+
+        !TomlRB.parse(pyproject.content).dig("tool", "poetry").nil?
+      rescue TomlRB::ParseError
+        raise Dependabot::DependencyFileNotParseable, pyproject.path
+      end
+
+      def pyproject
+        @pyproject ||= get_original_file("pyproject.toml")
+      end
+
+      def pyproject_lock
+        @pyproject_lock ||= get_original_file("pyproject.lock")
+      end
+
+      def poetry_lock
+        @poetry_lock ||= get_original_file("poetry.lock")
+      end
+
+      def setup_file
+        @setup_file ||= get_original_file("setup.py")
+      end
+
+      def pip_compile_files
+        @pip_compile_files ||=
+          dependency_files.select { |f| f.name.end_with?(".in") }
+      end
+    end
+  end
+end
+
+Dependabot::FileParsers.
+  register("pip", Dependabot::Python::FileParser)

data/lib/dependabot/python/file_parser/pipfile_files_parser.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/dependency"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/python/file_parser"
+require "dependabot/errors"
+
+module Dependabot
+  module Python
+    class FileParser
+      class PipfileFilesParser
+        DEPENDENCY_GROUP_KEYS = [
+          {
+            pipfile: "packages",
+            lockfile: "default"
+          },
+          {
+            pipfile: "dev-packages",
+            lockfile: "develop"
+          }
+        ].freeze
+
+        def initialize(dependency_files:)
+          @dependency_files = dependency_files
+        end
+
+        def dependency_set
+          dependency_set = Dependabot::FileParsers::Base::DependencySet.new
+
+          dependency_set += pipfile_dependencies
+          dependency_set += pipfile_lock_dependencies
+
+          dependency_set
+        end
+
+        private
+
+        attr_reader :dependency_files
+
+        def pipfile_dependencies
+          dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+          DEPENDENCY_GROUP_KEYS.each do |keys|
+            next unless parsed_pipfile[keys[:pipfile]]
+
+            parsed_pipfile[keys[:pipfile]].map do |dep_name, req|
+              group = keys[:lockfile]
+              next unless req.is_a?(String) || req["version"]
+              next if pipfile_lock && !dependency_version(dep_name, req, group)
+
+              dependencies <<
+                Dependency.new(
+                  name: normalised_name(dep_name),
+                  version: dependency_version(dep_name, req, group),
+                  requirements: [{
+                    requirement: req.is_a?(String) ? req : req["version"],
+                    file: pipfile.name,
+                    source: nil,
+                    groups: [group]
+                  }],
+                  package_manager: "pip"
+                )
+            end
+          end
+
+          dependencies
+        end
+
+        # Create a DependencySet where each element has no requirement. Any
+        # requirements will be added when combining the DependencySet with
+        # other DependencySets.
+        def pipfile_lock_dependencies
+          dependencies = Dependabot::FileParsers::Base::DependencySet.new
+          return dependencies unless pipfile_lock
+
+          DEPENDENCY_GROUP_KEYS.map { |h| h.fetch(:lockfile) }.each do |key|
+            next unless parsed_pipfile_lock[key]
+
+            parsed_pipfile_lock[key].each do |dep_name, details|
+              version = case details
+                        when String then details
+                        when Hash then details["version"]
+                        end
+              next unless version
+
+              dependencies <<
+                Dependency.new(
+                  name: dep_name,
+                  version: version&.gsub(/^===?/, ""),
+                  requirements: [],
+                  package_manager: "pip"
+                )
+            end
+          end
+
+          dependencies
+        end
+
+        def dependency_version(dep_name, requirement, group)
+          req = version_from_hash_or_string(requirement)
+
+          if pipfile_lock
+            details = parsed_pipfile_lock.
+                      dig(group, normalised_name(dep_name))
+
+            version = version_from_hash_or_string(details)
+            version&.gsub(/^===?/, "")
+          elsif req.start_with?("==") && !req.include?("*")
+            req.strip.gsub(/^===?/, "")
+          end
+        end
+
+        def version_from_hash_or_string(obj)
+          case obj
+          when String then obj.strip
+          when Hash then obj["version"]
+          end
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalised_name(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def parsed_pipfile
+          @parsed_pipfile ||= TomlRB.parse(pipfile.content)
+        rescue TomlRB::ParseError
+          raise Dependabot::DependencyFileNotParseable, pipfile.path
+        end
+
+        def parsed_pipfile_lock
+          @parsed_pipfile_lock ||= JSON.parse(pipfile_lock.content)
+        rescue JSON::ParserError
+          raise Dependabot::DependencyFileNotParseable, pipfile_lock.path
+        end
+
+        def pipfile
+          @pipfile ||= dependency_files.find { |f| f.name == "Pipfile" }
+        end
+
+        def pipfile_lock
+          @pipfile_lock ||=
+            dependency_files.find { |f| f.name == "Pipfile.lock" }
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/file_parser/poetry_files_parser.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/dependency"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/python/file_parser"
+require "dependabot/errors"
+
+module Dependabot
+  module Python
+    class FileParser
+      class PoetryFilesParser
+        POETRY_DEPENDENCY_TYPES = %w(dependencies dev-dependencies).freeze
+
+        def initialize(dependency_files:)
+          @dependency_files = dependency_files
+        end
+
+        def dependency_set
+          dependency_set = Dependabot::FileParsers::Base::DependencySet.new
+
+          dependency_set += pyproject_dependencies
+          dependency_set += lockfile_dependencies if lockfile
+
+          dependency_set
+        end
+
+        private
+
+        attr_reader :dependency_files
+
+        def pyproject_dependencies
+          dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+          POETRY_DEPENDENCY_TYPES.each do |type|
+            deps_hash = parsed_pyproject.dig("tool", "poetry", type) || {}
+
+            deps_hash.each do |name, req|
+              next if normalised_name(name) == "python"
+              next if req.is_a?(Hash) && req.key?("git")
+
+              dependencies <<
+                Dependency.new(
+                  name: normalised_name(name),
+                  version: version_from_lockfile(name),
+                  requirements: [{
+                    requirement: req.is_a?(String) ? req : req["version"],
+                    file: pyproject.name,
+                    source: nil,
+                    groups: [type]
+                  }],
+                  package_manager: "pip"
+                )
+            end
+          end
+
+          dependencies
+        end
+
+        # Create a DependencySet where each element has no requirement. Any
+        # requirements will be added when combining the DependencySet with
+        # other DependencySets.
+        def lockfile_dependencies
+          dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+          parsed_lockfile.fetch("package", []).each do |details|
+            next if details.dig("source", "type") == "git"
+
+            dependencies <<
+              Dependency.new(
+                name: details.fetch("name"),
+                version: details.fetch("version"),
+                requirements: [],
+                package_manager: "pip"
+              )
+          end
+
+          dependencies
+        end
+
+        def version_from_lockfile(dep_name)
+          return unless parsed_lockfile
+
+          parsed_lockfile.fetch("package", []).
+            find { |p| p.fetch("name") == normalised_name(dep_name) }&.
+            fetch("verison", nil)
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalised_name(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def parsed_pyproject
+          @parsed_pyproject ||= TomlRB.parse(pyproject.content)
+        rescue TomlRB::ParseError
+          raise Dependabot::DependencyFileNotParseable, pyproject.path
+        end
+
+        def parsed_pyproject_lock
+          @parsed_pyproject_lock ||= TomlRB.parse(pyproject_lock.content)
+        rescue TomlRB::ParseError
+          raise Dependabot::DependencyFileNotParseable, pyproject_lock.path
+        end
+
+        def parsed_poetry_lock
+          @parsed_poetry_lock ||= TomlRB.parse(poetry_lock.content)
+        rescue TomlRB::ParseError
+          raise Dependabot::DependencyFileNotParseable, poetry_lock.path
+        end
+
+        def pyproject
+          @pyproject ||=
+            dependency_files.find { |f| f.name == "pyproject.toml" }
+        end
+
+        def lockfile
+          poetry_lock || pyproject_lock
+        end
+
+        def parsed_lockfile
+          return parsed_poetry_lock if poetry_lock
+          return parsed_pyproject_lock if pyproject_lock
+        end
+
+        def pyproject_lock
+          @pyproject_lock ||=
+            dependency_files.find { |f| f.name == "pyproject.lock" }
+        end
+
+        def poetry_lock
+          @poetry_lock ||=
+            dependency_files.find { |f| f.name == "poetry.lock" }
+        end
+      end
+    end
+  end
+end