dependabot-core 0.78.0 → 0.79.0

data/lib/dependabot/file_parsers/python/pip/setup_file_parser.rb

@@ -1,164 +0,0 @@
-# frozen_string_literal: true
-
-require "dependabot/dependency"
-require "dependabot/errors"
-require "dependabot/file_parsers/base/dependency_set"
-require "dependabot/file_parsers/python/pip"
-require "dependabot/shared_helpers"
-
-module Dependabot
-  module FileParsers
-    module Python
-      class Pip
-        class SetupFileParser
-          INSTALL_REQUIRES_REGEX =
-            /install_requires\s*=\s*(\[.*?\])[,)\s]/m.freeze
-          SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*(\[.*?\])[,)\s]/m.freeze
-          TESTS_REQUIRE_REGEX = /tests_require\s*=\s*(\[.*?\])[,)\s]/m.freeze
-          EXTRAS_REQUIRE_REGEX = /extras_require\s*=\s*(\{.*?\})[,)\s]/m.freeze
-
-          def initialize(dependency_files:)
-            @dependency_files = dependency_files
-          end
-
-          def dependency_set
-            dependencies = Dependabot::FileParsers::Base::DependencySet.new
-
-            parsed_setup_file.each do |dep|
-              # If a requirement has a `<` or `<=` marker then updating it is
-              # probably blocked. Ignore it.
-              next if dep["markers"].include?("<")
-
-              # If the requirement is our inserted version, ignore it
-              # (we wouldn't be able to update it)
-              next if dep["version"] == "0.0.1+dependabot"
-
-              dependencies <<
-                Dependency.new(
-                  name: normalised_name(dep["name"]),
-                  version: dep["version"]&.include?("*") ? nil : dep["version"],
-                  requirements: [{
-                    requirement: dep["requirement"],
-                    file: Pathname.new(dep["file"]).cleanpath.to_path,
-                    source: nil,
-                    groups: [dep["requirement_type"]]
-                  }],
-                  package_manager: "pip"
-                )
-            end
-            dependencies
-          end
-
-          private
-
-          attr_reader :dependency_files
-
-          def parsed_setup_file
-            SharedHelpers.in_a_temporary_directory do
-              write_temporary_dependency_files
-
-              requirements = SharedHelpers.run_helper_subprocess(
-                command: "pyenv exec python #{python_helper_path}",
-                function: "parse_setup",
-                args: [Dir.pwd]
-              )
-
-              check_requirements(requirements)
-              requirements
-            end
-          rescue SharedHelpers::HelperSubprocessFailed => error
-            if error.message.start_with?("InstallationError")
-              raise Dependabot::DependencyFileNotEvaluatable, error.message
-            end
-
-            parsed_sanitized_setup_file
-          end
-
-          def parsed_sanitized_setup_file
-            SharedHelpers.in_a_temporary_directory do
-              write_sanitized_setup_file
-
-              requirements = SharedHelpers.run_helper_subprocess(
-                command: "pyenv exec python #{python_helper_path}",
-                function: "parse_setup",
-                args: [Dir.pwd]
-              )
-
-              check_requirements(requirements)
-              requirements
-            end
-          rescue SharedHelpers::HelperSubprocessFailed
-            # Assume there are no dependencies in setup.py files that fail to
-            # parse. This isn't ideal, and we should continue to improve
-            # parsing, but there are a *lot* of things that can go wrong at
-            # the moment!
-            []
-          end
-
-          def check_requirements(requirements)
-            requirements.each do |dep|
-              next unless dep["requirement"]
-
-              Utils::Python::Requirement.new(dep["requirement"].split(","))
-            rescue Gem::Requirement::BadRequirementError => error
-              raise Dependabot::DependencyFileNotEvaluatable, error.message
-            end
-          end
-
-          def write_temporary_dependency_files
-            dependency_files.
-              reject { |f| f.name == ".python-version" }.
-              each do |file|
-                path = file.name
-                FileUtils.mkdir_p(Pathname.new(path).dirname)
-                File.write(path, file.content)
-              end
-          end
-
-          # Write a setup.py with only entries for the requires fields.
-          #
-          # This sanitization is far from perfect (it will fail if any of the
-          # entries are dynamic), but it is an alternative approach to the one
-          # used in parser.py which sometimes succeeds when that has failed.
-          def write_sanitized_setup_file
-            original_content = setup_file.content
-
-            install_requires =
-              original_content.match(INSTALL_REQUIRES_REGEX)&.captures&.first
-            setup_requires =
-              original_content.match(SETUP_REQUIRES_REGEX)&.captures&.first
-            tests_require =
-              original_content.match(TESTS_REQUIRE_REGEX)&.captures&.first
-            extras_require =
-              original_content.match(EXTRAS_REQUIRE_REGEX)&.captures&.first
-
-            tmp = "from setuptools import setup\n\n"\
-                  "setup(name=\"sanitized-package\",version=\"0.0.1\","
-
-            tmp += "install_requires=#{install_requires}," if install_requires
-            tmp += "setup_requires=#{setup_requires}," if setup_requires
-            tmp += "tests_require=#{tests_require}," if tests_require
-            tmp += "extras_require=#{extras_require}," if extras_require
-            tmp += ")"
-
-            File.write("setup.py", tmp)
-          end
-
-          def python_helper_path
-            project_root = File.join(File.dirname(__FILE__), "../../../../..")
-            File.join(project_root, "helpers/python/run.py")
-          end
-
-          # See https://www.python.org/dev/peps/pep-0503/#normalized-names
-          def normalised_name(name)
-            name.downcase.gsub(/[-_.]+/, "-")
-          end
-
-          def setup_file
-            dependency_files.find { |f| f.name == "setup.py" }
-          end
-        end
-      end
-    end
-  end
-end

data/lib/dependabot/file_updaters/python/pip.rb

@@ -1,147 +0,0 @@
-# frozen_string_literal: true
-
-require "dependabot/file_updaters/base"
-require "dependabot/shared_helpers"
-
-module Dependabot
-  module FileUpdaters
-    module Python
-      class Pip < Dependabot::FileUpdaters::Base
-        require_relative "pip/pipfile_file_updater"
-        require_relative "pip/pip_compile_file_updater"
-        require_relative "pip/poetry_file_updater"
-        require_relative "pip/requirement_file_updater"
-
-        def self.updated_files_regex
-          [
-            /^Pipfile$/,
-            /^Pipfile\.lock$/,
-            /.*\.txt$/,
-            /.*\.in$/,
-            /^setup\.py$/,
-            /^pyproject\.toml$/,
-            /^pyproject\.lock$/
-          ]
-        end
-
-        def updated_dependency_files
-          updated_files =
-            case resolver_type
-            when :pipfile then updated_pipfile_based_files
-            when :poetry then updated_poetry_based_files
-            when :pip_compile then updated_pip_compile_based_files
-            when :requirements then updated_requirement_based_files
-            else raise "Unexpected resolver type: #{resolver_type}"
-            end
-
-          if updated_files.none? ||
-             updated_files.sort_by(&:name) == dependency_files.sort_by(&:name)
-            raise "No files have changed!"
-          end
-
-          updated_files
-        end
-
-        private
-
-        def resolver_type
-          reqs = dependencies.flat_map(&:requirements)
-          req_files = reqs.map { |r| r.fetch(:file) }
-
-          # If there are no requirements then this is a sub-dependency. It
-          # must come from one of Pipenv, Poetry or pip-tools, and can't come
-          # from the first two unless they have a lockfile.
-          return subdependency_resolver if reqs.none?
-
-          # Otherwise, this is a top-level dependency, and we can figure out
-          # which resolver to use based on the filename of its requirements
-          return :pipfile if req_files.any? { |f| f == "Pipfile" }
-          return :poetry if req_files.any? { |f| f == "pyproject.toml" }
-          return :pip_compile if req_files.any? { |f| f.end_with?(".in") }
-
-          # Finally, we should only ever be updating a requirements.txt file if
-          # some requirements have changed. Otherwise, this must be a case where
-          # we have a requirements.txt *and* some other resolver of which the
-          # dependency is a sub-dependency.
-          changed_reqs = reqs - dependencies.flat_map(&:previous_requirements)
-          changed_reqs.none? ? subdependency_resolver : :requirements
-        end
-
-        def subdependency_resolver
-          return :pipfile if pipfile_lock
-          return :poetry if poetry_lock || pyproject_lock
-          return :pip_compile if pip_compile_files.any?
-
-          raise "Claimed to be a sub-dependency, but no lockfile exists!"
-        end
-
-        def updated_pipfile_based_files
-          PipfileFileUpdater.new(
-            dependencies: dependencies,
-            dependency_files: dependency_files,
-            credentials: credentials
-          ).updated_dependency_files
-        end
-
-        def updated_poetry_based_files
-          PoetryFileUpdater.new(
-            dependencies: dependencies,
-            dependency_files: dependency_files,
-            credentials: credentials
-          ).updated_dependency_files
-        end
-
-        def updated_pip_compile_based_files
-          PipCompileFileUpdater.new(
-            dependencies: dependencies,
-            dependency_files: dependency_files,
-            credentials: credentials
-          ).updated_dependency_files
-        end
-
-        def updated_requirement_based_files
-          RequirementFileUpdater.new(
-            dependencies: dependencies,
-            dependency_files: dependency_files,
-            credentials: credentials
-          ).updated_dependency_files
-        end
-
-        def check_required_files
-          filenames = dependency_files.map(&:name)
-          return if filenames.any? { |name| name.end_with?(".txt", ".in") }
-          return if pipfile
-          return if pyproject
-          return if get_original_file("setup.py")
-
-          raise "No requirements.txt or setup.py!"
-        end
-
-        def pipfile
-          @pipfile ||= get_original_file("Pipfile")
-        end
-
-        def pipfile_lock
-          @pipfile_lock ||= get_original_file("Pipfile.lock")
-        end
-
-        def pyproject
-          @pyproject ||= get_original_file("pyproject.toml")
-        end
-
-        def pyproject_lock
-          @pyproject_lock ||= get_original_file("pyproject.lock")
-        end
-
-        def poetry_lock
-          @poetry_lock ||= get_original_file("poetry.lock")
-        end
-
-        def pip_compile_files
-          @pip_compile_files ||=
-            dependency_files.select { |f| f.name.end_with?(".in") }
-        end
-      end
-    end
-  end
-end

data/lib/dependabot/file_updaters/python/pip/pip_compile_file_updater.rb

@@ -1,363 +0,0 @@
-# frozen_string_literal: true
-
-require "python_requirement_parser"
-require "dependabot/file_fetchers/python/pip"
-require "dependabot/file_updaters/python/pip"
-require "dependabot/shared_helpers"
-
-# rubocop:disable Metrics/ClassLength
-module Dependabot
-  module FileUpdaters
-    module Python
-      class Pip
-        class PipCompileFileUpdater
-          require_relative "requirement_replacer"
-          require_relative "requirement_file_updater"
-          require_relative "setup_file_sanitizer"
-
-          UNSAFE_PACKAGES = %w(setuptools distribute pip).freeze
-
-          attr_reader :dependencies, :dependency_files, :credentials
-
-          def initialize(dependencies:, dependency_files:, credentials:)
-            @dependencies = dependencies
-            @dependency_files = dependency_files
-            @credentials = credentials
-          end
-
-          def updated_dependency_files
-            return @updated_dependency_files if @update_already_attempted
-
-            @update_already_attempted = true
-            @updated_dependency_files ||= fetch_updated_dependency_files
-          end
-
-          private
-
-          def dependency
-            # For now, we'll only ever be updating a single dependency
-            dependencies.first
-          end
-
-          def fetch_updated_dependency_files
-            updated_compiled_files = compile_new_requirement_files
-            updated_manifest_files = update_manifest_files
-
-            updated_files = updated_compiled_files + updated_manifest_files
-            updated_uncompiled_files = update_uncompiled_files(updated_files)
-
-            [
-              *updated_manifest_files,
-              *updated_compiled_files,
-              *updated_uncompiled_files
-            ]
-          end
-
-          def compile_new_requirement_files
-            SharedHelpers.in_a_temporary_directory do
-              write_updated_dependency_files
-
-              filenames_to_compile.each do |filename|
-                # Shell out to pip-compile, generate a new set of requirements.
-                # This is slow, as pip-compile needs to do installs.
-                run_command(
-                  "pyenv exec pip-compile #{pip_compile_options(filename)} "\
-                  "-P #{dependency.name} #{filename}"
-                )
-              end
-
-              dependency_files.map do |file|
-                next unless file.name.end_with?(".txt")
-
-                updated_content = File.read(file.name)
-
-                updated_content =
-                  replace_header_with_original(updated_content, file.content)
-                next if updated_content == file.content
-
-                file.dup.tap { |f| f.content = updated_content }
-              end.compact
-            end
-          end
-
-          def update_manifest_files
-            dependency_files.map do |file|
-              next unless file.name.end_with?(".in")
-
-              file = file.dup
-              updated_content = update_dependency_requirement(file)
-              next if updated_content == file.content
-
-              file.content = updated_content
-              file
-            end.compact
-          end
-
-          def update_uncompiled_files(updated_files)
-            updated_filenames = updated_files.map(&:name)
-            old_reqs = dependency.previous_requirements.
-                       reject { |r| updated_filenames.include?(r[:file]) }
-            new_reqs = dependency.requirements.
-                       reject { |r| updated_filenames.include?(r[:file]) }
-
-            return [] if new_reqs.none?
-
-            files = dependency_files.
-                    reject { |file| updated_filenames.include?(file.name) }
-
-            args = dependency.to_h
-            args = Hash[args.keys.map { |k| [k.to_sym, args[k]] }]
-            args[:requirements] = new_reqs
-            args[:previous_requirements] = old_reqs
-
-            RequirementFileUpdater.new(
-              dependencies: [Dependency.new(**args)],
-              dependency_files: files,
-              credentials: credentials
-            ).updated_dependency_files
-          end
-
-          def run_command(command)
-            command = command.dup
-            raw_response = nil
-            IO.popen(command, err: %i(child out)) do |process|
-              raw_response = process.read
-            end
-
-            # Raise an error with the output from the shell session if
-            # pip-compile returns a non-zero status
-            return if $CHILD_STATUS.success?
-
-            raise SharedHelpers::HelperSubprocessFailed.new(
-              raw_response,
-              command
-            )
-          rescue SharedHelpers::HelperSubprocessFailed => error
-            original_error ||= error
-            msg = error.message
-
-            relevant_error =
-              if error_suggests_bad_python_version?(msg) then original_error
-              else error
-              end
-
-            raise relevant_error unless error_suggests_bad_python_version?(msg)
-            raise relevant_error if File.exist?(".python-version")
-
-            command = "pyenv local 2.7.15 && " + command
-            retry
-          ensure
-            FileUtils.remove_entry(".python-version", true)
-          end
-
-          def error_suggests_bad_python_version?(message)
-            return true if message.include?("not find a version that satisfies")
-
-            message.include?('Command "python setup.py egg_info" failed')
-          end
-
-          def write_updated_dependency_files
-            dependency_files.each do |file|
-              next if file.name == ".python-version"
-
-              path = file.name
-              FileUtils.mkdir_p(Pathname.new(path).dirname)
-              File.write(path, freeze_dependency_requirement(file))
-            end
-
-            setup_files.each do |file|
-              path = file.name
-              FileUtils.mkdir_p(Pathname.new(path).dirname)
-              File.write(path, sanitized_setup_file_content(file))
-            end
-
-            setup_cfg_files.each do |file|
-              path = file.name
-              FileUtils.mkdir_p(Pathname.new(path).dirname)
-              File.write(path, "[metadata]\nname = sanitized-package\n")
-            end
-          end
-
-          def sanitized_setup_file_content(file)
-            @sanitized_setup_file_content ||= {}
-            if @sanitized_setup_file_content[file.name]
-              return @sanitized_setup_file_content[file.name]
-            end
-
-            @sanitized_setup_file_content[file.name] =
-              SetupFileSanitizer.
-              new(setup_file: file, setup_cfg: setup_cfg(file)).
-              sanitized_content
-          end
-
-          def setup_cfg(file)
-            dependency_files.find do |f|
-              f.name == file.name.sub(/\.py$/, ".cfg")
-            end
-          end
-
-          def freeze_dependency_requirement(file)
-            return file.content unless file.name.end_with?(".in")
-
-            old_req = dependency.previous_requirements.
-                      find { |r| r[:file] == file.name }
-
-            return file.content unless old_req
-            return file.content if old_req == "==#{dependency.version}"
-
-            RequirementReplacer.new(
-              content: file.content,
-              dependency_name: dependency.name,
-              old_requirement: old_req[:requirement],
-              new_requirement: "==#{dependency.version}"
-            ).updated_content
-          end
-
-          def update_dependency_requirement(file)
-            return file.content unless file.name.end_with?(".in")
-
-            old_req = dependency.previous_requirements.
-                      find { |r| r[:file] == file.name }
-            new_req = dependency.requirements.
-                      find { |r| r[:file] == file.name }
-            return file.content unless old_req&.fetch(:requirement)
-            return file.content if old_req == new_req
-
-            RequirementReplacer.new(
-              content: file.content,
-              dependency_name: dependency.name,
-              old_requirement: old_req[:requirement],
-              new_requirement: new_req[:requirement]
-            ).updated_content
-          end
-
-          def replace_header_with_original(updated_content, original_content)
-            original_header_lines =
-              original_content.lines.take_while { |l| l.start_with?("#") }
-
-            updated_content_lines =
-              updated_content.lines.drop_while { |l| l.start_with?("#") }
-
-            [*original_header_lines, *updated_content_lines].join
-          end
-
-          def pip_compile_options(filename)
-            current_requirements_file_name = filename.sub(/\.in$/, ".txt")
-
-            requirements_file =
-              dependency_files.
-              find { |f| f.name == current_requirements_file_name }
-
-            return unless requirements_file
-
-            options = ""
-
-            if requirements_file.content.include?("--hash=sha")
-              options += " --generate-hashes"
-            end
-
-            if includes_unsafe_packages?(requirements_file.content)
-              options += " --allow-unsafe"
-            end
-
-            unless requirements_file.content.include?("# via ")
-              options += " --no-annotate"
-            end
-
-            unless requirements_file.content.include?("autogenerated by pip-c")
-              options += " --no-header"
-            end
-
-            options.strip
-          end
-
-          def includes_unsafe_packages?(content)
-            UNSAFE_PACKAGES.any? { |n| content.match?(/^#{Regexp.quote(n)}==/) }
-          end
-
-          def filenames_to_compile
-            files_from_reqs =
-              dependency.requirements.
-              map { |r| r[:file] }.
-              select { |fn| fn.end_with?(".in") }
-
-            files_from_compiled_files =
-              pip_compile_files.map(&:name).select do |fn|
-                compiled_file = dependency_files.
-                                find { |f| f.name == fn.gsub(/\.in$/, ".txt") }
-                compiled_file_includes_dependency?(compiled_file)
-              end
-
-            filenames = [*files_from_reqs, *files_from_compiled_files].uniq
-
-            order_filenames_for_compilation(filenames)
-          end
-
-          def compiled_file_includes_dependency?(compiled_file)
-            return false unless compiled_file
-
-            regex = PythonRequirementParser::INSTALL_REQ_WITH_REQUIREMENT
-
-            matches = []
-            compiled_file.content.scan(regex) { matches << Regexp.last_match }
-            matches.any? { |m| normalise(m[:name]) == dependency.name }
-          end
-
-          # See https://www.python.org/dev/peps/pep-0503/#normalized-names
-          def normalise(name)
-            name.downcase.gsub(/[-_.]+/, "-")
-          end
-
-          # If the files we need to update require one another then we need to
-          # update them in the right order
-          def order_filenames_for_compilation(filenames)
-            ordered_filenames = []
-
-            while (remaining_filenames = filenames - ordered_filenames).any?
-              ordered_filenames +=
-                remaining_filenames.
-                select do |fn|
-                  unupdated_reqs = requirement_map[fn] - ordered_filenames
-                  (unupdated_reqs & filenames).empty?
-                end
-            end
-
-            ordered_filenames
-          end
-
-          def requirement_map
-            child_req_regex = FileFetchers::Python::Pip::CHILD_REQUIREMENT_REGEX
-            @requirement_map ||=
-              pip_compile_files.each_with_object({}) do |file, req_map|
-                paths = file.content.scan(child_req_regex).flatten
-                current_dir = File.dirname(file.name)
-
-                req_map[file.name] =
-                  paths.map do |path|
-                    path = File.join(current_dir, path) if current_dir != "."
-                    path = Pathname.new(path).cleanpath.to_path
-                    path = path.gsub(/\.txt$/, ".in")
-                    next if path == file.name
-
-                    path
-                  end.uniq.compact
-              end
-          end
-
-          def setup_files
-            dependency_files.select { |f| f.name.end_with?("setup.py") }
-          end
-
-          def pip_compile_files
-            dependency_files.select { |f| f.name.end_with?(".in") }
-          end
-
-          def setup_cfg_files
-            dependency_files.select { |f| f.name.end_with?("setup.cfg") }
-          end
-        end
-      end
-    end
-  end
-end
-# rubocop:enable Metrics/ClassLength