dependabot-python 0.79.0

data/lib/dependabot/python/file_parser/setup_file_parser.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require "dependabot/dependency"
+require "dependabot/errors"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/shared_helpers"
+require "dependabot/python/file_parser"
+require "dependabot/python/native_helpers"
+
+module Dependabot
+  module Python
+    class FileParser
+      class SetupFileParser
+        INSTALL_REQUIRES_REGEX =
+          /install_requires\s*=\s*(\[.*?\])[,)\s]/m.freeze
+        SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*(\[.*?\])[,)\s]/m.freeze
+        TESTS_REQUIRE_REGEX = /tests_require\s*=\s*(\[.*?\])[,)\s]/m.freeze
+        EXTRAS_REQUIRE_REGEX = /extras_require\s*=\s*(\{.*?\})[,)\s]/m.freeze
+
+        def initialize(dependency_files:)
+          @dependency_files = dependency_files
+        end
+
+        def dependency_set
+          dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+          parsed_setup_file.each do |dep|
+            # If a requirement has a `<` or `<=` marker then updating it is
+            # probably blocked. Ignore it.
+            next if dep["markers"].include?("<")
+
+            # If the requirement is our inserted version, ignore it
+            # (we wouldn't be able to update it)
+            next if dep["version"] == "0.0.1+dependabot"
+
+            dependencies <<
+              Dependency.new(
+                name: normalised_name(dep["name"]),
+                version: dep["version"]&.include?("*") ? nil : dep["version"],
+                requirements: [{
+                  requirement: dep["requirement"],
+                  file: Pathname.new(dep["file"]).cleanpath.to_path,
+                  source: nil,
+                  groups: [dep["requirement_type"]]
+                }],
+                package_manager: "pip"
+              )
+          end
+          dependencies
+        end
+
+        private
+
+        attr_reader :dependency_files
+
+        def parsed_setup_file
+          SharedHelpers.in_a_temporary_directory do
+            write_temporary_dependency_files
+
+            requirements = SharedHelpers.run_helper_subprocess(
+              command: "pyenv exec python #{NativeHelpers.python_helper_path}",
+              function: "parse_setup",
+              args: [Dir.pwd]
+            )
+
+            check_requirements(requirements)
+            requirements
+          end
+        rescue SharedHelpers::HelperSubprocessFailed => error
+          if error.message.start_with?("InstallationError")
+            raise Dependabot::DependencyFileNotEvaluatable, error.message
+          end
+
+          parsed_sanitized_setup_file
+        end
+
+        def parsed_sanitized_setup_file
+          SharedHelpers.in_a_temporary_directory do
+            write_sanitized_setup_file
+
+            requirements = SharedHelpers.run_helper_subprocess(
+              command: "pyenv exec python #{NativeHelpers.python_helper_path}",
+              function: "parse_setup",
+              args: [Dir.pwd]
+            )
+
+            check_requirements(requirements)
+            requirements
+          end
+        rescue SharedHelpers::HelperSubprocessFailed
+          # Assume there are no dependencies in setup.py files that fail to
+          # parse. This isn't ideal, and we should continue to improve
+          # parsing, but there are a *lot* of things that can go wrong at
+          # the moment!
+          []
+        end
+
+        def check_requirements(requirements)
+          requirements.each do |dep|
+            next unless dep["requirement"]
+
+            Python::Requirement.new(dep["requirement"].split(","))
+          rescue Gem::Requirement::BadRequirementError => error
+            raise Dependabot::DependencyFileNotEvaluatable, error.message
+          end
+        end
+
+        def write_temporary_dependency_files
+          dependency_files.
+            reject { |f| f.name == ".python-version" }.
+            each do |file|
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, file.content)
+            end
+        end
+
+        # Write a setup.py with only entries for the requires fields.
+        #
+        # This sanitization is far from perfect (it will fail if any of the
+        # entries are dynamic), but it is an alternative approach to the one
+        # used in parser.py which sometimes succeeds when that has failed.
+        def write_sanitized_setup_file
+          original_content = setup_file.content
+
+          install_requires =
+            original_content.match(INSTALL_REQUIRES_REGEX)&.captures&.first
+          setup_requires =
+            original_content.match(SETUP_REQUIRES_REGEX)&.captures&.first
+          tests_require =
+            original_content.match(TESTS_REQUIRE_REGEX)&.captures&.first
+          extras_require =
+            original_content.match(EXTRAS_REQUIRE_REGEX)&.captures&.first
+
+          tmp = "from setuptools import setup\n\n"\
+                "setup(name=\"sanitized-package\",version=\"0.0.1\","
+
+          tmp += "install_requires=#{install_requires}," if install_requires
+          tmp += "setup_requires=#{setup_requires}," if setup_requires
+          tmp += "tests_require=#{tests_require}," if tests_require
+          tmp += "extras_require=#{extras_require}," if extras_require
+          tmp += ")"
+
+          File.write("setup.py", tmp)
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalised_name(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def setup_file
+          dependency_files.find { |f| f.name == "setup.py" }
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/file_updater.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "dependabot/file_updaters"
+require "dependabot/file_updaters/base"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module Python
+    class FileUpdater < Dependabot::FileUpdaters::Base
+      require_relative "file_updater/pipfile_file_updater"
+      require_relative "file_updater/pip_compile_file_updater"
+      require_relative "file_updater/poetry_file_updater"
+      require_relative "file_updater/requirement_file_updater"
+
+      def self.updated_files_regex
+        [
+          /^Pipfile$/,
+          /^Pipfile\.lock$/,
+          /.*\.txt$/,
+          /.*\.in$/,
+          /^setup\.py$/,
+          /^pyproject\.toml$/,
+          /^pyproject\.lock$/
+        ]
+      end
+
+      def updated_dependency_files
+        updated_files =
+          case resolver_type
+          when :pipfile then updated_pipfile_based_files
+          when :poetry then updated_poetry_based_files
+          when :pip_compile then updated_pip_compile_based_files
+          when :requirements then updated_requirement_based_files
+          else raise "Unexpected resolver type: #{resolver_type}"
+          end
+
+        if updated_files.none? ||
+           updated_files.sort_by(&:name) == dependency_files.sort_by(&:name)
+          raise "No files have changed!"
+        end
+
+        updated_files
+      end
+
+      private
+
+      def resolver_type
+        reqs = dependencies.flat_map(&:requirements)
+        req_files = reqs.map { |r| r.fetch(:file) }
+
+        # If there are no requirements then this is a sub-dependency. It
+        # must come from one of Pipenv, Poetry or pip-tools, and can't come
+        # from the first two unless they have a lockfile.
+        return subdependency_resolver if reqs.none?
+
+        # Otherwise, this is a top-level dependency, and we can figure out
+        # which resolver to use based on the filename of its requirements
+        return :pipfile if req_files.any? { |f| f == "Pipfile" }
+        return :poetry if req_files.any? { |f| f == "pyproject.toml" }
+        return :pip_compile if req_files.any? { |f| f.end_with?(".in") }
+
+        # Finally, we should only ever be updating a requirements.txt file if
+        # some requirements have changed. Otherwise, this must be a case where
+        # we have a requirements.txt *and* some other resolver of which the
+        # dependency is a sub-dependency.
+        changed_reqs = reqs - dependencies.flat_map(&:previous_requirements)
+        changed_reqs.none? ? subdependency_resolver : :requirements
+      end
+
+      def subdependency_resolver
+        return :pipfile if pipfile_lock
+        return :poetry if poetry_lock || pyproject_lock
+        return :pip_compile if pip_compile_files.any?
+
+        raise "Claimed to be a sub-dependency, but no lockfile exists!"
+      end
+
+      def updated_pipfile_based_files
+        PipfileFileUpdater.new(
+          dependencies: dependencies,
+          dependency_files: dependency_files,
+          credentials: credentials
+        ).updated_dependency_files
+      end
+
+      def updated_poetry_based_files
+        PoetryFileUpdater.new(
+          dependencies: dependencies,
+          dependency_files: dependency_files,
+          credentials: credentials
+        ).updated_dependency_files
+      end
+
+      def updated_pip_compile_based_files
+        PipCompileFileUpdater.new(
+          dependencies: dependencies,
+          dependency_files: dependency_files,
+          credentials: credentials
+        ).updated_dependency_files
+      end
+
+      def updated_requirement_based_files
+        RequirementFileUpdater.new(
+          dependencies: dependencies,
+          dependency_files: dependency_files,
+          credentials: credentials
+        ).updated_dependency_files
+      end
+
+      def check_required_files
+        filenames = dependency_files.map(&:name)
+        return if filenames.any? { |name| name.end_with?(".txt", ".in") }
+        return if pipfile
+        return if pyproject
+        return if get_original_file("setup.py")
+
+        raise "No requirements.txt or setup.py!"
+      end
+
+      def pipfile
+        @pipfile ||= get_original_file("Pipfile")
+      end
+
+      def pipfile_lock
+        @pipfile_lock ||= get_original_file("Pipfile.lock")
+      end
+
+      def pyproject
+        @pyproject ||= get_original_file("pyproject.toml")
+      end
+
+      def pyproject_lock
+        @pyproject_lock ||= get_original_file("pyproject.lock")
+      end
+
+      def poetry_lock
+        @poetry_lock ||= get_original_file("poetry.lock")
+      end
+
+      def pip_compile_files
+        @pip_compile_files ||=
+          dependency_files.select { |f| f.name.end_with?(".in") }
+      end
+    end
+  end
+end
+
+Dependabot::FileUpdaters.
+  register("pip", Dependabot::Python::FileUpdater)

data/lib/dependabot/python/file_updater/pip_compile_file_updater.rb

@@ -0,0 +1,361 @@
+# frozen_string_literal: true
+
+require "dependabot/python/requirement_parser"
+require "dependabot/python/file_fetcher"
+require "dependabot/python/file_updater"
+require "dependabot/shared_helpers"
+
+# rubocop:disable Metrics/ClassLength
+module Dependabot
+  module Python
+    class FileUpdater
+      class PipCompileFileUpdater
+        require_relative "requirement_replacer"
+        require_relative "requirement_file_updater"
+        require_relative "setup_file_sanitizer"
+
+        UNSAFE_PACKAGES = %w(setuptools distribute pip).freeze
+
+        attr_reader :dependencies, :dependency_files, :credentials
+
+        def initialize(dependencies:, dependency_files:, credentials:)
+          @dependencies = dependencies
+          @dependency_files = dependency_files
+          @credentials = credentials
+        end
+
+        def updated_dependency_files
+          return @updated_dependency_files if @update_already_attempted
+
+          @update_already_attempted = true
+          @updated_dependency_files ||= fetch_updated_dependency_files
+        end
+
+        private
+
+        def dependency
+          # For now, we'll only ever be updating a single dependency
+          dependencies.first
+        end
+
+        def fetch_updated_dependency_files
+          updated_compiled_files = compile_new_requirement_files
+          updated_manifest_files = update_manifest_files
+
+          updated_files = updated_compiled_files + updated_manifest_files
+          updated_uncompiled_files = update_uncompiled_files(updated_files)
+
+          [
+            *updated_manifest_files,
+            *updated_compiled_files,
+            *updated_uncompiled_files
+          ]
+        end
+
+        def compile_new_requirement_files
+          SharedHelpers.in_a_temporary_directory do
+            write_updated_dependency_files
+
+            filenames_to_compile.each do |filename|
+              # Shell out to pip-compile, generate a new set of requirements.
+              # This is slow, as pip-compile needs to do installs.
+              run_command(
+                "pyenv exec pip-compile #{pip_compile_options(filename)} "\
+                "-P #{dependency.name} #{filename}"
+              )
+            end
+
+            dependency_files.map do |file|
+              next unless file.name.end_with?(".txt")
+
+              updated_content = File.read(file.name)
+
+              updated_content =
+                replace_header_with_original(updated_content, file.content)
+              next if updated_content == file.content
+
+              file.dup.tap { |f| f.content = updated_content }
+            end.compact
+          end
+        end
+
+        def update_manifest_files
+          dependency_files.map do |file|
+            next unless file.name.end_with?(".in")
+
+            file = file.dup
+            updated_content = update_dependency_requirement(file)
+            next if updated_content == file.content
+
+            file.content = updated_content
+            file
+          end.compact
+        end
+
+        def update_uncompiled_files(updated_files)
+          updated_filenames = updated_files.map(&:name)
+          old_reqs = dependency.previous_requirements.
+                     reject { |r| updated_filenames.include?(r[:file]) }
+          new_reqs = dependency.requirements.
+                     reject { |r| updated_filenames.include?(r[:file]) }
+
+          return [] if new_reqs.none?
+
+          files = dependency_files.
+                  reject { |file| updated_filenames.include?(file.name) }
+
+          args = dependency.to_h
+          args = Hash[args.keys.map { |k| [k.to_sym, args[k]] }]
+          args[:requirements] = new_reqs
+          args[:previous_requirements] = old_reqs
+
+          RequirementFileUpdater.new(
+            dependencies: [Dependency.new(**args)],
+            dependency_files: files,
+            credentials: credentials
+          ).updated_dependency_files
+        end
+
+        def run_command(command)
+          command = command.dup
+          raw_response = nil
+          IO.popen(command, err: %i(child out)) do |process|
+            raw_response = process.read
+          end
+
+          # Raise an error with the output from the shell session if
+          # pip-compile returns a non-zero status
+          return if $CHILD_STATUS.success?
+
+          raise SharedHelpers::HelperSubprocessFailed.new(
+            raw_response,
+            command
+          )
+        rescue SharedHelpers::HelperSubprocessFailed => error
+          original_error ||= error
+          msg = error.message
+
+          relevant_error =
+            if error_suggests_bad_python_version?(msg) then original_error
+            else error
+            end
+
+          raise relevant_error unless error_suggests_bad_python_version?(msg)
+          raise relevant_error if File.exist?(".python-version")
+
+          command = "pyenv local 2.7.15 && " + command
+          retry
+        ensure
+          FileUtils.remove_entry(".python-version", true)
+        end
+
+        def error_suggests_bad_python_version?(message)
+          return true if message.include?("not find a version that satisfies")
+
+          message.include?('Command "python setup.py egg_info" failed')
+        end
+
+        def write_updated_dependency_files
+          dependency_files.each do |file|
+            next if file.name == ".python-version"
+
+            path = file.name
+            FileUtils.mkdir_p(Pathname.new(path).dirname)
+            File.write(path, freeze_dependency_requirement(file))
+          end
+
+          setup_files.each do |file|
+            path = file.name
+            FileUtils.mkdir_p(Pathname.new(path).dirname)
+            File.write(path, sanitized_setup_file_content(file))
+          end
+
+          setup_cfg_files.each do |file|
+            path = file.name
+            FileUtils.mkdir_p(Pathname.new(path).dirname)
+            File.write(path, "[metadata]\nname = sanitized-package\n")
+          end
+        end
+
+        def sanitized_setup_file_content(file)
+          @sanitized_setup_file_content ||= {}
+          if @sanitized_setup_file_content[file.name]
+            return @sanitized_setup_file_content[file.name]
+          end
+
+          @sanitized_setup_file_content[file.name] =
+            SetupFileSanitizer.
+            new(setup_file: file, setup_cfg: setup_cfg(file)).
+            sanitized_content
+        end
+
+        def setup_cfg(file)
+          dependency_files.find do |f|
+            f.name == file.name.sub(/\.py$/, ".cfg")
+          end
+        end
+
+        def freeze_dependency_requirement(file)
+          return file.content unless file.name.end_with?(".in")
+
+          old_req = dependency.previous_requirements.
+                    find { |r| r[:file] == file.name }
+
+          return file.content unless old_req
+          return file.content if old_req == "==#{dependency.version}"
+
+          RequirementReplacer.new(
+            content: file.content,
+            dependency_name: dependency.name,
+            old_requirement: old_req[:requirement],
+            new_requirement: "==#{dependency.version}"
+          ).updated_content
+        end
+
+        def update_dependency_requirement(file)
+          return file.content unless file.name.end_with?(".in")
+
+          old_req = dependency.previous_requirements.
+                    find { |r| r[:file] == file.name }
+          new_req = dependency.requirements.
+                    find { |r| r[:file] == file.name }
+          return file.content unless old_req&.fetch(:requirement)
+          return file.content if old_req == new_req
+
+          RequirementReplacer.new(
+            content: file.content,
+            dependency_name: dependency.name,
+            old_requirement: old_req[:requirement],
+            new_requirement: new_req[:requirement]
+          ).updated_content
+        end
+
+        def replace_header_with_original(updated_content, original_content)
+          original_header_lines =
+            original_content.lines.take_while { |l| l.start_with?("#") }
+
+          updated_content_lines =
+            updated_content.lines.drop_while { |l| l.start_with?("#") }
+
+          [*original_header_lines, *updated_content_lines].join
+        end
+
+        def pip_compile_options(filename)
+          current_requirements_file_name = filename.sub(/\.in$/, ".txt")
+
+          requirements_file =
+            dependency_files.
+            find { |f| f.name == current_requirements_file_name }
+
+          return unless requirements_file
+
+          options = ""
+
+          if requirements_file.content.include?("--hash=sha")
+            options += " --generate-hashes"
+          end
+
+          if includes_unsafe_packages?(requirements_file.content)
+            options += " --allow-unsafe"
+          end
+
+          unless requirements_file.content.include?("# via ")
+            options += " --no-annotate"
+          end
+
+          unless requirements_file.content.include?("autogenerated by pip-c")
+            options += " --no-header"
+          end
+
+          options.strip
+        end
+
+        def includes_unsafe_packages?(content)
+          UNSAFE_PACKAGES.any? { |n| content.match?(/^#{Regexp.quote(n)}==/) }
+        end
+
+        def filenames_to_compile
+          files_from_reqs =
+            dependency.requirements.
+            map { |r| r[:file] }.
+            select { |fn| fn.end_with?(".in") }
+
+          files_from_compiled_files =
+            pip_compile_files.map(&:name).select do |fn|
+              compiled_file = dependency_files.
+                              find { |f| f.name == fn.gsub(/\.in$/, ".txt") }
+              compiled_file_includes_dependency?(compiled_file)
+            end
+
+          filenames = [*files_from_reqs, *files_from_compiled_files].uniq
+
+          order_filenames_for_compilation(filenames)
+        end
+
+        def compiled_file_includes_dependency?(compiled_file)
+          return false unless compiled_file
+
+          regex = RequirementParser::INSTALL_REQ_WITH_REQUIREMENT
+
+          matches = []
+          compiled_file.content.scan(regex) { matches << Regexp.last_match }
+          matches.any? { |m| normalise(m[:name]) == dependency.name }
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalise(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        # If the files we need to update require one another then we need to
+        # update them in the right order
+        def order_filenames_for_compilation(filenames)
+          ordered_filenames = []
+
+          while (remaining_filenames = filenames - ordered_filenames).any?
+            ordered_filenames +=
+              remaining_filenames.
+              select do |fn|
+                unupdated_reqs = requirement_map[fn] - ordered_filenames
+                (unupdated_reqs & filenames).empty?
+              end
+          end
+
+          ordered_filenames
+        end
+
+        def requirement_map
+          child_req_regex = Python::FileFetcher::CHILD_REQUIREMENT_REGEX
+          @requirement_map ||=
+            pip_compile_files.each_with_object({}) do |file, req_map|
+              paths = file.content.scan(child_req_regex).flatten
+              current_dir = File.dirname(file.name)
+
+              req_map[file.name] =
+                paths.map do |path|
+                  path = File.join(current_dir, path) if current_dir != "."
+                  path = Pathname.new(path).cleanpath.to_path
+                  path = path.gsub(/\.txt$/, ".in")
+                  next if path == file.name
+
+                  path
+                end.uniq.compact
+            end
+        end
+
+        def setup_files
+          dependency_files.select { |f| f.name.end_with?("setup.py") }
+        end
+
+        def pip_compile_files
+          dependency_files.select { |f| f.name.end_with?(".in") }
+        end
+
+        def setup_cfg_files
+          dependency_files.select { |f| f.name.end_with?("setup.cfg") }
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength