dependabot-core 0.76.1

data/lib/dependabot/file_updaters/python/pip/pip_compile_file_updater.rb

@@ -0,0 +1,363 @@
+# frozen_string_literal: true
+
+require "python_requirement_parser"
+require "dependabot/file_fetchers/python/pip"
+require "dependabot/file_updaters/python/pip"
+require "dependabot/shared_helpers"
+
+# rubocop:disable Metrics/ClassLength
+module Dependabot
+  module FileUpdaters
+    module Python
+      class Pip
+        class PipCompileFileUpdater
+          require_relative "requirement_replacer"
+          require_relative "requirement_file_updater"
+          require_relative "setup_file_sanitizer"
+
+          UNSAFE_PACKAGES = %w(setuptools distribute pip).freeze
+
+          attr_reader :dependencies, :dependency_files, :credentials
+
+          def initialize(dependencies:, dependency_files:, credentials:)
+            @dependencies = dependencies
+            @dependency_files = dependency_files
+            @credentials = credentials
+          end
+
+          def updated_dependency_files
+            return @updated_dependency_files if @update_already_attempted
+
+            @update_already_attempted = true
+            @updated_dependency_files ||= fetch_updated_dependency_files
+          end
+
+          private
+
+          def dependency
+            # For now, we'll only ever be updating a single dependency
+            dependencies.first
+          end
+
+          def fetch_updated_dependency_files
+            updated_compiled_files = compile_new_requirement_files
+            updated_manifest_files = update_manifest_files
+
+            updated_files = updated_compiled_files + updated_manifest_files
+            updated_uncompiled_files = update_uncompiled_files(updated_files)
+
+            [
+              *updated_manifest_files,
+              *updated_compiled_files,
+              *updated_uncompiled_files
+            ]
+          end
+
+          def compile_new_requirement_files
+            SharedHelpers.in_a_temporary_directory do
+              write_updated_dependency_files
+
+              filenames_to_compile.each do |filename|
+                # Shell out to pip-compile, generate a new set of requirements.
+                # This is slow, as pip-compile needs to do installs.
+                run_command(
+                  "pyenv exec pip-compile #{pip_compile_options(filename)} "\
+                  "-P #{dependency.name} #{filename}"
+                )
+              end
+
+              dependency_files.map do |file|
+                next unless file.name.end_with?(".txt")
+
+                updated_content = File.read(file.name)
+
+                updated_content =
+                  replace_header_with_original(updated_content, file.content)
+                next if updated_content == file.content
+
+                file.dup.tap { |f| f.content = updated_content }
+              end.compact
+            end
+          end
+
+          def update_manifest_files
+            dependency_files.map do |file|
+              next unless file.name.end_with?(".in")
+
+              file = file.dup
+              updated_content = update_dependency_requirement(file)
+              next if updated_content == file.content
+
+              file.content = updated_content
+              file
+            end.compact
+          end
+
+          def update_uncompiled_files(updated_files)
+            updated_filenames = updated_files.map(&:name)
+            old_reqs = dependency.previous_requirements.
+                       reject { |r| updated_filenames.include?(r[:file]) }
+            new_reqs = dependency.requirements.
+                       reject { |r| updated_filenames.include?(r[:file]) }
+
+            return [] if new_reqs.none?
+
+            files = dependency_files.
+                    reject { |file| updated_filenames.include?(file.name) }
+
+            args = dependency.to_h
+            args = Hash[args.keys.map { |k| [k.to_sym, args[k]] }]
+            args[:requirements] = new_reqs
+            args[:previous_requirements] = old_reqs
+
+            RequirementFileUpdater.new(
+              dependencies: [Dependency.new(**args)],
+              dependency_files: files,
+              credentials: credentials
+            ).updated_dependency_files
+          end
+
+          def run_command(command)
+            command = command.dup
+            raw_response = nil
+            IO.popen(command, err: %i(child out)) do |process|
+              raw_response = process.read
+            end
+
+            # Raise an error with the output from the shell session if
+            # pip-compile returns a non-zero status
+            return if $CHILD_STATUS.success?
+
+            raise SharedHelpers::HelperSubprocessFailed.new(
+              raw_response,
+              command
+            )
+          rescue SharedHelpers::HelperSubprocessFailed => error
+            original_error ||= error
+            msg = error.message
+
+            relevant_error =
+              if error_suggests_bad_python_version?(msg) then original_error
+              else error
+              end
+
+            raise relevant_error unless error_suggests_bad_python_version?(msg)
+            raise relevant_error if File.exist?(".python-version")
+
+            command = "pyenv local 2.7.15 && " + command
+            retry
+          ensure
+            FileUtils.remove_entry(".python-version", true)
+          end
+
+          def error_suggests_bad_python_version?(message)
+            return true if message.include?("not find a version that satisfies")
+
+            message.include?('Command "python setup.py egg_info" failed')
+          end
+
+          def write_updated_dependency_files
+            dependency_files.each do |file|
+              next if file.name == ".python-version"
+
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, freeze_dependency_requirement(file))
+            end
+
+            setup_files.each do |file|
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, sanitized_setup_file_content(file))
+            end
+
+            setup_cfg_files.each do |file|
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, "[metadata]\nname = sanitized-package\n")
+            end
+          end
+
+          def sanitized_setup_file_content(file)
+            @sanitized_setup_file_content ||= {}
+            if @sanitized_setup_file_content[file.name]
+              return @sanitized_setup_file_content[file.name]
+            end
+
+            @sanitized_setup_file_content[file.name] =
+              SetupFileSanitizer.
+              new(setup_file: file, setup_cfg: setup_cfg(file)).
+              sanitized_content
+          end
+
+          def setup_cfg(file)
+            dependency_files.find do |f|
+              f.name == file.name.sub(/\.py$/, ".cfg")
+            end
+          end
+
+          def freeze_dependency_requirement(file)
+            return file.content unless file.name.end_with?(".in")
+
+            old_req = dependency.previous_requirements.
+                      find { |r| r[:file] == file.name }
+
+            return file.content unless old_req
+            return file.content if old_req == "==#{dependency.version}"
+
+            RequirementReplacer.new(
+              content: file.content,
+              dependency_name: dependency.name,
+              old_requirement: old_req[:requirement],
+              new_requirement: "==#{dependency.version}"
+            ).updated_content
+          end
+
+          def update_dependency_requirement(file)
+            return file.content unless file.name.end_with?(".in")
+
+            old_req = dependency.previous_requirements.
+                      find { |r| r[:file] == file.name }
+            new_req = dependency.requirements.
+                      find { |r| r[:file] == file.name }
+            return file.content unless old_req&.fetch(:requirement)
+            return file.content if old_req == new_req
+
+            RequirementReplacer.new(
+              content: file.content,
+              dependency_name: dependency.name,
+              old_requirement: old_req[:requirement],
+              new_requirement: new_req[:requirement]
+            ).updated_content
+          end
+
+          def replace_header_with_original(updated_content, original_content)
+            original_header_lines =
+              original_content.lines.take_while { |l| l.start_with?("#") }
+
+            updated_content_lines =
+              updated_content.lines.drop_while { |l| l.start_with?("#") }
+
+            [*original_header_lines, *updated_content_lines].join
+          end
+
+          def pip_compile_options(filename)
+            current_requirements_file_name = filename.sub(/\.in$/, ".txt")
+
+            requirements_file =
+              dependency_files.
+              find { |f| f.name == current_requirements_file_name }
+
+            return unless requirements_file
+
+            options = ""
+
+            if requirements_file.content.include?("--hash=sha")
+              options += " --generate-hashes"
+            end
+
+            if includes_unsafe_packages?(requirements_file.content)
+              options += " --allow-unsafe"
+            end
+
+            unless requirements_file.content.include?("# via ")
+              options += " --no-annotate"
+            end
+
+            unless requirements_file.content.include?("autogenerated by pip-c")
+              options += " --no-header"
+            end
+
+            options.strip
+          end
+
+          def includes_unsafe_packages?(content)
+            UNSAFE_PACKAGES.any? { |n| content.match?(/^#{Regexp.quote(n)}==/) }
+          end
+
+          def filenames_to_compile
+            files_from_reqs =
+              dependency.requirements.
+              map { |r| r[:file] }.
+              select { |fn| fn.end_with?(".in") }
+
+            files_from_compiled_files =
+              pip_compile_files.map(&:name).select do |fn|
+                compiled_file = dependency_files.
+                                find { |f| f.name == fn.gsub(/\.in$/, ".txt") }
+                compiled_file_includes_dependency?(compiled_file)
+              end
+
+            filenames = [*files_from_reqs, *files_from_compiled_files].uniq
+
+            order_filenames_for_compilation(filenames)
+          end
+
+          def compiled_file_includes_dependency?(compiled_file)
+            return false unless compiled_file
+
+            regex = PythonRequirementParser::INSTALL_REQ_WITH_REQUIREMENT
+
+            matches = []
+            compiled_file.content.scan(regex) { matches << Regexp.last_match }
+            matches.any? { |m| normalise(m[:name]) == dependency.name }
+          end
+
+          # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+          def normalise(name)
+            name.downcase.gsub(/[-_.]+/, "-")
+          end
+
+          # If the files we need to update require one another then we need to
+          # update them in the right order
+          def order_filenames_for_compilation(filenames)
+            ordered_filenames = []
+
+            while (remaining_filenames = filenames - ordered_filenames).any?
+              ordered_filenames +=
+                remaining_filenames.
+                select do |fn|
+                  unupdated_reqs = requirement_map[fn] - ordered_filenames
+                  (unupdated_reqs & filenames).empty?
+                end
+            end
+
+            ordered_filenames
+          end
+
+          def requirement_map
+            child_req_regex = FileFetchers::Python::Pip::CHILD_REQUIREMENT_REGEX
+            @requirement_map ||=
+              pip_compile_files.each_with_object({}) do |file, req_map|
+                paths = file.content.scan(child_req_regex).flatten
+                current_dir = File.dirname(file.name)
+
+                req_map[file.name] =
+                  paths.map do |path|
+                    path = File.join(current_dir, path) if current_dir != "."
+                    path = Pathname.new(path).cleanpath.to_path
+                    path = path.gsub(/\.txt$/, ".in")
+                    next if path == file.name
+
+                    path
+                  end.uniq.compact
+              end
+          end
+
+          def setup_files
+            dependency_files.select { |f| f.name.end_with?("setup.py") }
+          end
+
+          def pip_compile_files
+            dependency_files.select { |f| f.name.end_with?(".in") }
+          end
+
+          def setup_cfg_files
+            dependency_files.select { |f| f.name.end_with?("setup.cfg") }
+          end
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength

data/lib/dependabot/file_updaters/python/pip/pipfile_file_updater.rb

@@ -0,0 +1,397 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "python_requirement_parser"
+require "dependabot/file_updaters/python/pip"
+require "dependabot/shared_helpers"
+
+# rubocop:disable Metrics/ClassLength
+module Dependabot
+  module FileUpdaters
+    module Python
+      class Pip
+        class PipfileFileUpdater
+          require_relative "pipfile_preparer"
+          require_relative "setup_file_sanitizer"
+
+          attr_reader :dependencies, :dependency_files, :credentials
+
+          def initialize(dependencies:, dependency_files:, credentials:)
+            @dependencies = dependencies
+            @dependency_files = dependency_files
+            @credentials = credentials
+          end
+
+          def updated_dependency_files
+            return @updated_dependency_files if @update_already_attempted
+
+            @update_already_attempted = true
+            @updated_dependency_files ||= fetch_updated_dependency_files
+          end
+
+          private
+
+          def dependency
+            # For now, we'll only ever be updating a single dependency
+            dependencies.first
+          end
+
+          def fetch_updated_dependency_files
+            updated_files = []
+
+            if file_changed?(pipfile)
+              updated_files <<
+                updated_file(file: pipfile, content: updated_pipfile_content)
+            end
+
+            if lockfile
+              if lockfile.content == updated_lockfile_content
+                raise "Expected Pipfile.lock to change!"
+              end
+
+              updated_files <<
+                updated_file(file: lockfile, content: updated_lockfile_content)
+            end
+
+            updated_files += updated_generated_requirements_files
+            updated_files
+          end
+
+          def updated_pipfile_content
+            dependencies.
+              select { |dep| requirement_changed?(pipfile, dep) }.
+              reduce(pipfile.content.dup) do |content, dep|
+                updated_requirement =
+                  dep.requirements.find { |r| r[:file] == pipfile.name }.
+                  fetch(:requirement)
+
+                old_req =
+                  dep.previous_requirements.
+                  find { |r| r[:file] == pipfile.name }.
+                  fetch(:requirement)
+
+                updated_content =
+                  content.gsub(declaration_regex(dep)) do |line|
+                    line.gsub(old_req, updated_requirement)
+                  end
+
+                raise "Content did not change!" if content == updated_content
+
+                updated_content
+              end
+          end
+
+          def updated_lockfile_content
+            @updated_lockfile_content ||=
+              updated_generated_files.fetch(:lockfile)
+          end
+
+          def generate_updated_requirements_files?
+            return true if generated_requirements_files("default").any?
+
+            generated_requirements_files("develop").any?
+          end
+
+          def generated_requirements_files(type)
+            return [] unless lockfile
+
+            pipfile_lock_deps = parsed_lockfile[type]&.keys&.sort || []
+            pipfile_lock_deps = pipfile_lock_deps.map { |n| normalise(n) }
+
+            regex = PythonRequirementParser::INSTALL_REQ_WITH_REQUIREMENT
+
+            # Find any requirement files that list the same dependencies as
+            # the (old) Pipfile.lock. Any such files were almost certainly
+            # generated using `pipenv lock -r`
+            requirements_files.select do |req_file|
+              deps = []
+              req_file.content.scan(regex) { deps << Regexp.last_match }
+              deps = deps.map { |m| normalise(m[:name]) }
+              deps.sort == pipfile_lock_deps
+            end
+          end
+
+          def updated_generated_requirements_files
+            updated_files = []
+
+            generated_requirements_files("default").each do |file|
+              next if file.content == updated_req_content
+
+              updated_files <<
+                updated_file(file: file, content: updated_req_content)
+            end
+
+            generated_requirements_files("develop").each do |file|
+              next if file.content == updated_dev_req_content
+
+              updated_files <<
+                updated_file(file: file, content: updated_dev_req_content)
+            end
+
+            updated_files
+          end
+
+          def updated_req_content
+            updated_generated_files.fetch(:requirements_txt)
+          end
+
+          def updated_dev_req_content
+            updated_generated_files.fetch(:dev_requirements_txt)
+          end
+
+          def prepared_pipfile_content
+            content = updated_pipfile_content
+            content = freeze_other_dependencies(content)
+            content = freeze_dependencies_being_updated(content)
+            content = add_private_sources(content)
+            content
+          end
+
+          def freeze_other_dependencies(pipfile_content)
+            PipfilePreparer.
+              new(pipfile_content: pipfile_content).
+              freeze_top_level_dependencies_except(dependencies, lockfile)
+          end
+
+          def freeze_dependencies_being_updated(pipfile_content)
+            pipfile_object = TomlRB.parse(pipfile_content)
+
+            dependencies.each do |dep|
+              %w(packages dev-packages).each do |type|
+                names = pipfile_object[type]&.keys || []
+                pkg_name = names.find { |nm| normalise(nm) == dep.name }
+                next unless pkg_name
+
+                if pipfile_object[type][pkg_name].is_a?(Hash)
+                  pipfile_object[type][pkg_name]["version"] =
+                    "==#{dep.version}"
+                else
+                  pipfile_object[type][pkg_name] = "==#{dep.version}"
+                end
+              end
+            end
+
+            TomlRB.dump(pipfile_object)
+          end
+
+          def add_private_sources(pipfile_content)
+            PipfilePreparer.
+              new(pipfile_content: pipfile_content).
+              replace_sources(credentials)
+          end
+
+          def updated_generated_files
+            @updated_generated_files ||=
+              SharedHelpers.in_a_temporary_directory do
+                SharedHelpers.with_git_configured(credentials: credentials) do
+                  write_temporary_dependency_files(prepared_pipfile_content)
+
+                  # Initialize a git repo to appease pip-tools
+                  IO.popen("git init", err: %i(child out)) if setup_files.any?
+
+                  run_pipenv_command(
+                    pipenv_environment_variables + "pyenv exec pipenv lock"
+                  )
+
+                  result = { lockfile: File.read("Pipfile.lock") }
+                  result[:lockfile] = post_process_lockfile(result[:lockfile])
+
+                  # Generate updated requirement.txt entries, if needed.
+                  if generate_updated_requirements_files?
+                    generate_updated_requirements_files
+
+                    result[:requirements_txt] = File.read("req.txt")
+                    result[:dev_requirements_txt] = File.read("dev-req.txt")
+                  end
+
+                  result
+                end
+              end
+          end
+
+          def post_process_lockfile(updated_lockfile_content)
+            pipfile_hash = pipfile_hash_for(updated_pipfile_content)
+            original_reqs = parsed_lockfile["_meta"]["requires"]
+            original_source = parsed_lockfile["_meta"]["sources"]
+
+            new_lockfile = updated_lockfile_content.dup
+            new_lockfile_json = JSON.parse(new_lockfile)
+            new_lockfile_json["_meta"]["hash"]["sha256"] = pipfile_hash
+            new_lockfile_json["_meta"]["requires"] = original_reqs
+            new_lockfile_json["_meta"]["sources"] = original_source
+
+            JSON.pretty_generate(new_lockfile_json, indent: "    ").
+              gsub(/\{\n\s*\}/, "{}").
+              gsub(/\}\z/, "}\n")
+          end
+
+          def generate_updated_requirements_files
+            run_pipenv_command(
+              pipenv_environment_variables +
+                "pyenv exec pipenv lock -r > req.txt"
+            )
+            run_pipenv_command(
+              pipenv_environment_variables +
+                "pyenv exec pipenv lock -r -d > dev-req.txt"
+            )
+          end
+
+          def run_pipenv_command(cmd)
+            raw_response = nil
+            IO.popen(cmd, err: %i(child out)) { |p| raw_response = p.read }
+
+            # Raise an error with the output from the shell session if Pipenv
+            # returns a non-zero status
+            return if $CHILD_STATUS.success?
+
+            raise SharedHelpers::HelperSubprocessFailed.new(raw_response, cmd)
+          rescue SharedHelpers::HelperSubprocessFailed => error
+            original_error ||= error
+            msg = error.message
+
+            relevant_error =
+              if error_suggests_bad_python_version?(msg) then original_error
+              else error
+              end
+
+            raise relevant_error unless error_suggests_bad_python_version?(msg)
+            raise relevant_error if cmd.include?("--two")
+
+            cmd = cmd.gsub("pipenv ", "pipenv --two ")
+            retry
+          end
+
+          def error_suggests_bad_python_version?(message)
+            return true if message.include?("UnsupportedPythonVersion")
+
+            message.include?('Command "python setup.py egg_info" failed')
+          end
+
+          def write_temporary_dependency_files(pipfile_content)
+            dependency_files.each do |file|
+              next if file.name == ".python-version"
+
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, file.content)
+            end
+
+            setup_files.each do |file|
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, sanitized_setup_file_content(file))
+            end
+
+            setup_cfg_files.each do |file|
+              path = file.name
+              FileUtils.mkdir_p(Pathname.new(path).dirname)
+              File.write(path, "[metadata]\nname = sanitized-package\n")
+            end
+
+            # Overwrite the pipfile with updated content
+            File.write("Pipfile", pipfile_content)
+          end
+
+          def sanitized_setup_file_content(file)
+            @sanitized_setup_file_content ||= {}
+            if @sanitized_setup_file_content[file.name]
+              return @sanitized_setup_file_content[file.name]
+            end
+
+            @sanitized_setup_file_content[file.name] =
+              SetupFileSanitizer.
+              new(setup_file: file, setup_cfg: setup_cfg(file)).
+              sanitized_content
+          end
+
+          def setup_cfg(file)
+            dependency_files.find do |f|
+              f.name == file.name.sub(/\.py$/, ".cfg")
+            end
+          end
+
+          def pipfile_hash_for(pipfile_content)
+            SharedHelpers.in_a_temporary_directory do |dir|
+              File.write(File.join(dir, "Pipfile"), pipfile_content)
+              SharedHelpers.run_helper_subprocess(
+                command: "pyenv exec python #{python_helper_path}",
+                function: "get_pipfile_hash",
+                args: [dir]
+              )
+            end
+          end
+
+          def declaration_regex(dep)
+            escaped_name = Regexp.escape(dep.name).gsub("\\-", "[-_.]")
+            /(?:^|["'])#{escaped_name}["']?\s*=.*$/i
+          end
+
+          def file_changed?(file)
+            dependencies.any? { |dep| requirement_changed?(file, dep) }
+          end
+
+          def requirement_changed?(file, dependency)
+            changed_requirements =
+              dependency.requirements - dependency.previous_requirements
+
+            changed_requirements.any? { |f| f[:file] == file.name }
+          end
+
+          def updated_file(file:, content:)
+            updated_file = file.dup
+            updated_file.content = content
+            updated_file
+          end
+
+          def python_helper_path
+            project_root = File.join(File.dirname(__FILE__), "../../../../..")
+            File.join(project_root, "helpers/python/run.py")
+          end
+
+          # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+          def normalise(name)
+            name.downcase.gsub(/[-_.]+/, "-")
+          end
+
+          def parsed_lockfile
+            @parsed_lockfile ||= JSON.parse(lockfile.content)
+          end
+
+          def pipfile
+            @pipfile ||= dependency_files.find { |f| f.name == "Pipfile" }
+          end
+
+          def lockfile
+            @lockfile ||= dependency_files.find { |f| f.name == "Pipfile.lock" }
+          end
+
+          def setup_files
+            dependency_files.select { |f| f.name.end_with?("setup.py") }
+          end
+
+          def setup_cfg_files
+            dependency_files.select { |f| f.name.end_with?("setup.cfg") }
+          end
+
+          def requirements_files
+            dependency_files.select { |f| f.name.end_with?(".txt") }
+          end
+
+          def pipenv_environment_variables
+            environment_variables = [
+              "PIPENV_YES=true",       # Install new Python versions if needed
+              "PIPENV_MAX_RETRIES=3",  # Retry timeouts
+              "PIPENV_NOSPIN=1",       # Don't pollute logs with spinner
+              "PIPENV_TIMEOUT=600",    # Set install timeout to 10 minutes
+              "PIP_DEFAULT_TIMEOUT=60" # Set pip timeout to 1 minute
+            ]
+
+            environment_variables.join(" ") + " "
+          end
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength