dependabot-python 0.79.0

data/lib/dependabot/python/file_updater/poetry_file_updater.rb

@@ -0,0 +1,282 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/shared_helpers"
+require "dependabot/python/version"
+require "dependabot/python/requirement"
+require "dependabot/python/python_versions"
+require "dependabot/python/file_updater"
+require "dependabot/python/native_helpers"
+
+module Dependabot
+  module Python
+    class FileUpdater
+      class PoetryFileUpdater
+        require_relative "pyproject_preparer"
+
+        attr_reader :dependencies, :dependency_files, :credentials
+
+        def initialize(dependencies:, dependency_files:, credentials:)
+          @dependencies = dependencies
+          @dependency_files = dependency_files
+          @credentials = credentials
+        end
+
+        def updated_dependency_files
+          return @updated_dependency_files if @update_already_attempted
+
+          @update_already_attempted = true
+          @updated_dependency_files ||= fetch_updated_dependency_files
+        end
+
+        private
+
+        def dependency
+          # For now, we'll only ever be updating a single dependency
+          dependencies.first
+        end
+
+        def fetch_updated_dependency_files
+          updated_files = []
+
+          if file_changed?(pyproject)
+            updated_files <<
+              updated_file(
+                file: pyproject,
+                content: updated_pyproject_content
+              )
+          end
+
+          if lockfile && lockfile.content == updated_lockfile_content
+            raise "Expected lockfile to change!"
+          end
+
+          if lockfile
+            updated_files <<
+              updated_file(file: lockfile, content: updated_lockfile_content)
+          end
+
+          updated_files
+        end
+
+        def updated_pyproject_content
+          dependencies.
+            select { |dep| requirement_changed?(pyproject, dep) }.
+            reduce(pyproject.content.dup) do |content, dep|
+              updated_requirement =
+                dep.requirements.find { |r| r[:file] == pyproject.name }.
+                fetch(:requirement)
+
+              old_req =
+                dep.previous_requirements.
+                find { |r| r[:file] == pyproject.name }.
+                fetch(:requirement)
+
+              updated_content =
+                content.gsub(declaration_regex(dep)) do |line|
+                  line.gsub(old_req, updated_requirement)
+                end
+
+              raise "Content did not change!" if content == updated_content
+
+              updated_content
+            end
+        end
+
+        def updated_lockfile_content
+          @updated_lockfile_content ||=
+            begin
+              new_lockfile = updated_lockfile_content_for(prepared_pyproject)
+
+              tmp_hash =
+                TomlRB.parse(new_lockfile)["metadata"]["content-hash"]
+              correct_hash = pyproject_hash_for(updated_pyproject_content)
+
+              new_lockfile.gsub(tmp_hash, correct_hash)
+            end
+        end
+
+        def prepared_pyproject
+          content = updated_pyproject_content
+          content = freeze_other_dependencies(content)
+          content = freeze_dependencies_being_updated(content)
+          content = add_private_sources(content)
+          content = sanitize(content)
+          content
+        end
+
+        def freeze_other_dependencies(pyproject_content)
+          PyprojectPreparer.
+            new(pyproject_content: pyproject_content).
+            freeze_top_level_dependencies_except(dependencies, lockfile)
+        end
+
+        def freeze_dependencies_being_updated(pyproject_content)
+          pyproject_object = TomlRB.parse(pyproject_content)
+          poetry_object = pyproject_object.fetch("tool").fetch("poetry")
+
+          dependencies.each do |dep|
+            %w(dependencies dev-dependencies).each do |type|
+              names = poetry_object[type]&.keys || []
+              pkg_name = names.find { |nm| normalise(nm) == dep.name }
+              next unless pkg_name
+
+              if poetry_object[type][pkg_name].is_a?(Hash)
+                poetry_object[type][pkg_name]["version"] = dep.version
+              else
+                poetry_object[type][pkg_name] = dep.version
+              end
+            end
+          end
+
+          TomlRB.dump(pyproject_object)
+        end
+
+        def add_private_sources(pyproject_content)
+          PyprojectPreparer.
+            new(pyproject_content: pyproject_content).
+            replace_sources(credentials)
+        end
+
+        def sanitize(pyproject_content)
+          PyprojectPreparer.
+            new(pyproject_content: pyproject_content).
+            sanitize
+        end
+
+        def updated_lockfile_content_for(pyproject_content)
+          SharedHelpers.in_a_temporary_directory do
+            write_temporary_dependency_files(pyproject_content)
+
+            if python_version && !pre_installed_python?(python_version)
+              run_poetry_command("pyenv install -s")
+              run_poetry_command("pyenv exec pip install --upgrade pip")
+              run_poetry_command(
+                "pyenv exec pip install -r #{python_requirements_path}"
+              )
+            end
+
+            run_poetry_command("pyenv exec poetry lock")
+
+            return File.read("poetry.lock") if File.exist?("poetry.lock")
+
+            File.read("pyproject.lock")
+          end
+        end
+
+        def run_poetry_command(cmd)
+          raw_response = nil
+          IO.popen(cmd, err: %i(child out)) { |p| raw_response = p.read }
+
+          # Raise an error with the output from the shell session if Pipenv
+          # returns a non-zero status
+          return if $CHILD_STATUS.success?
+
+          raise SharedHelpers::HelperSubprocessFailed.new(raw_response, cmd)
+        end
+
+        def write_temporary_dependency_files(pyproject_content)
+          dependency_files.each do |file|
+            path = file.name
+            FileUtils.mkdir_p(Pathname.new(path).dirname)
+            File.write(path, file.content)
+          end
+
+          # Overwrite the .python-version with updated content
+          File.write(".python-version", python_version) if python_version
+
+          # Overwrite the pyproject with updated content
+          File.write("pyproject.toml", pyproject_content)
+        end
+
+        def python_version
+          pyproject_object = TomlRB.parse(prepared_pyproject)
+          poetry_object = pyproject_object.dig("tool", "poetry")
+
+          requirement =
+            poetry_object&.dig("dependencies", "python") ||
+            poetry_object&.dig("dev-dependencies", "python")
+
+          return python_version_file&.content unless requirement
+
+          requirements = Python::Requirement.requirements_array(requirement)
+
+          PythonVersions::PYTHON_VERSIONS.find do |version|
+            requirements.any? do |r|
+              r.satisfied_by?(Python::Version.new(version))
+            end
+          end
+        end
+
+        def pre_installed_python?(version)
+          PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version)
+        end
+
+        def pyproject_hash_for(pyproject_content)
+          SharedHelpers.in_a_temporary_directory do |dir|
+            File.write(File.join(dir, "pyproject.toml"), pyproject_content)
+            SharedHelpers.run_helper_subprocess(
+              command: "pyenv exec python #{NativeHelpers.python_helper_path}",
+              function: "get_pyproject_hash",
+              args: [dir]
+            )
+          end
+        end
+
+        def declaration_regex(dep)
+          escaped_name = Regexp.escape(dep.name).gsub("\\-", "[-_.]")
+          /(?:^|["'])#{escaped_name}["']?\s*=.*$/i
+        end
+
+        def file_changed?(file)
+          dependencies.any? { |dep| requirement_changed?(file, dep) }
+        end
+
+        def requirement_changed?(file, dependency)
+          changed_requirements =
+            dependency.requirements - dependency.previous_requirements
+
+          changed_requirements.any? { |f| f[:file] == file.name }
+        end
+
+        def updated_file(file:, content:)
+          updated_file = file.dup
+          updated_file.content = content
+          updated_file
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalise(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def pyproject
+          @pyproject ||=
+            dependency_files.find { |f| f.name == "pyproject.toml" }
+        end
+
+        def lockfile
+          @lockfile ||= pyproject_lock || poetry_lock
+        end
+
+        def pyproject_lock
+          dependency_files.find { |f| f.name == "pyproject.lock" }
+        end
+
+        def poetry_lock
+          dependency_files.find { |f| f.name == "poetry.lock" }
+        end
+
+        def python_version_file
+          dependency_files.find { |f| f.name == ".python-version" }
+        end
+
+        def python_requirements_path
+          project_root = File.join(File.dirname(__FILE__), "../../../../..")
+          File.join(project_root, "helpers/python/requirements.txt")
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/file_updater/pyproject_preparer.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "toml-rb"
+
+require "dependabot/python/file_parser"
+require "dependabot/python/file_updater"
+
+module Dependabot
+  module Python
+    class FileUpdater
+      class PyprojectPreparer
+        def initialize(pyproject_content:)
+          @pyproject_content = pyproject_content
+        end
+
+        def replace_sources(credentials)
+          pyproject_object = TomlRB.parse(pyproject_content)
+          poetry_object = pyproject_object.fetch("tool").fetch("poetry")
+
+          poetry_object["source"] = pyproject_sources +
+                                    config_variable_sources(credentials)
+
+          TomlRB.dump(pyproject_object)
+        end
+
+        def sanitize
+          # {{ name }} syntax not allowed
+          pyproject_content.gsub(/\{\{.*?\}\}/, "something")
+        end
+
+        # rubocop:disable Metrics/PerceivedComplexity
+        def freeze_top_level_dependencies_except(dependencies, lockfile)
+          return pyproject_content unless lockfile
+
+          pyproject_object = TomlRB.parse(pyproject_content)
+          poetry_object = pyproject_object["tool"]["poetry"]
+          excluded_names = dependencies.map(&:name) + ["python"]
+
+          %w(dependencies dev-dependencies).each do |key|
+            next unless poetry_object[key]
+
+            poetry_object.fetch(key).each do |dep_name, _|
+              next if excluded_names.include?(normalise(dep_name))
+
+              locked_details = locked_details(dep_name, lockfile)
+
+              next unless (locked_version = locked_details&.fetch("version"))
+
+              if locked_details&.dig("source", "type") == "git"
+                poetry_object[key][dep_name] = {
+                  "git" => locked_details&.dig("source", "url"),
+                  "rev" => locked_details&.dig("source", "reference")
+                }
+              elsif poetry_object[dep_name].is_a?(Hash)
+                poetry_object[key][dep_name]["version"] = locked_version
+              else
+                poetry_object[key][dep_name] = locked_version
+              end
+            end
+          end
+
+          TomlRB.dump(pyproject_object)
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        private
+
+        attr_reader :pyproject_content
+
+        def locked_details(dep_name, lockfile)
+          parsed_lockfile = TomlRB.parse(lockfile.content)
+
+          parsed_lockfile.fetch("package").
+            find { |d| d["name"] == normalise(dep_name) }
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalise(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def pyproject_sources
+          return @pyproject_sources if @pyproject_sources
+
+          pyproject_sources ||=
+            TomlRB.parse(pyproject_content).
+            dig("tool", "poetry", "source")
+
+          @pyproject_sources ||=
+            (pyproject_sources || []).
+            map { |h| h.dup.merge("url" => h["url"].gsub(%r{/*$}, "") + "/") }
+        end
+
+        def config_variable_sources(credentials)
+          @config_variable_sources ||=
+            credentials.
+            select { |cred| cred["type"] == "python_index" }.
+            map { |cred| { "url" => cred["index-url"] } }
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/file_updater/requirement_file_updater.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require "dependabot/python/requirement_parser"
+require "dependabot/python/file_updater"
+require "dependabot/shared_helpers"
+require "dependabot/python/native_helpers"
+
+module Dependabot
+  module Python
+    class FileUpdater
+      class RequirementFileUpdater
+        attr_reader :dependencies, :dependency_files, :credentials
+
+        def initialize(dependencies:, dependency_files:, credentials:)
+          @dependencies = dependencies
+          @dependency_files = dependency_files
+          @credentials = credentials
+        end
+
+        def updated_dependency_files
+          return @updated_dependency_files if @update_already_attempted
+
+          @update_already_attempted = true
+          @updated_dependency_files ||= fetch_updated_dependency_files
+        end
+
+        private
+
+        def dependency
+          # For now, we'll only ever be updating a single dependency
+          dependencies.first
+        end
+
+        def fetch_updated_dependency_files
+          reqs = dependency.requirements.zip(dependency.previous_requirements)
+
+          reqs.map do |(new_req, old_req)|
+            next if new_req == old_req
+
+            file = get_original_file(new_req.fetch(:file)).dup
+            updated_content =
+              updated_requirement_or_setup_file_content(new_req, old_req)
+            next if updated_content == file.content
+
+            file.content = updated_content
+            file
+          end.compact
+        end
+
+        def updated_requirement_or_setup_file_content(new_req, old_req)
+          content = get_original_file(new_req.fetch(:file)).content
+
+          updated_content =
+            content.gsub(
+              original_declaration_replacement_regex(old_req),
+              updated_dependency_declaration_string(new_req, old_req)
+            )
+
+          raise "Expected content to change!" if content == updated_content
+
+          updated_content
+        end
+
+        def original_dependency_declaration_string(requirement)
+          regex = RequirementParser::INSTALL_REQ_WITH_REQUIREMENT
+          matches = []
+
+          get_original_file(requirement.fetch(:file)).
+            content.scan(regex) { matches << Regexp.last_match }
+          dec = matches.
+                select { |m| normalise(m[:name]) == dependency.name }.
+                find do |m|
+                  # The FileParser can mess up a requirement's spacing so we
+                  # sanitize both requirements before comparing
+                  f_req = m[:requirements]&.gsub(/\s/, "")&.split(",")&.sort
+                  p_req = requirement.fetch(:requirement)&.
+                          gsub(/\s/, "")&.split(",")&.sort
+                  f_req == p_req
+                end
+
+          raise "Declaration not found for #{dependency.name}!" unless dec
+
+          dec.to_s.strip
+        end
+
+        def updated_dependency_declaration_string(new_req, old_req)
+          updated_string =
+            original_dependency_declaration_string(old_req).sub(
+              RequirementParser::REQUIREMENTS,
+              new_req.fetch(:requirement)
+            )
+          return updated_string unless requirement_includes_hashes?(old_req)
+
+          updated_string.sub(
+            RequirementParser::HASHES,
+            package_hashes_for(
+              name: dependency.name,
+              version: dependency.version,
+              algorithm: hash_algorithm(old_req)
+            ).join(hash_separator(old_req))
+          )
+        end
+
+        def original_declaration_replacement_regex(requirement)
+          original_string =
+            original_dependency_declaration_string(requirement)
+          /(?<![\-\w])#{Regexp.escape(original_string)}(?![\-\w])/
+        end
+
+        def requirement_includes_hashes?(requirement)
+          original_dependency_declaration_string(requirement).
+            match?(RequirementParser::HASHES)
+        end
+
+        def hash_algorithm(requirement)
+          return unless requirement_includes_hashes?(requirement)
+
+          original_dependency_declaration_string(requirement).
+            match(RequirementParser::HASHES).
+            named_captures.fetch("algorithm")
+        end
+
+        def hash_separator(requirement)
+          return unless requirement_includes_hashes?(requirement)
+
+          hash_regex = RequirementParser::HASH
+          current_separator =
+            original_dependency_declaration_string(requirement).
+            match(/#{hash_regex}((?<separator>\s*\\?\s*?)#{hash_regex})*/).
+            named_captures.fetch("separator")
+
+          default_separator =
+            original_dependency_declaration_string(requirement).
+            match(RequirementParser::HASH).
+            pre_match.match(/(?<separator>\s*\\?\s*?)\z/).
+            named_captures.fetch("separator")
+
+          current_separator || default_separator
+        end
+
+        def package_hashes_for(name:, version:, algorithm:)
+          SharedHelpers.run_helper_subprocess(
+            command: "pyenv exec python #{NativeHelpers.python_helper_path}",
+            function: "get_dependency_hash",
+            args: [name, version, algorithm]
+          ).map { |h| "--hash=#{algorithm}:#{h['hash']}" }
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalise(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def get_original_file(filename)
+          dependency_files.find { |f| f.name == filename }
+        end
+      end
+    end
+  end
+end