dependabot-common 0.95.1 → 0.95.2

data/lib/dependabot/metadata_finders/base/changelog_pruner.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require "dependabot/metadata_finders/base"
+
+module Dependabot
+  module MetadataFinders
+    class Base
+      class ChangelogPruner
+        attr_reader :dependency, :changelog_text
+
+        def initialize(dependency:, changelog_text:)
+          @dependency = dependency
+          @changelog_text = changelog_text
+        end
+
+        def includes_new_version?
+          !new_version_changelog_line.nil?
+        end
+
+        # rubocop:disable Metrics/PerceivedComplexity
+        # rubocop:disable Metrics/CyclomaticComplexity
+        def pruned_text
+          changelog_lines = changelog_text.split("\n")
+
+          slice_range =
+            if old_version_changelog_line && new_version_changelog_line
+              if old_version_changelog_line < new_version_changelog_line
+                Range.new(old_version_changelog_line, -1)
+              else
+                Range.new(new_version_changelog_line,
+                          old_version_changelog_line - 1)
+              end
+            elsif old_version_changelog_line
+              return if old_version_changelog_line.zero?
+
+              # Assumes changelog is in descending order
+              Range.new(0, old_version_changelog_line - 1)
+            elsif new_version_changelog_line
+              # Assumes changelog is in descending order
+              Range.new(new_version_changelog_line, -1)
+            else
+              return unless changelog_contains_relevant_versions?
+
+              # If the changelog contains any relevant versions, return it in
+              # full. We could do better here by fully parsing the changelog
+              Range.new(0, -1)
+            end
+
+          changelog_lines.slice(slice_range).join("\n").sub(/\n*\z/, "")
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+        # rubocop:enable Metrics/CyclomaticComplexity
+
+        private
+
+        def old_version_changelog_line
+          old_version = git_source? ? previous_ref : dependency.previous_version
+          return nil unless old_version
+
+          changelog_line_for_version(old_version)
+        end
+
+        def new_version_changelog_line
+          return nil unless new_version
+
+          changelog_line_for_version(new_version)
+        end
+
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/PerceivedComplexity
+        def changelog_line_for_version(version)
+          raise "No changelog text" unless changelog_text
+          return nil unless version
+
+          version = version.gsub(/^v/, "")
+          escaped_version = Regexp.escape(version)
+
+          changelog_lines = changelog_text.split("\n")
+
+          changelog_lines.find_index.with_index do |line, index|
+            next false unless line.match?(/(?<!\.)#{escaped_version}(?![.\-])/)
+            next false if line.match?(/#{escaped_version}\.\./)
+            next true if line.start_with?("#", "!", "==")
+            next true if line.match?(/^v?#{escaped_version}:?/)
+            next true if line.match?(/^[\+\*\-] (version )?#{escaped_version}/i)
+            next true if line.match?(/^\d{4}-\d{2}-\d{2}/)
+            next true if changelog_lines[index + 1]&.match?(/^[=\-\+]{3,}\s*$/)
+
+            false
+          end
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def changelog_contains_relevant_versions?
+          # Assume the changelog is relevant if we can't parse the new version
+          return true unless version_class.correct?(dependency.version)
+
+          # Assume the changelog is relevant if it mentions the new version
+          # anywhere
+          return true if changelog_text.include?(dependency.version)
+
+          # Otherwise check if any intermediate versions are included in headers
+          versions_in_changelog_headers.any? do |version|
+            next false unless version <= version_class.new(dependency.version)
+            next true unless dependency.previous_version
+            next true unless version_class.correct?(dependency.previous_version)
+
+            version > version_class.new(dependency.previous_version)
+          end
+        end
+
+        def versions_in_changelog_headers
+          changelog_lines = changelog_text.split("\n")
+          header_lines =
+            changelog_lines.select.with_index do |line, index|
+              next true if line.start_with?("#", "!")
+              next true if line.match?(/^v?\d\.\d/)
+              next true if changelog_lines[index + 1]&.match?(/^[=-]+\s*$/)
+
+              false
+            end
+
+          versions = []
+          header_lines.each do |line|
+            cleaned_line = line.gsub(/^[^0-9]*/, "").gsub(/[\s,:].*/, "")
+            next if cleaned_line.empty? || !version_class.correct?(cleaned_line)
+
+            versions << version_class.new(cleaned_line)
+          end
+
+          versions
+        end
+
+        def new_version
+          @new_version ||= git_source? ? new_ref : dependency.version
+          @new_version&.gsub(/^v/, "")
+        end
+
+        def previous_ref
+          dependency.previous_requirements.map do |r|
+            r.dig(:source, "ref") || r.dig(:source, :ref)
+          end.compact.first
+        end
+
+        def new_ref
+          dependency.requirements.map do |r|
+            r.dig(:source, "ref") || r.dig(:source, :ref)
+          end.compact.first
+        end
+
+        def ref_changed?
+          previous_ref && new_ref && previous_ref != new_ref
+        end
+
+        # TODO: Refactor me so that Composer doesn't need to be special cased
+        def git_source?
+          # Special case Composer, which uses git as a source but handles tags
+          # internally
+          return false if dependency.package_manager == "composer"
+
+          requirements = dependency.requirements
+          sources = requirements.map { |r| r.fetch(:source) }.uniq.compact
+          return false if sources.empty?
+          raise "Multiple sources! #{sources.join(', ')}" if sources.count > 1
+
+          source_type = sources.first[:type] || sources.first.fetch("type")
+          source_type == "git"
+        end
+
+        def version_class
+          Utils.version_class_for_package_manager(dependency.package_manager)
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/metadata_finders/base/commits_finder.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require "dependabot/clients/github_with_retries"
+require "dependabot/clients/gitlab"
+require "dependabot/clients/bitbucket"
+require "dependabot/shared_helpers"
+require "dependabot/metadata_finders/base"
+
+module Dependabot
+  module MetadataFinders
+    class Base
+      class CommitsFinder
+        attr_reader :source, :dependency, :credentials
+
+        def initialize(source:, dependency:, credentials:)
+          @source = source
+          @dependency = dependency
+          @credentials = credentials
+        end
+
+        def commits_url
+          return unless source
+          return if source.provider == "azure" # TODO: Fetch Azure commits
+
+          path =
+            case source.provider
+            when "github" then github_compare_path(new_tag, previous_tag)
+            when "bitbucket" then bitbucket_compare_path(new_tag, previous_tag)
+            when "gitlab" then gitlab_compare_path(new_tag, previous_tag)
+            else raise "Unexpected source provider '#{source.provider}'"
+            end
+
+          "#{source.url}/#{path}"
+        end
+
+        # rubocop:disable Metrics/CyclomaticComplexity
+        def commits
+          return [] unless source
+          return [] unless new_tag && previous_tag
+
+          case source.provider
+          when "github" then fetch_github_commits
+          when "bitbucket" then fetch_bitbucket_commits
+          when "gitlab" then fetch_gitlab_commits
+          when "azure" then [] # TODO: Fetch Azure commits
+          else raise "Unexpected source provider '#{source.provider}'"
+          end
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity
+
+        def new_tag
+          new_version = dependency.version
+
+          if git_source?(dependency.requirements) then new_version
+          else
+            tags = dependency_tags.
+                   select { |t| t =~ version_regex(new_version) }
+            tags.find { |t| t.include?(dependency.name) } || tags.first
+          end
+        end
+
+        private
+
+        def previous_tag
+          previous_version = dependency.previous_version
+
+          if git_source?(dependency.previous_requirements)
+            previous_version || previous_ref
+          else
+            tags = dependency_tags.
+                   select { |t| t =~ version_regex(previous_version) }
+            tags.find { |t| t.include?(dependency.name) } || tags.first
+          end
+        end
+
+        # TODO: Refactor me so that Composer doesn't need to be special cased
+        def git_source?(requirements)
+          # Special case Composer, which uses git as a source but handles tags
+          # internally
+          return false if dependency.package_manager == "composer"
+
+          sources = requirements.map { |r| r.fetch(:source) }.uniq.compact
+          return false if sources.empty?
+          raise "Multiple sources! #{sources.join(', ')}" if sources.count > 1
+
+          source_type = sources.first[:type] || sources.first.fetch("type")
+          source_type == "git"
+        end
+
+        def previous_ref
+          return unless git_source?(dependency.previous_requirements)
+
+          dependency.previous_requirements.map do |r|
+            r.dig(:source, "ref") || r.dig(:source, :ref)
+          end.compact.first
+        end
+
+        def version_regex(version)
+          /(?:[^0-9\.]|\A)#{Regexp.escape(version || "unknown")}\z/
+        end
+
+        def dependency_tags
+          @dependency_tags ||= fetch_dependency_tags
+        end
+
+        def fetch_dependency_tags
+          return [] unless source
+
+          case source.provider
+          when "github"
+            github_client.tags(source.repo, per_page: 100).map(&:name)
+          when "bitbucket"
+            bitbucket_client.tags(source.repo).map { |tag| tag["name"] }
+          when "gitlab"
+            gitlab_client.tags(source.repo).map(&:name)
+          when "azure"
+            [] # TODO: Fetch Azure tags
+          else raise "Unexpected source provider '#{source.provider}'"
+          end
+        rescue Octokit::NotFound, Gitlab::Error::NotFound,
+               Dependabot::Clients::Bitbucket::NotFound,
+               Dependabot::Clients::Bitbucket::Unauthorized,
+               Dependabot::Clients::Bitbucket::Forbidden
+          []
+        end
+
+        def github_compare_path(new_tag, previous_tag)
+          if new_tag && previous_tag
+            "compare/#{previous_tag}...#{new_tag}"
+          elsif new_tag
+            "commits/#{new_tag}"
+          else
+            "commits"
+          end
+        end
+
+        def bitbucket_compare_path(new_tag, previous_tag)
+          if new_tag && previous_tag
+            "branches/compare/#{new_tag}..#{previous_tag}"
+          elsif new_tag
+            "commits/tag/#{new_tag}"
+          else
+            "commits"
+          end
+        end
+
+        def gitlab_compare_path(new_tag, previous_tag)
+          if new_tag && previous_tag
+            "compare/#{previous_tag}...#{new_tag}"
+          elsif new_tag
+            "commits/#{new_tag}"
+          else
+            "commits/master"
+          end
+        end
+
+        def fetch_github_commits
+          commits =
+            github_client.compare(source.repo, previous_tag, new_tag).commits
+          return [] unless commits
+
+          commits.map do |commit|
+            {
+              message: commit.commit.message,
+              sha: commit.sha,
+              html_url: commit.html_url
+            }
+          end
+        rescue Octokit::NotFound
+          []
+        end
+
+        def fetch_bitbucket_commits
+          bitbucket_client.
+            compare(source.repo, previous_tag, new_tag).
+            map do |commit|
+              {
+                message: commit.dig("summary", "raw"),
+                sha: commit["hash"],
+                html_url: commit.dig("links", "html", "href")
+              }
+            end
+        rescue Dependabot::Clients::Bitbucket::NotFound,
+               Dependabot::Clients::Bitbucket::Unauthorized,
+               Dependabot::Clients::Bitbucket::Forbidden
+          []
+        end
+
+        def fetch_gitlab_commits
+          gitlab_client.
+            compare(source.repo, previous_tag, new_tag).
+            commits.
+            map do |commit|
+              {
+                message: commit["message"],
+                sha: commit["id"],
+                html_url: "#{source.url}/commit/#{commit['id']}"
+              }
+            end
+        rescue Gitlab::Error::NotFound
+          []
+        end
+
+        def gitlab_client
+          @gitlab_client ||= Dependabot::Clients::Gitlab.
+                             for_gitlab_dot_com(credentials: credentials)
+        end
+
+        def github_client
+          @github_client ||= Dependabot::Clients::GithubWithRetries.
+                             for_github_dot_com(credentials: credentials)
+        end
+
+        def bitbucket_client
+          @bitbucket_client ||= Dependabot::Clients::Bitbucket.
+                                for_bitbucket_dot_org(credentials: credentials)
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/metadata_finders/base/release_finder.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+require "dependabot/clients/github_with_retries"
+require "dependabot/clients/gitlab"
+require "dependabot/metadata_finders/base"
+require "dependabot/utils"
+
+module Dependabot
+  module MetadataFinders
+    class Base
+      class ReleaseFinder
+        attr_reader :dependency, :credentials, :source
+
+        def initialize(source:, dependency:, credentials:)
+          @source = source
+          @dependency = dependency
+          @credentials = credentials
+        end
+
+        def releases_url
+          return unless source
+
+          case source.provider
+          when "github" then "#{source.url}/releases"
+          when "gitlab" then "#{source.url}/tags"
+          when "bitbucket" then nil
+          when "azure" then "#{source.url}/tags"
+          else raise "Unexpected repo provider '#{source.provider}'"
+          end
+        end
+
+        def releases_text
+          return unless relevant_releases.any?
+          return if relevant_releases.all? { |r| r.body.nil? || r.body == "" }
+
+          relevant_releases.map { |r| serialize_release(r) }.join("\n\n")
+        end
+
+        private
+
+        def all_dep_releases
+          releases = all_releases
+          dep_prefix = dependency.name.downcase
+
+          releases_with_dependency_name =
+            releases.
+            reject { |r| r.tag_name.nil? }.
+            select { |r| r.tag_name.downcase.include?(dep_prefix) }
+
+          return releases unless releases_with_dependency_name.any?
+
+          releases_with_dependency_name
+        end
+
+        def all_releases
+          @all_releases ||= fetch_dependency_releases
+        end
+
+        def relevant_releases
+          releases = releases_since_previous_version
+
+          # Sometimes we can't filter the releases properly (if they're
+          # prefixed by a number that gets confused with the version). In this
+          # case, the best we can do is return nil.
+          return [] unless releases.any?
+
+          if updated_release && version_class.correct?(dependency.version)
+            releases = filter_releases_using_updated_release(releases)
+            filter_releases_using_updated_version(releases, conservative: true)
+          elsif updated_release
+            filter_releases_using_updated_release(releases)
+          elsif version_class.correct?(dependency.version)
+            filter_releases_using_updated_version(releases, conservative: false)
+          else
+            [updated_release].compact
+          end
+        end
+
+        def releases_since_previous_version
+          previous_version = dependency.previous_version
+          return [updated_release].compact unless previous_version
+
+          if previous_release && version_class.correct?(previous_version)
+            releases = filter_releases_using_previous_release(all_dep_releases)
+            filter_releases_using_previous_version(releases, conservative: true)
+          elsif previous_release
+            filter_releases_using_previous_release(all_dep_releases)
+          elsif version_class.correct?(previous_version)
+            filter_releases_using_previous_version(
+              all_dep_releases,
+              conservative: false
+            )
+          else
+            [updated_release].compact
+          end
+        end
+
+        def filter_releases_using_previous_release(releases)
+          return releases if releases.index(previous_release).nil?
+
+          releases.first(releases.index(previous_release))
+        end
+
+        def filter_releases_using_updated_release(releases)
+          return releases if releases.index(updated_release).nil?
+
+          releases[releases.index(updated_release)..-1]
+        end
+
+        def filter_releases_using_previous_version(releases, conservative:)
+          previous_version = version_class.new(dependency.previous_version)
+
+          releases.reject do |release|
+            cleaned_tag = release.tag_name.gsub(/^[^0-9]*/, "")
+            cleaned_name = release.name&.gsub(/^[^0-9]*/, "")
+
+            tag_version = [cleaned_tag, cleaned_name].compact.reject(&:empty?).
+                          select { |nm| version_class.correct?(nm) }.
+                          map { |nm| version_class.new(nm) }.max
+
+            next conservative unless tag_version
+
+            # Reject any releases that are less than the previous version
+            # (e.g., if two major versions are being maintained)
+            tag_version <= previous_version
+          end
+        end
+
+        def filter_releases_using_updated_version(releases, conservative:)
+          updated_version = version_class.new(dependency.version)
+
+          releases.reject do |release|
+            cleaned_tag = release.tag_name.gsub(/^[^0-9]*/, "")
+            cleaned_name = release.name&.gsub(/^[^0-9]*/, "")
+
+            tag_version = [cleaned_tag, cleaned_name].compact.reject(&:empty?).
+                          select { |nm| version_class.correct?(nm) }.
+                          map { |nm| version_class.new(nm) }.min
+
+            next conservative unless tag_version
+
+            # Reject any releases that are greater than the updated version
+            # (e.g., if two major versions are being maintained)
+            tag_version > updated_version
+          end
+        end
+
+        def updated_release
+          release_for_version(dependency.version)
+        end
+
+        def previous_release
+          release_for_version(dependency.previous_version)
+        end
+
+        def release_for_version(version)
+          return nil unless version
+
+          release_regex = version_regex(version)
+          # Doing two loops looks inefficient, but it ensures consistency
+          all_dep_releases.find { |r| release_regex.match?(r.tag_name.to_s) } ||
+            all_dep_releases.find { |r| release_regex.match?(r.name.to_s) }
+        end
+
+        def serialize_release(release)
+          rel = release
+          title = "## #{rel.name.to_s != '' ? rel.name : rel.tag_name}\n"
+          body = if rel.body.to_s.gsub(/\n*\z/m, "") == ""
+                   "No release notes provided."
+                 else
+                   rel.body.gsub(/\n*\z/m, "")
+                 end
+
+          release_body_includes_title?(rel) ? body : title + body
+        end
+
+        def release_body_includes_title?(release)
+          title = release.name.to_s != "" ? release.name : release.tag_name
+          release.body.to_s.match?(/\A\s*\#*\s*#{Regexp.quote(title)}/m)
+        end
+
+        def version_regex(version)
+          /(?:[^0-9\.]|\A)#{Regexp.escape(version || "unknown")}\z/
+        end
+
+        def version_class
+          Utils.version_class_for_package_manager(dependency.package_manager)
+        end
+
+        def fetch_dependency_releases
+          return [] unless source
+
+          case source.provider
+          when "github" then fetch_github_releases
+          when "bitbucket" then [] # Bitbucket doesn't support releases
+          when "gitlab" then fetch_gitlab_releases
+          when "azure" then [] # Azure can't list API for annotated tags
+          else raise "Unexpected repo provider '#{source.provider}'"
+          end
+        end
+
+        def fetch_github_releases
+          releases = github_client.releases(source.repo, per_page: 100)
+
+          # Remove any releases without a tag name. These are draft releases and
+          # aren't yet associated with a tag, so shouldn't be used.
+          releases = releases.reject { |r| r.tag_name.nil? }
+
+          clean_release_names =
+            releases.map { |r| r.tag_name.gsub(/^[^0-9\.]*/, "") }
+
+          if clean_release_names.all? { |nm| version_class.correct?(nm) }
+            releases.sort_by do |r|
+              version_class.new(r.tag_name.gsub(/^[^0-9\.]*/, ""))
+            end.reverse
+          else
+            releases.sort_by(&:id).reverse
+          end
+        rescue Octokit::NotFound
+          []
+        end
+
+        def fetch_gitlab_releases
+          releases =
+            gitlab_client.
+            tags(source.repo).
+            select(&:release).
+            sort_by { |r| r.commit.authored_date }.
+            reverse
+
+          releases.map do |tag|
+            OpenStruct.new(
+              name: tag.name,
+              tag_name: tag.release.tag_name,
+              body: tag.release.description,
+              html_url: "#{source.url}/tags/#{tag.name}"
+            )
+          end
+        rescue Gitlab::Error::NotFound
+          []
+        end
+
+        def gitlab_client
+          @gitlab_client ||= Dependabot::Clients::Gitlab.
+                             for_gitlab_dot_com(credentials: credentials)
+        end
+
+        def github_client
+          @github_client ||= Dependabot::Clients::GithubWithRetries.
+                             for_github_dot_com(credentials: credentials)
+        end
+      end
+    end
+  end
+end