gitlab-dangerfiles 0.1.0

data/lib/danger/sidekiq_queues.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Danger
+  # Common helper functions for our danger scripts. See Danger::Helper
+  # for more details
+  class SidekiqQueues < Danger::Plugin
+    def changed_queue_files
+      @changed_queue_files ||= git.modified_files.grep(%r{\A(ee/)?app/workers/all_queues\.yml})
+    end
+
+    def added_queue_names
+      @added_queue_names ||= new_queues.keys - old_queues.keys
+    end
+
+    def changed_queue_names
+      @changed_queue_names ||=
+        (new_queues.values_at(*old_queues.keys) - old_queues.values)
+          .compact.map { |queue| queue[:name] }
+    end
+
+    private
+
+    def old_queues
+      @old_queues ||= queues_for(gitlab.base_commit)
+    end
+
+    def new_queues
+      @new_queues ||= queues_for(gitlab.head_commit)
+    end
+
+    def queues_for(branch)
+      changed_queue_files
+        .flat_map { |file| YAML.safe_load(`git show #{branch}:#{file}`, permitted_classes: [Symbol]) }
+        .to_h { |queue| [queue[:name], queue] }
+    end
+  end
+end

data/lib/gitlab-dangerfiles.rb

@@ -0,0 +1 @@
+require_relative "gitlab/dangerfiles"

data/lib/gitlab/Dangerfile

@@ -0,0 +1 @@
+danger.import_plugin(File.expand_path("../danger/*.rb", __dir__))

data/lib/gitlab/dangerfiles.rb

@@ -0,0 +1,40 @@
+require "gitlab/dangerfiles/version"
+
+module Gitlab
+  module Dangerfiles
+    LOCAL_RULES ||= %w[
+      changes_size
+      commit_messages
+      database
+      documentation
+      duplicate_yarn_dependencies
+      eslint
+      frozen_string
+      karma
+      prettier
+      telemetry
+      utility_css
+    ].freeze
+    CI_ONLY_RULES ||= %w[
+      ce_ee_vue_templates
+      changelog
+      metadata
+      roulette
+      sidekiq_queues
+      specs
+    ].freeze
+    MESSAGE_PREFIX = "==>".freeze
+
+    def self.load_tasks
+      require "gitlab/dangerfiles/tasks"
+    end
+
+    def self.local_warning_message
+      "#{MESSAGE_PREFIX} Only the following Danger rules can be run locally: #{LOCAL_RULES.join(", ")}"
+    end
+
+    def self.success_message
+      "#{MESSAGE_PREFIX} No Danger rule violations!"
+    end
+  end
+end

data/lib/gitlab/dangerfiles/bundle_size/Dangerfile

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+analysis_result = "./bundle-size-review/analysis.json"
+markdown_result = "./bundle-size-review/comparison.md"
+
+# Executing the webpack-entry-point-analyser
+# We would like to do that in the CI file directly,
+# but unfortunately the head_commit SHA is not available
+# as a CI variable due to our merge into master simulation
+analyze_cmd = [
+  "webpack-entry-point-analyser",
+  "--from-file ./webpack-report/stats.json",
+  "--json #{analysis_result}",
+  " --sha #{gitlab&.head_commit}"
+].join(" ")
+
+# execute analysis
+`#{analyze_cmd}`
+
+# We are executing the comparison by comparing the start_sha
+# to the current pipeline result. The start_sha is the commit
+# from master that was merged into for the merged pipeline.
+comparison_cmd = [
+  "webpack-compare-reports",
+  "--job #{ENV["CI_JOB_ID"]}",
+  "--to-file #{analysis_result}",
+  "--html ./bundle-size-review/comparison.html",
+  "--markdown #{markdown_result}"
+].join(" ")
+
+# execute comparison
+`#{comparison_cmd}`
+
+comment = `cat #{markdown_result}`
+
+markdown(<<~MARKDOWN)
+    #{comment}
+MARKDOWN

data/lib/gitlab/dangerfiles/ce_ee_vue_templates/Dangerfile

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+require 'cgi'
+
+def get_vue_files_with_ce_and_ee_versions(files)
+  files.select do |file|
+    if file.end_with?('.vue')
+      counterpart_path = if file.start_with?('ee/')
+                           file.delete_prefix('ee/')
+                         else
+                           "ee/#{file}"
+                         end
+
+      escaped_path = CGI.escape(counterpart_path)
+      api_endpoint = "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-ee/repository/files/#{escaped_path}?ref=master"
+      response = HTTParty.get(api_endpoint) # rubocop:disable Gitlab/HTTParty
+      response.code != 404
+    else
+      false
+    end
+  end
+end
+
+vue_candidates = get_vue_files_with_ce_and_ee_versions(helper.all_changed_files)
+
+return if vue_candidates.empty?
+
+message 'This merge request includes changes to Vue files that have both CE and EE versions.'
+
+markdown(<<~MARKDOWN)
+  ## Vue `<template>` in CE and EE
+
+  Some Vue files in CE have a counterpart in EE.
+  (For example, `path/to/file.vue` and `ee/path/to/file.vue`.)
+
+  When run in the context of CE, the `<template>` of the CE Vue file is used.
+  When run in the context of EE, the `<template>` of the EE Vue file is used.
+
+  It's easy to accidentally make a change to a CE `<template>` that _should_
+  appear in both CE and EE without making the change in both places.
+  When this happens, the change only takes effect in CE.
+
+  The following Vue files were changed as part of this merge request that
+  include both a CE and EE version of the file:
+
+  * #{vue_candidates.map { |path| "`#{path}`" }.join("\n* ")}
+
+  If you made a change to the `<template>` of any of these Vue files that
+  should be visible in both CE and EE, please ensure you have made your
+  change to both versions of the file.
+
+  ### A better alternative
+
+  An even _better_ alternative is to refactor this component to only use
+  a single template for both CE and EE.  More info on this approach here:
+  https://docs.gitlab.com/ee/development/ee_features.html#template-tag
+MARKDOWN

data/lib/gitlab/dangerfiles/changelog/Dangerfile

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+# rubocop:disable Style/SignalException
+
+require 'yaml'
+
+SEE_DOC = "See the [changelog documentation](https://docs.gitlab.com/ee/development/changelog.html)."
+CREATE_CHANGELOG_MESSAGE = <<~MSG
+If you want to create a changelog entry for GitLab FOSS, run the following:
+
+```
+bin/changelog -m %<mr_iid>s "%<mr_title>s"
+```
+
+If you want to create a changelog entry for GitLab EE, run the following instead:
+
+```
+bin/changelog --ee -m %<mr_iid>s "%<mr_title>s"
+```
+
+If this merge request [doesn't need a CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry), feel free to ignore this message.
+MSG
+
+SUGGEST_MR_COMMENT = <<~SUGGEST_COMMENT
+```suggestion
+merge_request: %<mr_iid>s
+```
+
+#{SEE_DOC}
+SUGGEST_COMMENT
+
+def check_changelog_yaml(path)
+  raw_file = File.read(path)
+  yaml = YAML.safe_load(raw_file)
+
+  fail "`title` should be set, in #{helper.html_link(path)}! #{SEE_DOC}" if yaml["title"].nil?
+  fail "`type` should be set, in #{helper.html_link(path)}! #{SEE_DOC}" if yaml["type"].nil?
+
+  return if helper.security_mr?
+
+  cherry_pick_against_stable_branch = helper.cherry_pick_mr? && helper.stable_branch?
+
+  if yaml["merge_request"].nil?
+    mr_line = raw_file.lines.find_index("merge_request:\n")
+
+    if mr_line
+      markdown(format(SUGGEST_MR_COMMENT, mr_iid: gitlab.mr_json["iid"]), file: path, line: mr_line.succ)
+    else
+      message "Consider setting `merge_request` to #{gitlab.mr_json["iid"]} in #{helper.html_link(path)}. #{SEE_DOC}"
+    end
+  elsif yaml["merge_request"] != gitlab.mr_json["iid"] && !cherry_pick_against_stable_branch
+    fail "Merge request ID was not set to #{gitlab.mr_json["iid"]}! #{SEE_DOC}"
+  end
+rescue Psych::SyntaxError, Psych::DisallowedClass, Psych::BadAlias
+  # YAML could not be parsed, fail the build.
+  fail "#{helper.html_link(path)} isn't valid YAML! #{SEE_DOC}"
+rescue StandardError => e
+  warn "There was a problem trying to check the Changelog. Exception: #{e.class.name} - #{e.message}"
+end
+
+def check_changelog_path(path)
+  ee_changes = helper.all_ee_changes.dup
+  ee_changes.delete(path)
+
+  if ee_changes.any? && !changelog.ee_changelog?
+    warn "This MR has a Changelog file outside `ee/`, but code changes in `ee/`. Consider moving the Changelog file into `ee/`."
+  end
+
+  if ee_changes.empty? && changelog.ee_changelog?
+    warn "This MR has a Changelog file in `ee/`, but no code changes in `ee/`. Consider moving the Changelog file outside `ee/`."
+  end
+end
+
+def sanitized_mr_title
+  helper.sanitize_mr_title(gitlab.mr_json["title"])
+end
+
+if git.modified_files.include?("CHANGELOG.md")
+  fail "**CHANGELOG.md was edited.** Please remove the additions and create a CHANGELOG entry.\n\n" +
+    format(CREATE_CHANGELOG_MESSAGE, mr_iid: gitlab.mr_json["iid"], mr_title: sanitized_mr_title)
+end
+
+changelog_found = changelog.found
+
+if changelog_found
+  check_changelog_yaml(changelog_found)
+  check_changelog_path(changelog_found)
+elsif changelog.needed?
+  message "**[CHANGELOG missing](https://docs.gitlab.com/ee/development/changelog.html)**:\n\n" +
+          format(CREATE_CHANGELOG_MESSAGE, mr_iid: gitlab.mr_json["iid"], mr_title: sanitized_mr_title)
+end

data/lib/gitlab/dangerfiles/changes_size/Dangerfile

@@ -0,0 +1,17 @@
+# FIXME: git.info_for_file raises the following error
+# /usr/local/bundle/gems/git-1.4.0/lib/git/lib.rb:956:in `command':  (Danger::DSLError)
+# [!] Invalid `Dangerfile` file:
+# [!] Invalid `Dangerfile` file: git '--git-dir=/builds/gitlab-org/gitlab/.git' '--work-tree=/builds/gitlab-org/gitlab' cat-file '-t' ''  2>&1:fatal: Not a valid object name
+# This seems to be the same as https://github.com/danger/danger/issues/535.
+
+# locale_files_updated = git.modified_files.select { |path| path.start_with?('locale') }
+# locale_files_updated.each do |locale_file_updated|
+#   git_stats = git.info_for_file(locale_file_updated)
+#   message "Git stats for #{locale_file_updated}: #{git_stats[:insertions]} insertions, #{git_stats[:deletions]} insertions"
+# end
+
+if git.lines_of_code > 2_000
+  warn "This merge request is definitely too big (more than #{git.lines_of_code} lines changed), please split it into multiple merge requests."
+elsif git.lines_of_code > 500
+  warn "This merge request is quite big  (more than #{git.lines_of_code} lines changed), please consider splitting it into multiple merge requests."
+end

data/lib/gitlab/dangerfiles/commit_linter.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+emoji_checker_path = File.expand_path("emoji_checker", __dir__)
+defined?(Rails) ? require_dependency(emoji_checker_path) : require_relative(emoji_checker_path)
+
+module Gitlab
+  module Dangerfiles
+    class CommitLinter
+      MIN_SUBJECT_WORDS_COUNT = 3
+      MAX_LINE_LENGTH = 72
+      MAX_CHANGED_FILES_IN_COMMIT = 3
+      MAX_CHANGED_LINES_IN_COMMIT = 30
+      SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(#|!|&|%)\d+\b}.freeze
+      DEFAULT_SUBJECT_DESCRIPTION = "commit subject"
+      WIP_PREFIX = "WIP: "
+      PROBLEMS = {
+        subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words",
+        subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters",
+        subject_starts_with_lowercase: "The %s must start with a capital letter",
+        subject_ends_with_a_period: "The %s must not end with a period",
+        separator_missing: "The commit subject and body must be separated by a blank line",
+        details_too_many_changes: "Commits that change #{MAX_CHANGED_LINES_IN_COMMIT} or more lines across " \
+        "at least #{MAX_CHANGED_FILES_IN_COMMIT} files must describe these changes in the commit body",
+        details_line_too_long: "The commit body should not contain more than #{MAX_LINE_LENGTH} characters per line",
+        message_contains_text_emoji: "Avoid the use of Markdown Emoji such as `:+1:`. These add limited value " \
+        "to the commit message, and are displayed as plain text outside of GitLab",
+        message_contains_unicode_emoji: "Avoid the use of Unicode Emoji. These add no value to the commit " \
+        "message, and may not be displayed properly everywhere",
+        message_contains_short_reference: "Use full URLs instead of short references (`gitlab-org/gitlab#123` or " \
+        "`!123`), as short references are displayed as plain text outside of GitLab",
+      }.freeze
+
+      attr_reader :commit, :problems
+
+      def initialize(commit)
+        @commit = commit
+        @problems = {}
+        @linted = false
+      end
+
+      def fixup?
+        commit.message.start_with?("fixup!", "squash!")
+      end
+
+      def suggestion?
+        commit.message.start_with?("Apply suggestion to")
+      end
+
+      def merge?
+        commit.message.start_with?("Merge branch")
+      end
+
+      def revert?
+        commit.message.start_with?('Revert "')
+      end
+
+      def multi_line?
+        !details.nil? && !details.empty?
+      end
+
+      def failed?
+        problems.any?
+      end
+
+      def add_problem(problem_key, *args)
+        @problems[problem_key] = sprintf(PROBLEMS[problem_key], *args)
+      end
+
+      def lint(subject_description = "commit subject")
+        return self if @linted
+
+        @linted = true
+        lint_subject(subject_description)
+        lint_separator
+        lint_details
+        lint_message
+
+        self
+      end
+
+      def lint_subject(subject_description)
+        if subject_too_short?
+          add_problem(:subject_too_short, subject_description)
+        end
+
+        if subject_too_long?
+          add_problem(:subject_too_long, subject_description)
+        end
+
+        if subject_starts_with_lowercase?
+          add_problem(:subject_starts_with_lowercase, subject_description)
+        end
+
+        if subject_ends_with_a_period?
+          add_problem(:subject_ends_with_a_period, subject_description)
+        end
+
+        self
+      end
+
+      private
+
+      def lint_separator
+        return self unless separator && !separator.empty?
+
+        add_problem(:separator_missing)
+
+        self
+      end
+
+      def lint_details
+        if !multi_line? && many_changes?
+          add_problem(:details_too_many_changes)
+        end
+
+        details&.each_line do |line|
+          line = line.strip
+
+          next unless line_too_long?(line)
+
+          url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # rubocop:disable CodeReuse/ActiveRecord
+
+          # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but
+          # only if the line _without_ the URL does not exceed this limit.
+          next unless line_too_long?(line.length - url_size)
+
+          add_problem(:details_line_too_long)
+          break
+        end
+
+        self
+      end
+
+      def lint_message
+        if message_contains_text_emoji?
+          add_problem(:message_contains_text_emoji)
+        end
+
+        if message_contains_unicode_emoji?
+          add_problem(:message_contains_unicode_emoji)
+        end
+
+        if message_contains_short_reference?
+          add_problem(:message_contains_short_reference)
+        end
+
+        self
+      end
+
+      def files_changed
+        commit.diff_parent.stats[:total][:files]
+      end
+
+      def lines_changed
+        commit.diff_parent.stats[:total][:lines]
+      end
+
+      def many_changes?
+        files_changed > MAX_CHANGED_FILES_IN_COMMIT && lines_changed > MAX_CHANGED_LINES_IN_COMMIT
+      end
+
+      def subject
+        message_parts[0].delete_prefix(WIP_PREFIX)
+      end
+
+      def separator
+        message_parts[1]
+      end
+
+      def details
+        message_parts[2]&.gsub(/^Signed-off-by.*$/, "")
+      end
+
+      def line_too_long?(line)
+        case line
+        when String
+          line.length > MAX_LINE_LENGTH
+        when Integer
+          line > MAX_LINE_LENGTH
+        else
+          raise ArgumentError, "The line argument (#{line}) should be a String or an Integer! #{line.class} given."
+        end
+      end
+
+      def subject_too_short?
+        subject.split(" ").length < MIN_SUBJECT_WORDS_COUNT
+      end
+
+      def subject_too_long?
+        line_too_long?(subject)
+      end
+
+      def subject_starts_with_lowercase?
+        first_char = subject.sub(/\A\[.+\]\s/, "")[0]
+        first_char_downcased = first_char.downcase
+        return true unless ("a".."z").cover?(first_char_downcased)
+
+        first_char.downcase == first_char
+      end
+
+      def subject_ends_with_a_period?
+        subject.end_with?(".")
+      end
+
+      def message_contains_text_emoji?
+        emoji_checker.includes_text_emoji?(commit.message)
+      end
+
+      def message_contains_unicode_emoji?
+        emoji_checker.includes_unicode_emoji?(commit.message)
+      end
+
+      def message_contains_short_reference?
+        commit.message.match?(SHORT_REFERENCE_REGEX)
+      end
+
+      def emoji_checker
+        @emoji_checker ||= Gitlab::Dangerfiles::EmojiChecker.new
+      end
+
+      def message_parts
+        @message_parts ||= commit.message.split("\n", 3)
+      end
+    end
+  end
+end