gitlab-dangerfiles 0.1.0

data/gitlab-dangerfiles.gemspec

@@ -0,0 +1,38 @@
+require_relative "lib/gitlab/dangerfiles/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "gitlab-dangerfiles"
+  spec.version = Gitlab::Dangerfiles::VERSION
+  spec.authors = ["Rémy Coutable"]
+  spec.email = ["remy@rymai.me"]
+
+  spec.summary = %q{This gem provides common Dangerfile and plugins for GitLab projects.}
+  spec.description = %q{This gem provides common Dangerfile and plugins for GitLab projects.}
+  spec.homepage = "https://gitlab.com/gitlab-org/gitlab-dangerfiles"
+  spec.license = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://gitlab.com/gitlab-org/gitlab-dangerfiles"
+  spec.metadata["changelog_uri"] = "https://gitlab.com/gitlab-org/gitlab-dangerfiles"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "danger"
+
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec-parameterized"
+  spec.add_development_dependency "timecop"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "climate_control"
+  spec.add_development_dependency "rufo"
+end

data/lib/danger/changelog.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Danger
+  # Common helper functions for our danger scripts. See Danger::Helper
+  # for more details
+  class Changelog < Danger::Plugin
+    NO_CHANGELOG_LABELS = [
+      "backstage", # To be removed by https://gitlab.com/gitlab-org/gitlab/-/issues/222360.
+      "tooling",
+      "tooling::pipelines",
+      "tooling::workflow",
+      "ci-build",
+      "meta",
+    ].freeze
+    NO_CHANGELOG_CATEGORIES = %i[docs none].freeze
+
+    def needed?
+      categories_need_changelog? && (gitlab.mr_labels & NO_CHANGELOG_LABELS).empty?
+    end
+
+    def found
+      @found ||= git.added_files.find { |path| path =~ %r{\A(ee/)?(changelogs/unreleased)(-ee)?/} }
+    end
+
+    def sanitized_mr_title
+      gitlab.mr_json["title"].gsub(/^WIP: */, "").gsub(/`/, '\\\`')
+    end
+
+    def ee_changelog?
+      found.start_with?("ee/")
+    end
+
+    private
+
+    def categories_need_changelog?
+      (helper.changes_by_category.keys - NO_CHANGELOG_CATEGORIES).any?
+    end
+  end
+end

data/lib/danger/helper.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require_relative "../gitlab/dangerfiles/teammate"
+
+module Danger
+  # Common helper functions for our danger scripts.
+  class Helper < Danger::Plugin
+    RELEASE_TOOLS_BOT = "gitlab-release-tools-bot"
+    CATEGORY_LABELS = {
+      docs: "~documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now.
+      none: "",
+      qa: "~QA",
+      test: "~test ~Quality for `spec/features/*`",
+      engineering_productivity: '~"Engineering Productivity" for CI, Danger',
+    }.freeze
+    # First-match win, so be sure to put more specific regex at the top...
+    CATEGORIES = {
+      %r{\Adoc/} => :docs,
+      %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs,
+
+      %r{\A(ee/)?app/(assets|views)/} => :frontend,
+      %r{\A(ee/)?public/} => :frontend,
+      %r{\A(ee/)?spec/(javascripts|frontend)/} => :frontend,
+      %r{\A(ee/)?vendor/assets/} => :frontend,
+      %r{\A(ee/)?scripts/frontend/} => :frontend,
+      %r{(\A|/)(
+        \.babelrc |
+        \.eslintignore |
+        \.eslintrc(\.yml)? |
+        \.nvmrc |
+        \.prettierignore |
+        \.prettierrc |
+        \.scss-lint.yml |
+        \.stylelintrc |
+        \.haml-lint.yml |
+        \.haml-lint_todo.yml |
+        babel\.config\.js |
+        jest\.config\.js |
+        package\.json |
+        yarn\.lock |
+        config/.+\.js
+      )\z}x => :frontend,
+
+      %r{(\A|/)(
+        \.gitlab/ci/frontend\.gitlab-ci\.yml
+      )\z}x => %i[frontend engineering_productivity],
+
+      %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database,
+      %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database,
+      %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database,
+      %r{\A(ee/)?app/finders/} => :database,
+      %r{\Arubocop/cop/migration(/|\.rb)} => :database,
+
+      %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity,
+      %r{\A\.codeclimate\.yml\z} => :engineering_productivity,
+      %r{\A\.overcommit\.yml\.example\z} => :engineering_productivity,
+      %r{\A\.editorconfig\z} => :engineering_productivity,
+      %r{Dangerfile\z} => :engineering_productivity,
+      %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity,
+      %r{\A(ee/)?scripts/} => :engineering_productivity,
+      %r{\Atooling/} => :engineering_productivity,
+
+      %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend,
+      %r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend,
+      %r{\A(ee/)?spec/features/} => :test,
+      %r{\A(ee/)?spec/} => :backend,
+      %r{\A(ee/)?vendor/} => :backend,
+      %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend,
+      %r{\A[A-Z_]+_VERSION\z} => :backend,
+      %r{\A\.rubocop(_todo)?\.yml\z} => :backend,
+      %r{\Afile_hooks/} => :backend,
+
+      %r{\A(ee/)?qa/} => :qa,
+
+      # Files that don't fit into any category are marked with :none
+      %r{\A(ee/)?changelogs/} => :none,
+      %r{\Alocale/gitlab\.pot\z} => :none,
+
+      # Fallbacks in case the above patterns miss anything
+      %r{\.rb\z} => :backend,
+      %r{(
+        \.(md|txt)\z |
+        \.markdownlint\.json
+      )}x => :none, # To reinstate roulette for documentation, set to `:docs`.
+      %r{\.js\z} => :frontend,
+    }.freeze
+
+    HTTPError = Class.new(StandardError)
+
+    def gitlab_helper
+      # Unfortunately the following does not work:
+      # - respond_to?(:gitlab)
+      # - respond_to?(:gitlab, true)
+      gitlab
+    rescue NoMethodError
+      nil
+    end
+
+    def rule_names
+      ci? ? Gitlab::Dangerfiles::LOCAL_RULES | Gitlab::Dangerfiles::CI_ONLY_RULES : Gitlab::Dangerfiles::LOCAL_RULES
+    end
+
+    def html_link(str)
+      ci? ? gitlab_helper.html_link(str) : str
+    end
+
+    def ci?
+      !gitlab_helper.nil?
+    end
+
+    # @param [String] url
+    def http_get_json(url)
+      rsp = Net::HTTP.get_response(URI.parse(url))
+
+      unless rsp.is_a?(Net::HTTPOK)
+        raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}"
+      end
+
+      JSON.parse(rsp.body)
+    end
+
+    # Returns a list of all files that have been added, modified or renamed.
+    # `git.modified_files` might contain paths that already have been renamed,
+    # so we need to remove them from the list.
+    #
+    # Considering these changes:
+    #
+    # - A new_file.rb
+    # - D deleted_file.rb
+    # - M modified_file.rb
+    # - R renamed_file_before.rb -> renamed_file_after.rb
+    #
+    # it will return
+    # ```
+    # [ 'new_file.rb', 'modified_file.rb', 'renamed_file_after.rb' ]
+    # ```
+    #
+    # @return [Array<String>]
+    def all_changed_files
+      Set.new
+        .merge(git.added_files.to_a)
+        .merge(git.modified_files.to_a)
+        .merge(git.renamed_files.map { |x| x[:after] })
+        .subtract(git.renamed_files.map { |x| x[:before] })
+        .to_a
+        .sort
+    end
+
+    def all_ee_changes
+      all_changed_files.grep(%r{\Aee/})
+    end
+
+    def ee?
+      # Support former project name for `dev` and support local Danger run
+      %w[gitlab gitlab-ee].include?(ENV["CI_PROJECT_NAME"]) || Dir.exist?("../../ee")
+    end
+
+    def release_automation?
+      gitlab_helper&.mr_author == RELEASE_TOOLS_BOT
+    end
+
+    def project_name
+      ee? ? "gitlab" : "gitlab-foss"
+    end
+
+    def markdown_list(items)
+      list = items.map { |item| "* `#{item}`" }.join("\n")
+
+      if items.size > 10
+        "\n<details>\n\n#{list}\n\n</details>\n"
+      else
+        list
+      end
+    end
+
+    # @return [Hash<String,Array<String>>]
+    def changes_by_category
+      all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash|
+        categories_for_file(file).each { |category| hash[category] << file }
+      end
+    end
+
+    # Determines the categories a file is in, e.g., `[:frontend]`, `[:backend]`, or  `%i[frontend engineering_productivity]`.
+    # @return Array<Symbol>
+    def categories_for_file(file)
+      _, categories = CATEGORIES.find { |regexp, _| regexp.match?(file) }
+
+      Array(categories || :unknown)
+    end
+
+    # Returns the GFM for a category label, making its best guess if it's not
+    # a category we know about.
+    #
+    # @return[String]
+    def label_for_category(category)
+      CATEGORY_LABELS.fetch(category, "~#{category}")
+    end
+
+    def new_teammates(usernames)
+      usernames.map { |u| Gitlab::Dangerfiles::Teammate.new("username" => u) }
+    end
+
+    def missing_database_labels(current_mr_labels)
+      labels = if has_database_scoped_labels?(current_mr_labels)
+          ["database"]
+        else
+          ["database", "database::review pending"]
+        end
+
+      labels - current_mr_labels
+    end
+
+    def sanitize_mr_title(title)
+      title.gsub(/^WIP: */, "").gsub(/`/, '\\\`')
+    end
+
+    def security_mr?
+      return false unless gitlab_helper
+
+      gitlab_helper.mr_json["web_url"].include?("/gitlab-org/security/")
+    end
+
+    def cherry_pick_mr?
+      return false unless gitlab_helper
+
+      /cherry[\s-]*pick/i.match?(gitlab_helper.mr_json["title"])
+    end
+
+    def stable_branch?
+      return false unless gitlab_helper
+
+      /\A\d+-\d+-stable-ee/i.match?(gitlab_helper.mr_json["target_branch"])
+    end
+
+    def mr_has_labels?(*labels)
+      return false unless gitlab_helper
+
+      labels = labels.flatten.uniq
+      (labels & gitlab_helper.mr_labels) == labels
+    end
+
+    def labels_list(labels, sep: ", ")
+      labels.map { |label| %Q{~"#{label}"} }.join(sep)
+    end
+
+    def prepare_labels_for_mr(labels)
+      return "" unless labels.any?
+
+      "/label #{labels_list(labels, sep: " ")}"
+    end
+
+    private
+
+    def has_database_scoped_labels?(current_mr_labels)
+      current_mr_labels.any? { |label| label.start_with?("database::") }
+    end
+  end
+end

data/lib/danger/roulette.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require_relative "../gitlab/dangerfiles/teammate"
+
+module Danger
+  # Common helper functions for our danger scripts. See Danger::Helper
+  # for more details
+  class Roulette < Danger::Plugin
+    ROULETTE_DATA_URL = "https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json"
+    HOURS_WHEN_PERSON_CAN_BE_PICKED = (6..14).freeze
+
+    Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role)
+
+    # Assigns GitLab team members to be reviewer and maintainer
+    # for each change category that a Merge Request contains.
+    #
+    # @return [Array<Spin>]
+    def spin(project, categories, branch_name, timezone_experiment: false)
+      team = begin
+          project_team(project)
+        rescue => err
+          warn("Reviewer roulette failed to load team data: #{err.message}")
+          []
+        end
+
+      canonical_branch_name = canonical_branch_name(branch_name)
+
+      spin_per_category = categories.each_with_object({}) do |category, memo|
+        memo[category] = spin_for_category(team, project, category, canonical_branch_name, timezone_experiment: timezone_experiment)
+      end
+
+      spin_per_category.map do |category, spin|
+        case category
+        when :test
+          if spin.reviewer.nil?
+            # Fetch an already picked backend reviewer, or pick one otherwise
+            spin.reviewer = spin_per_category[:backend]&.reviewer || spin_for_category(team, project, :backend, canonical_branch_name).reviewer
+          end
+        when :engineering_productivity
+          if spin.maintainer.nil?
+            # Fetch an already picked backend maintainer, or pick one otherwise
+            spin.maintainer = spin_per_category[:backend]&.maintainer || spin_for_category(team, project, :backend, canonical_branch_name).maintainer
+          end
+        end
+
+        spin
+      end
+    end
+
+    # Looks up the current list of GitLab team members and parses it into a
+    # useful form
+    #
+    # @return [Array<Teammate>]
+    def team
+      @team ||= begin
+          data = helper.http_get_json(ROULETTE_DATA_URL)
+          data.map { |hash| Gitlab::Dangerfiles::Teammate.new(hash) }
+        rescue JSON::ParserError
+          raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}"
+        end
+    end
+
+    # Like +team+, but only returns teammates in the current project, based on
+    # project_name.
+    #
+    # @return [Array<Teammate>]
+    def project_team(project_name)
+      team.select { |member| member.in_project?(project_name) }
+    end
+
+    def canonical_branch_name(branch_name)
+      branch_name.gsub(/^[ce]e-|-[ce]e$/, "")
+    end
+
+    def new_random(seed)
+      Random.new(Digest::MD5.hexdigest(seed).to_i(16))
+    end
+
+    # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the
+    # selection will change on next spin
+    # @param [Array<Teammate>] people
+    def spin_for_person(people, random:, timezone_experiment: false)
+      shuffled_people = people.shuffle(random: random)
+
+      if timezone_experiment
+        shuffled_people.find(&method(:valid_person_with_timezone?))
+      else
+        shuffled_people.find(&method(:valid_person?))
+      end
+    end
+
+    private
+
+    # @param [Teammate] person
+    # @return [Boolean]
+    def valid_person?(person)
+      !mr_author?(person) && person.available
+    end
+
+    # @param [Teammate] person
+    # @return [Boolean]
+    def valid_person_with_timezone?(person)
+      valid_person?(person) && HOURS_WHEN_PERSON_CAN_BE_PICKED.cover?(person.local_hour)
+    end
+
+    # @param [Teammate] person
+    # @return [Boolean]
+    def mr_author?(person)
+      person.username == gitlab.mr_author
+    end
+
+    def spin_role_for_category(team, role, project, category)
+      team.select do |member|
+        member.public_send("#{role}?", project, category, gitlab.mr_labels) # rubocop:disable GitlabSecurity/PublicSend
+      end
+    end
+
+    def spin_for_category(team, project, category, branch_name, timezone_experiment: false)
+      reviewers, traintainers, maintainers =
+        %i[reviewer traintainer maintainer].map do |role|
+          spin_role_for_category(team, role, project, category)
+        end
+
+      # TODO: take CODEOWNERS into account?
+      # https://gitlab.com/gitlab-org/gitlab/issues/26723
+
+      # Make traintainers have triple the chance to be picked as a reviewer
+      random = new_random(branch_name)
+      reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random, timezone_experiment: timezone_experiment)
+      maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment)
+
+      Spin.new(category, reviewer, maintainer)
+    end
+  end
+end