gitlab-housekeeper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8b911737b1ef300020b8cd648c514329d2eee7dfd8f86610727d8f4c4c47bd20
+  data.tar.gz: 47ef5bec137ec296a9bc61032aaee4c3be6e26fd8d404a1b4483f1797a8ad2bf
+SHA512:
+  metadata.gz: 767b5444231e67c439b7bd659c2e974445041e3e8140ed2d7c3336aecf3f7a5628505122cf32a83be68f5277578afe04b0ea74cfa6402108f466850a9d827a74
+  data.tar.gz: 6f50658e7fc7d37694af6f504a046bd2e1510d85852a37c3e4a84ac54391b8cea985c729f34be24f49f2ec7e53202de7bcc9b8f309c6fa61de445394e3c55641

data/bin/gitlab-housekeeper

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+require 'gitlab/housekeeper'
+
+options = {}
+
+OptionParser.new do |opts|
+  opts.banner = 'Creates merge requests that can be inferred from the current state of the codebase'
+
+  opts.on('-m=M', '--max-mrs=M', Integer, 'Limit of MRs to create. Defaults to 1.') do |m|
+    options[:max_mrs] = m
+  end
+
+  opts.on('-d', '--dry-run', 'Dry-run only. Print the MR titles, descriptions and diffs') do
+    options[:dry_run] = true
+  end
+
+  opts.on('-k OverdueFinalizeBackgroundMigration,AnotherKeep', '--keeps OverdueFinalizeBackgroundMigration,AnotherKeep', Array, 'Require keeps specified') do |k|
+    options[:keeps] = k
+  end
+
+  opts.on('--filter-identifiers some-identifier-regex,another-regex', Array, 'Skip any changes where none of the identifiers match these regexes. The identifiers is an array, so at least one element must match at least one regex.') do |filters|
+    options[:filter_identifiers] = filters.map { |f| Regexp.new(f) }
+  end
+
+  opts.on('-h', '--help', 'Prints this help') do
+    abort opts.to_s
+  end
+end.parse!
+
+Gitlab::Housekeeper::Runner.new(**options).run

data/lib/gitlab/housekeeper/change.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Housekeeper
+    class Change
+      attr_accessor :identifiers,
+        :title,
+        :description,
+        :changed_files,
+        :labels,
+        :reviewers
+
+      def initialize
+        @labels = []
+        @reviewers = []
+      end
+
+      def mr_description
+        <<~MARKDOWN
+        #{description}
+
+        This change was generated by
+        [gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper)
+        MARKDOWN
+      end
+
+      def commit_message
+        <<~MARKDOWN
+        #{title}
+
+        #{mr_description}
+
+        Changelog: other
+        MARKDOWN
+      end
+
+      def matches_filters?(filters)
+        filters.all? do |filter|
+          identifiers.any? do |identifier|
+            identifier.match?(filter)
+          end
+        end
+      end
+
+      def valid?
+        @identifiers && @title && @description && @changed_files
+      end
+    end
+  end
+end

data/lib/gitlab/housekeeper/git.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'gitlab/housekeeper/shell'
+
+module Gitlab
+  module Housekeeper
+    class Git
+      def initialize(logger:, branch_from: 'master')
+        @logger = logger
+        @branch_from = branch_from
+      end
+
+      def commit_in_branch(change)
+        branch_name = branch_name(change.identifiers)
+
+        create_commit(branch_name, change)
+
+        branch_name
+      end
+
+      def with_branch_from_branch
+        stashed = false
+        current_branch = Shell.execute('git', 'branch', '--show-current').chomp
+
+        result = Shell.execute('git', 'stash')
+        stashed = !result.include?('No local changes to save')
+
+        Shell.execute("git", "checkout", @branch_from)
+
+        yield
+      ensure
+        # The `current_branch` won't be set in CI due to how the repo is cloned. Therefore we should only checkout
+        # `current_branch` if we actually have one.
+        Shell.execute("git", "checkout", current_branch) if current_branch.present?
+        Shell.execute('git', 'stash', 'pop') if stashed
+      end
+
+      def create_commit(branch_name, change)
+        current_branch = Shell.execute('git', 'branch', '--show-current').chomp
+
+        begin
+          Shell.execute("git", "branch", '-D', branch_name)
+        rescue Shell::Error # Might not exist yet
+        end
+
+        Shell.execute("git", "checkout", "-b", branch_name)
+        Shell.execute("git", "add", *change.changed_files)
+        Shell.execute("git", "commit", "-m", change.commit_message)
+      ensure
+        Shell.execute("git", "checkout", current_branch)
+      end
+
+      def branch_name(identifiers)
+        # Hyphen-case each identifier then join together with hyphens.
+        branch_name = identifiers
+          .map { |i| i.gsub(/[[:upper:]]/) { |w| "-#{w.downcase}" } }
+          .join('-')
+          .delete_prefix("-")
+
+        # Truncate if it's too long and add a digest
+        if branch_name.length > 240
+          branch_name = branch_name[0...200] + OpenSSL::Digest::SHA256.hexdigest(branch_name)[0...15]
+        end
+
+        branch_name
+      end
+    end
+  end
+end

data/lib/gitlab/housekeeper/gitlab_client.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require 'httparty'
+require 'json'
+
+module Gitlab
+  module Housekeeper
+    class GitlabClient
+      Error = Class.new(StandardError)
+
+      def initialize
+        @token = ENV.fetch("HOUSEKEEPER_GITLAB_API_TOKEN")
+        @base_uri = 'https://gitlab.com/api/v4'
+      end
+
+      # This looks at the system notes of the merge request to detect if it has been updated by anyone other than the
+      # current housekeeper user. If it has then it assumes that they did this for a reason and we can skip updating
+      # this detail of the merge request. Otherwise we assume we should generate it again using the latest output.
+      def non_housekeeper_changes(
+        source_project_id:,
+        source_branch:,
+        target_branch:,
+        target_project_id:
+      )
+
+        iid = get_existing_merge_request(
+          source_project_id: source_project_id,
+          source_branch: source_branch,
+          target_branch: target_branch,
+          target_project_id: target_project_id
+        )
+
+        return [] if iid.nil?
+
+        merge_request_notes = get_merge_request_notes(target_project_id: target_project_id, iid: iid)
+
+        changes = Set.new
+
+        merge_request_notes.each do |note|
+          next false unless note["system"]
+          next false if note["author"]["id"] == current_user_id
+
+          changes << :title if note['body'].start_with?("changed title from")
+          changes << :description if note['body'] == "changed the description"
+          changes << :code if note['body'].match?(/added \d+ commit/)
+
+          changes << :reviewers if note['body'].include?('requested review from')
+          changes << :reviewers if note['body'].include?('removed review request for')
+        end
+
+        resource_label_events = get_merge_request_resource_label_events(target_project_id: target_project_id, iid: iid)
+
+        resource_label_events.each do |event|
+          next if event["user"]["id"] == current_user_id
+
+          # Labels are routinely added by both humans and bots, so addition events aren't cause for concern.
+          # However, if labels have been removed it may mean housekeeper added an incorrect label, and we shouldn't
+          # re-add them.
+          #
+          # TODO: Inspect the actual labels housekeeper wants to add, and add if they haven't previously been removed.
+          changes << :labels if event["action"] == "remove"
+        end
+
+        changes.to_a
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def create_or_update_merge_request(
+        change:,
+        source_project_id:,
+        source_branch:,
+        target_branch:,
+        target_project_id:,
+        update_title:,
+        update_description:,
+        update_labels:,
+        update_reviewers:
+      )
+        existing_iid = get_existing_merge_request(
+          source_project_id: source_project_id,
+          source_branch: source_branch,
+          target_branch: target_branch,
+          target_project_id: target_project_id
+        )
+
+        if existing_iid
+          update_existing_merge_request(
+            change: change,
+            existing_iid: existing_iid,
+            target_project_id: target_project_id,
+            update_title: update_title,
+            update_description: update_description,
+            update_labels: update_labels,
+            update_reviewers: update_reviewers
+          )
+        else
+          create_merge_request(
+            change: change,
+            source_project_id: source_project_id,
+            source_branch: source_branch,
+            target_branch: target_branch,
+            target_project_id: target_project_id
+          )
+        end
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      private
+
+      def get_merge_request_notes(target_project_id:, iid:)
+        request(:get, "/projects/#{target_project_id}/merge_requests/#{iid}/notes", query: { per_page: 100 })
+      end
+
+      def get_merge_request_resource_label_events(target_project_id:, iid:)
+        request(:get, "/projects/#{target_project_id}/merge_requests/#{iid}/resource_label_events",
+          query: { per_page: 100 })
+      end
+
+      def current_user_id
+        @current_user_id ||= request(:get, "/user")['id']
+      end
+
+      def get_existing_merge_request(source_project_id:, source_branch:, target_branch:, target_project_id:)
+        data = request(:get, "/projects/#{target_project_id}/merge_requests", query: {
+          state: :opened,
+          source_branch: source_branch,
+          target_branch: target_branch,
+          source_project_id: source_project_id
+        })
+
+        return nil if data.empty?
+
+        iids = data.pluck('iid')
+
+        raise Error, "More than one matching MR exists: iids: #{iids.join(',')}" unless data.size == 1
+
+        iids.first
+      end
+
+      def create_merge_request(
+        change:,
+        source_project_id:,
+        source_branch:,
+        target_branch:,
+        target_project_id:
+      )
+        request(:post, "/projects/#{source_project_id}/merge_requests", body: {
+          title: change.title,
+          description: change.mr_description,
+          labels: Array(change.labels).join(','),
+          source_branch: source_branch,
+          target_branch: target_branch,
+          target_project_id: target_project_id,
+          remove_source_branch: true,
+          reviewer_ids: usernames_to_ids(change.reviewers)
+        })
+      end
+
+      def update_existing_merge_request(
+        change:,
+        existing_iid:,
+        target_project_id:,
+        update_title:,
+        update_description:,
+        update_labels:,
+        update_reviewers:
+      )
+        body = {}
+
+        body[:title] = change.title if update_title
+        body[:description] = change.mr_description if update_description
+        body[:add_labels] = Array(change.labels).join(',') if update_labels
+        body[:reviewer_ids] = usernames_to_ids(change.reviewers) if update_reviewers
+
+        return if body.empty?
+
+        request(:put, "/projects/#{target_project_id}/merge_requests/#{existing_iid}", body: body)
+      end
+
+      def usernames_to_ids(usernames)
+        usernames.map do |username|
+          data = request(:get, "/users", query: { username: username })
+          data[0]['id']
+        end
+      end
+
+      def request(method, path, query: {}, body: {})
+        response = HTTParty.public_send(method, "#{@base_uri}#{path}", query: query, body: body.to_json, headers: { # rubocop:disable GitlabSecurity/PublicSend
+          'Private-Token' => @token,
+          'Content-Type' => 'application/json'
+        })
+
+        unless (200..299).cover?(response.code)
+          raise Error,
+            "Failed with response code: #{response.code} and body:\n#{response.body}"
+        end
+
+        JSON.parse(response.body)
+      end
+    end
+  end
+end

data/lib/gitlab/housekeeper/keep.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Housekeeper
+    # A Keep is analogous to a Cop in RuboCop. The Keep is responsible for:
+    # - Detecting a specific code change that should be made (eg. removing an old feature flag)
+    # - Making the code change (eg. delete the feature flag YML file)
+    # - Yielding a Change object that describes this change, For example:
+    #   ```
+    #   yield Gitlab::Housekeeper::Change.new(
+    #     identifiers: ['remove-old-ff', 'old_ff_name'], # Unique and stable identifier for branch name
+    #     title: "Remove old feature flag old_ff_name as it has been enabled since %15.0",
+    #     description: "This feature flag was enabled in 15.0 and is not needed anymore ...",
+    #     changed_files: ["config/feature_flags/ops/old_ff_name.yml", "app/models/user.rb"]
+    #   )
+    #   ```
+    class Keep
+      # The each_change method must update local working copy files and yield a Change object which describes the
+      # specific changed files and other data that will be used to generate a merge request. This is the core
+      # implementation details for a specific housekeeper keep. This does not need to commit the changes or create the
+      # merge request as that is handled by the gitlab-housekeeper gem.
+      #
+      # @yieldparam [Gitlab::Housekeeper::Change]
+      def each_change
+        raise NotImplementedError, "A Keep must implement each_change method"
+      end
+    end
+  end
+end

data/lib/gitlab/housekeeper/runner.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/string'
+require 'gitlab/housekeeper/keep'
+require 'gitlab/housekeeper/gitlab_client'
+require 'gitlab/housekeeper/git'
+require 'gitlab/housekeeper/change'
+require 'awesome_print'
+require 'digest'
+
+module Gitlab
+  module Housekeeper
+    class Runner
+      def initialize(max_mrs: 1, dry_run: false, keeps: nil, filter_identifiers: [])
+        @max_mrs = max_mrs
+        @dry_run = dry_run
+        @logger = Logger.new($stdout)
+        require_keeps
+
+        @keeps = if keeps
+                   keeps.map { |k| k.is_a?(String) ? k.constantize : k }
+                 else
+                   all_keeps
+                 end
+
+        @filter_identifiers = filter_identifiers
+      end
+
+      def run
+        created = 0
+
+        git.with_branch_from_branch do
+          @keeps.each do |keep_class|
+            keep = keep_class.new
+            keep.each_change do |change|
+              unless change.valid?
+                puts "Ignoring invalid change from: #{keep_class}"
+                next
+              end
+
+              branch_name = git.commit_in_branch(change)
+              add_standard_change_data(change)
+
+              # Must be done after we commit so that we don't keep around changed files. We could checkout those files
+              # but then it might be riskier in local development in case we lose unrelated changes.
+              unless change.matches_filters?(@filter_identifiers)
+                puts "Skipping change: #{change.identifiers} due to not matching filter"
+                next
+              end
+
+              if @dry_run
+                dry_run(change, branch_name)
+              else
+                create(change, branch_name)
+              end
+
+              created += 1
+              break if created >= @max_mrs
+            end
+            break if created >= @max_mrs
+          end
+        end
+
+        puts "Housekeeper created #{created} MRs"
+      end
+
+      def add_standard_change_data(change)
+        change.labels ||= []
+        change.labels << 'automation:gitlab-housekeeper-authored'
+      end
+
+      def git
+        @git ||= ::Gitlab::Housekeeper::Git.new(logger: @logger)
+      end
+
+      def require_keeps
+        Dir.glob("keeps/*.rb").each do |f|
+          require(Pathname(f).expand_path.to_s)
+        end
+      end
+
+      def dry_run(change, branch_name)
+        puts "=> #{change.identifiers.join(': ')}".purple
+
+        puts '=> Title:'.purple
+        puts change.title.purple
+        puts
+
+        puts '=> Description:'
+        puts change.description
+        puts
+
+        if change.labels.present?
+          puts '=> Attributes:'
+          puts "Labels: #{change.labels.join(', ')}"
+          puts
+        end
+
+        puts '=> Diff:'
+        puts Shell.execute('git', '--no-pager', 'diff', '--color=always', 'master', branch_name, '--',
+          *change.changed_files)
+        puts
+      end
+
+      def create(change, branch_name)
+        dry_run(change, branch_name)
+
+        non_housekeeper_changes = gitlab_client.non_housekeeper_changes(
+          source_project_id: housekeeper_fork_project_id,
+          source_branch: branch_name,
+          target_branch: 'master',
+          target_project_id: housekeeper_target_project_id
+        )
+
+        unless non_housekeeper_changes.include?(:code)
+          Shell.execute('git', 'push', '-f', 'housekeeper', "#{branch_name}:#{branch_name}")
+        end
+
+        mr = gitlab_client.create_or_update_merge_request(
+          change: change,
+          source_project_id: housekeeper_fork_project_id,
+          source_branch: branch_name,
+          target_branch: 'master',
+          target_project_id: housekeeper_target_project_id,
+          update_title: !non_housekeeper_changes.include?(:title),
+          update_description: !non_housekeeper_changes.include?(:description),
+          update_labels: !non_housekeeper_changes.include?(:labels),
+          update_reviewers: !non_housekeeper_changes.include?(:reviewers)
+        )
+
+        puts "Merge request URL: #{mr['web_url'].yellowish}"
+      end
+
+      def housekeeper_fork_project_id
+        ENV.fetch('HOUSEKEEPER_FORK_PROJECT_ID')
+      end
+
+      def housekeeper_target_project_id
+        ENV.fetch('HOUSEKEEPER_TARGET_PROJECT_ID')
+      end
+
+      def gitlab_client
+        @gitlab_client ||= GitlabClient.new
+      end
+
+      def all_keeps
+        @all_keeps ||= ObjectSpace.each_object(Class).select { |klass| klass < Keep }
+      end
+    end
+  end
+end

data/lib/gitlab/housekeeper/shell.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Gitlab
+  module Housekeeper
+    class Shell
+      Error = Class.new(StandardError)
+
+      def self.execute(*cmd)
+        stdin, stdout, stderr, wait_thr = Open3.popen3(*cmd)
+
+        stdin.close
+        out = stdout.read
+        stdout.close
+        err = stderr.read
+        stderr.close
+
+        exit_status = wait_thr.value
+
+        raise Error, "Failed with #{exit_status}\n#{out}\n#{err}\n" unless exit_status.success?
+
+        out + err
+      end
+    end
+  end
+end

data/lib/gitlab/housekeeper/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Housekeeper
+    VERSION = "0.1.0"
+  end
+end

data/lib/gitlab/housekeeper.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "gitlab/housekeeper/version"
+require "gitlab/housekeeper/runner"
+
+module Gitlab
+  module Housekeeper
+    Error = Class.new(StandardError)
+    # Your code goes here...
+  end
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: gitlab-housekeeper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- group::tenant-scale
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-02-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gitlab-styles
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Housekeeping following https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134487
+email:
+- engineering@gitlab.com
+executables:
+- gitlab-housekeeper
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/gitlab-housekeeper
+- lib/gitlab/housekeeper.rb
+- lib/gitlab/housekeeper/change.rb
+- lib/gitlab/housekeeper/git.rb
+- lib/gitlab/housekeeper/gitlab_client.rb
+- lib/gitlab/housekeeper/keep.rb
+- lib/gitlab/housekeeper/runner.rb
+- lib/gitlab/housekeeper/shell.rb
+- lib/gitlab/housekeeper/version.rb
+homepage: https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.6
+signing_key:
+specification_version: 4
+summary: Gem summary
+test_files: []