gitlab-housekeeper 0.1.0.codechallenge5e4fde.pre
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/gitlab-housekeeper +32 -0
- data/lib/gitlab/housekeeper/change.rb +61 -0
- data/lib/gitlab/housekeeper/git.rb +89 -0
- data/lib/gitlab/housekeeper/gitlab_client.rb +206 -0
- data/lib/gitlab/housekeeper/keep.rb +29 -0
- data/lib/gitlab/housekeeper/keeps/rubocop_fixer.rb +91 -0
- data/lib/gitlab/housekeeper/keeps/upgrade_dependency.rb +37 -0
- data/lib/gitlab/housekeeper/runner.rb +185 -0
- data/lib/gitlab/housekeeper/shell.rb +27 -0
- data/lib/gitlab/housekeeper/substitutor.rb +22 -0
- data/lib/gitlab/housekeeper/version.rb +7 -0
- data/lib/gitlab/housekeeper.rb +11 -0
- metadata +169 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 7dac9bbc0d3bf745638e3c3497cec5195d05522400a14418f8e4ff91e51a0bc0
|
4
|
+
data.tar.gz: 0340674107e33efb3024db10af1ddfcdbdebc6b36e4d337486ec5497573bfc2d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8c78a8437a5d77429cd6caf4ee2890dc0e3616f44ceb234d4245959296f6d6667f389327daa5a6c5f719d5501929312ee0c77c5193d63e4a29687418d426cba7
|
7
|
+
data.tar.gz: adbf96a9a14d3bd57af90439defee3136a3233bd22a1f074218781cf22caf750d1fbe7f2061432a4b37c9c470c03cf0a0c975fbff074f13177fde62d84df7dbc
|
@@ -0,0 +1,32 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "optparse"
|
4
|
+
require 'gitlab/housekeeper'
|
5
|
+
|
6
|
+
options = {}
|
7
|
+
|
8
|
+
OptionParser.new do |opts|
|
9
|
+
opts.banner = 'Creates merge requests that can be inferred from the current state of the codebase'
|
10
|
+
|
11
|
+
opts.on('-m=M', '--max-mrs=M', Integer, 'Limit of MRs to create. Defaults to 1.') do |m|
|
12
|
+
options[:max_mrs] = m
|
13
|
+
end
|
14
|
+
|
15
|
+
opts.on('-d', '--dry-run', 'Dry-run only. Print the MR titles, descriptions and diffs') do
|
16
|
+
options[:dry_run] = true
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.on('-k OverdueFinalizeBackgroundMigration,AnotherKeep', '--keeps OverdueFinalizeBackgroundMigration,AnotherKeep', Array, 'Require keeps specified') do |k|
|
20
|
+
options[:keeps] = k
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on('--filter-identifiers some-identifier-regex,another-regex', Array, 'Skip any changes where none of the identifiers match these regexes. The identifiers is an array, so at least one element must match at least one regex.') do |filters|
|
24
|
+
options[:filter_identifiers] = filters.map { |f| Regexp.new(f) }
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on('-h', '--help', 'Prints this help') do
|
28
|
+
abort opts.to_s
|
29
|
+
end
|
30
|
+
end.parse!
|
31
|
+
|
32
|
+
Gitlab::Housekeeper::Runner.new(**options).run
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module Housekeeper
|
5
|
+
class Change
|
6
|
+
attr_accessor :identifiers,
|
7
|
+
:title,
|
8
|
+
:description,
|
9
|
+
:changed_files,
|
10
|
+
:labels,
|
11
|
+
:keep_class,
|
12
|
+
:changelog_type,
|
13
|
+
:mr_web_url
|
14
|
+
attr_reader :reviewers
|
15
|
+
|
16
|
+
def initialize
|
17
|
+
@labels = []
|
18
|
+
@reviewers = []
|
19
|
+
end
|
20
|
+
|
21
|
+
def reviewers=(reviewers)
|
22
|
+
@reviewers = Array(reviewers)
|
23
|
+
end
|
24
|
+
|
25
|
+
def mr_description
|
26
|
+
<<~MARKDOWN
|
27
|
+
#{description}
|
28
|
+
|
29
|
+
This change was generated by
|
30
|
+
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper)
|
31
|
+
using the #{keep_class} keep.
|
32
|
+
|
33
|
+
To provide feedback on your experience with `gitlab-housekeeper` please comment in
|
34
|
+
<https://gitlab.com/gitlab-org/gitlab/-/issues/442003>.
|
35
|
+
MARKDOWN
|
36
|
+
end
|
37
|
+
|
38
|
+
def commit_message
|
39
|
+
<<~MARKDOWN
|
40
|
+
#{title}
|
41
|
+
|
42
|
+
#{mr_description}
|
43
|
+
|
44
|
+
Changelog: #{changelog_type || 'other'}
|
45
|
+
MARKDOWN
|
46
|
+
end
|
47
|
+
|
48
|
+
def matches_filters?(filters)
|
49
|
+
filters.all? do |filter|
|
50
|
+
identifiers.any? do |identifier|
|
51
|
+
identifier.match?(filter)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def valid?
|
57
|
+
@identifiers && @title && @description && @changed_files
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'logger'
|
4
|
+
require 'gitlab/housekeeper/shell'
|
5
|
+
|
6
|
+
module Gitlab
|
7
|
+
module Housekeeper
|
8
|
+
class Git
|
9
|
+
def initialize(logger:, branch_from: 'master')
|
10
|
+
@logger = logger
|
11
|
+
@branch_from = branch_from
|
12
|
+
end
|
13
|
+
|
14
|
+
def with_clean_state
|
15
|
+
result = Shell.execute('git', 'stash')
|
16
|
+
stashed = !result.include?('No local changes to save')
|
17
|
+
|
18
|
+
with_return_to_current_branch(stashed: stashed) do
|
19
|
+
checkout_branch(@branch_from)
|
20
|
+
|
21
|
+
yield
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def create_branch(change)
|
26
|
+
branch_name = branch_name(change.identifiers)
|
27
|
+
|
28
|
+
Shell.execute("git", "branch", "-f", branch_name)
|
29
|
+
|
30
|
+
branch_name
|
31
|
+
end
|
32
|
+
|
33
|
+
def in_branch(branch_name)
|
34
|
+
with_return_to_current_branch do
|
35
|
+
checkout_branch(branch_name)
|
36
|
+
|
37
|
+
yield
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def create_commit(change)
|
42
|
+
Shell.execute("git", "add", *change.changed_files)
|
43
|
+
Shell.execute("git", "commit", "-m", change.commit_message)
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def create_and_checkout_branch(branch_name)
|
49
|
+
begin
|
50
|
+
Shell.execute("git", "branch", '-D', branch_name)
|
51
|
+
rescue Shell::Error # Might not exist yet
|
52
|
+
end
|
53
|
+
|
54
|
+
Shell.execute("git", "checkout", "-b", branch_name)
|
55
|
+
end
|
56
|
+
|
57
|
+
def checkout_branch(branch_name)
|
58
|
+
Shell.execute("git", "checkout", branch_name)
|
59
|
+
end
|
60
|
+
|
61
|
+
def branch_name(identifiers)
|
62
|
+
# Hyphen-case each identifier then join together with hyphens.
|
63
|
+
branch_name = identifiers
|
64
|
+
.map { |i| i.gsub(/[^\w]+/, '-') }
|
65
|
+
.map { |i| i.gsub(/[[:upper:]]/) { |w| "-#{w.downcase}" } }
|
66
|
+
.join('-')
|
67
|
+
.delete_prefix("-")
|
68
|
+
|
69
|
+
# Truncate if it's too long and add a digest
|
70
|
+
if branch_name.length > 240
|
71
|
+
branch_name = branch_name[0...200] + OpenSSL::Digest::SHA256.hexdigest(branch_name)[0...15]
|
72
|
+
end
|
73
|
+
|
74
|
+
branch_name
|
75
|
+
end
|
76
|
+
|
77
|
+
def with_return_to_current_branch(stashed: false)
|
78
|
+
current_branch = Shell.execute('git', 'branch', '--show-current').chomp
|
79
|
+
|
80
|
+
yield
|
81
|
+
ensure
|
82
|
+
# The `current_branch` won't be set in CI due to how the repo is cloned. Therefore we should only checkout
|
83
|
+
# `current_branch` if we actually have one.
|
84
|
+
checkout_branch(current_branch) if current_branch.present?
|
85
|
+
Shell.execute('git', 'stash', 'pop') if stashed
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,206 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'httparty'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
module Gitlab
|
7
|
+
module Housekeeper
|
8
|
+
class GitlabClient
|
9
|
+
Error = Class.new(StandardError)
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@token = ENV.fetch("HOUSEKEEPER_GITLAB_API_TOKEN")
|
13
|
+
@base_uri = 'https://gitlab.com/api/v4'
|
14
|
+
end
|
15
|
+
|
16
|
+
# This looks at the system notes of the merge request to detect if it has been updated by anyone other than the
|
17
|
+
# current housekeeper user. If it has then it assumes that they did this for a reason and we can skip updating
|
18
|
+
# this detail of the merge request. Otherwise we assume we should generate it again using the latest output.
|
19
|
+
def non_housekeeper_changes(
|
20
|
+
source_project_id:,
|
21
|
+
source_branch:,
|
22
|
+
target_branch:,
|
23
|
+
target_project_id:
|
24
|
+
)
|
25
|
+
|
26
|
+
existing_merge_request = get_existing_merge_request(
|
27
|
+
source_project_id: source_project_id,
|
28
|
+
source_branch: source_branch,
|
29
|
+
target_branch: target_branch,
|
30
|
+
target_project_id: target_project_id
|
31
|
+
)
|
32
|
+
|
33
|
+
return [] if existing_merge_request.nil?
|
34
|
+
|
35
|
+
merge_request_notes = get_merge_request_notes(
|
36
|
+
target_project_id: target_project_id,
|
37
|
+
iid: existing_merge_request['iid']
|
38
|
+
)
|
39
|
+
|
40
|
+
changes = Set.new
|
41
|
+
|
42
|
+
merge_request_notes.each do |note|
|
43
|
+
next false unless note["system"]
|
44
|
+
next false if note["author"]["id"] == current_user_id
|
45
|
+
|
46
|
+
changes << :title if note['body'].start_with?("changed title from")
|
47
|
+
changes << :description if note['body'] == "changed the description"
|
48
|
+
changes << :code if note['body'].match?(/added \d+ commit/)
|
49
|
+
|
50
|
+
changes << :reviewers if note['body'].include?('requested review from')
|
51
|
+
changes << :reviewers if note['body'].include?('removed review request for')
|
52
|
+
end
|
53
|
+
|
54
|
+
resource_label_events = get_merge_request_resource_label_events(
|
55
|
+
target_project_id: target_project_id,
|
56
|
+
iid: existing_merge_request['iid']
|
57
|
+
)
|
58
|
+
|
59
|
+
resource_label_events.each do |event|
|
60
|
+
next if event["user"]["id"] == current_user_id
|
61
|
+
|
62
|
+
# Labels are routinely added by both humans and bots, so addition events aren't cause for concern.
|
63
|
+
# However, if labels have been removed it may mean housekeeper added an incorrect label, and we shouldn't
|
64
|
+
# re-add them.
|
65
|
+
#
|
66
|
+
# TODO: Inspect the actual labels housekeeper wants to add, and add if they haven't previously been removed.
|
67
|
+
changes << :labels if event["action"] == "remove"
|
68
|
+
end
|
69
|
+
|
70
|
+
changes.to_a
|
71
|
+
end
|
72
|
+
|
73
|
+
# rubocop:disable Metrics/ParameterLists
|
74
|
+
def create_or_update_merge_request(
|
75
|
+
change:,
|
76
|
+
source_project_id:,
|
77
|
+
source_branch:,
|
78
|
+
target_branch:,
|
79
|
+
target_project_id:,
|
80
|
+
update_title:,
|
81
|
+
update_description:,
|
82
|
+
update_labels:,
|
83
|
+
update_reviewers:
|
84
|
+
)
|
85
|
+
existing_merge_request = get_existing_merge_request(
|
86
|
+
source_project_id: source_project_id,
|
87
|
+
source_branch: source_branch,
|
88
|
+
target_branch: target_branch,
|
89
|
+
target_project_id: target_project_id
|
90
|
+
)
|
91
|
+
|
92
|
+
if existing_merge_request
|
93
|
+
update_existing_merge_request(
|
94
|
+
change: change,
|
95
|
+
existing_iid: existing_merge_request['iid'],
|
96
|
+
target_project_id: target_project_id,
|
97
|
+
update_title: update_title,
|
98
|
+
update_description: update_description,
|
99
|
+
update_labels: update_labels,
|
100
|
+
update_reviewers: update_reviewers
|
101
|
+
)
|
102
|
+
else
|
103
|
+
create_merge_request(
|
104
|
+
change: change,
|
105
|
+
source_project_id: source_project_id,
|
106
|
+
source_branch: source_branch,
|
107
|
+
target_branch: target_branch,
|
108
|
+
target_project_id: target_project_id
|
109
|
+
)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
# rubocop:enable Metrics/ParameterLists
|
113
|
+
|
114
|
+
def get_existing_merge_request(source_project_id:, source_branch:, target_branch:, target_project_id:)
|
115
|
+
data = request(:get, "/projects/#{target_project_id}/merge_requests", query: {
|
116
|
+
state: :opened,
|
117
|
+
source_branch: source_branch,
|
118
|
+
target_branch: target_branch,
|
119
|
+
source_project_id: source_project_id
|
120
|
+
})
|
121
|
+
|
122
|
+
return nil if data.empty?
|
123
|
+
|
124
|
+
raise Error, "More than one matching MR exists: iids: #{data.pluck('iid').join(',')}" unless data.size == 1
|
125
|
+
|
126
|
+
data.first
|
127
|
+
end
|
128
|
+
|
129
|
+
private
|
130
|
+
|
131
|
+
def get_merge_request_notes(target_project_id:, iid:)
|
132
|
+
request(:get, "/projects/#{target_project_id}/merge_requests/#{iid}/notes", query: { per_page: 100 })
|
133
|
+
end
|
134
|
+
|
135
|
+
def get_merge_request_resource_label_events(target_project_id:, iid:)
|
136
|
+
request(:get, "/projects/#{target_project_id}/merge_requests/#{iid}/resource_label_events",
|
137
|
+
query: { per_page: 100 })
|
138
|
+
end
|
139
|
+
|
140
|
+
def current_user_id
|
141
|
+
@current_user_id ||= request(:get, "/user")['id']
|
142
|
+
end
|
143
|
+
|
144
|
+
def create_merge_request(
|
145
|
+
change:,
|
146
|
+
source_project_id:,
|
147
|
+
source_branch:,
|
148
|
+
target_branch:,
|
149
|
+
target_project_id:
|
150
|
+
)
|
151
|
+
request(:post, "/projects/#{source_project_id}/merge_requests", body: {
|
152
|
+
title: change.title,
|
153
|
+
description: change.mr_description,
|
154
|
+
labels: Array(change.labels).join(','),
|
155
|
+
source_branch: source_branch,
|
156
|
+
target_branch: target_branch,
|
157
|
+
target_project_id: target_project_id,
|
158
|
+
remove_source_branch: true,
|
159
|
+
reviewer_ids: usernames_to_ids(change.reviewers)
|
160
|
+
})
|
161
|
+
end
|
162
|
+
|
163
|
+
def update_existing_merge_request(
|
164
|
+
change:,
|
165
|
+
existing_iid:,
|
166
|
+
target_project_id:,
|
167
|
+
update_title:,
|
168
|
+
update_description:,
|
169
|
+
update_labels:,
|
170
|
+
update_reviewers:
|
171
|
+
)
|
172
|
+
body = {}
|
173
|
+
|
174
|
+
body[:title] = change.title if update_title
|
175
|
+
body[:description] = change.mr_description if update_description
|
176
|
+
body[:add_labels] = Array(change.labels).join(',') if update_labels
|
177
|
+
body[:reviewer_ids] = usernames_to_ids(change.reviewers) if update_reviewers
|
178
|
+
|
179
|
+
return if body.empty?
|
180
|
+
|
181
|
+
request(:put, "/projects/#{target_project_id}/merge_requests/#{existing_iid}", body: body)
|
182
|
+
end
|
183
|
+
|
184
|
+
def usernames_to_ids(usernames)
|
185
|
+
Array(usernames).map do |username|
|
186
|
+
data = request(:get, "/users", query: { username: username })
|
187
|
+
data[0]['id']
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
def request(method, path, query: {}, body: {})
|
192
|
+
response = HTTParty.public_send(method, "#{@base_uri}#{path}", query: query, body: body.to_json, headers: { # rubocop:disable GitlabSecurity/PublicSend
|
193
|
+
'Private-Token' => @token,
|
194
|
+
'Content-Type' => 'application/json'
|
195
|
+
})
|
196
|
+
|
197
|
+
unless (200..299).cover?(response.code)
|
198
|
+
raise Error,
|
199
|
+
"Failed with response code: #{response.code} and body:\n#{response.body}"
|
200
|
+
end
|
201
|
+
|
202
|
+
JSON.parse(response.body)
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module Housekeeper
|
5
|
+
# A Keep is analogous to a Cop in RuboCop. The Keep is responsible for:
|
6
|
+
# - Detecting a specific code change that should be made (eg. removing an old feature flag)
|
7
|
+
# - Making the code change (eg. delete the feature flag YML file)
|
8
|
+
# - Yielding a Change object that describes this change, For example:
|
9
|
+
# ```
|
10
|
+
# yield Gitlab::Housekeeper::Change.new(
|
11
|
+
# identifiers: ['remove-old-ff', 'old_ff_name'], # Unique and stable identifier for branch name
|
12
|
+
# title: "Remove old feature flag old_ff_name as it has been enabled since %15.0",
|
13
|
+
# description: "This feature flag was enabled in 15.0 and is not needed anymore ...",
|
14
|
+
# changed_files: ["config/feature_flags/ops/old_ff_name.yml", "app/models/user.rb"]
|
15
|
+
# )
|
16
|
+
# ```
|
17
|
+
class Keep
|
18
|
+
# The each_change method must update local working copy files and yield a Change object which describes the
|
19
|
+
# specific changed files and other data that will be used to generate a merge request. This is the core
|
20
|
+
# implementation details for a specific housekeeper keep. This does not need to commit the changes or create the
|
21
|
+
# merge request as that is handled by the gitlab-housekeeper gem.
|
22
|
+
#
|
23
|
+
# @yieldparam [Gitlab::Housekeeper::Change]
|
24
|
+
def each_change
|
25
|
+
raise NotImplementedError, "A Keep must implement each_change method"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
module Housekeeper
|
7
|
+
module Keeps
|
8
|
+
class RubocopFixer < Keep
|
9
|
+
LIMIT_FIXES = 20
|
10
|
+
RUBOCOP_TODO_DIR_PATTERN = ".rubocop_todo/**/*.yml"
|
11
|
+
|
12
|
+
def initialize(todo_dir_pattern: RUBOCOP_TODO_DIR_PATTERN, limit_fixes: LIMIT_FIXES)
|
13
|
+
@todo_dir_pattern = todo_dir_pattern
|
14
|
+
@limit_fixes = limit_fixes
|
15
|
+
end
|
16
|
+
|
17
|
+
def each_change
|
18
|
+
each_allowed_rubocop_rule do |rule, rule_file_path, violating_files|
|
19
|
+
remove_allow_rule = true
|
20
|
+
|
21
|
+
if violating_files.count > @limit_fixes
|
22
|
+
violating_files = violating_files.first(@limit_fixes)
|
23
|
+
remove_allow_rule = false
|
24
|
+
end
|
25
|
+
|
26
|
+
change = Change.new
|
27
|
+
change.title = "Fix #{violating_files.count} rubocop violations for #{rule}"
|
28
|
+
change.identifiers = [self.class.name, rule, violating_files.last]
|
29
|
+
change.description = <<~MARKDOWN
|
30
|
+
Fixes the #{violating_files.count} violations for the rubocop rule `#{rule}`
|
31
|
+
that were previously excluded in `#{rule_file_path}`.
|
32
|
+
The exclusions have now been removed.
|
33
|
+
MARKDOWN
|
34
|
+
|
35
|
+
if remove_allow_rule
|
36
|
+
FileUtils.rm(rule_file_path)
|
37
|
+
else
|
38
|
+
remove_first_exclusions(rule, rule_file_path, violating_files.count)
|
39
|
+
end
|
40
|
+
|
41
|
+
begin
|
42
|
+
::Gitlab::Housekeeper::Shell.execute('rubocop', '--autocorrect', *violating_files)
|
43
|
+
rescue ::Gitlab::Housekeeper::Shell::Error
|
44
|
+
# Ignore when it cannot be automatically fixed. But we need to checkout any files we might have updated.
|
45
|
+
::Gitlab::Housekeeper::Shell.execute('git', 'checkout', rule_file_path, *violating_files)
|
46
|
+
next
|
47
|
+
end
|
48
|
+
|
49
|
+
change.changed_files = [rule_file_path, *violating_files]
|
50
|
+
|
51
|
+
yield(change)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def each_allowed_rubocop_rule
|
56
|
+
Dir.glob(@todo_dir_pattern).each do |file|
|
57
|
+
content = File.read(file)
|
58
|
+
next unless content.include?('Cop supports --autocorrect')
|
59
|
+
|
60
|
+
data = YAML.safe_load(content)
|
61
|
+
|
62
|
+
# Assume one per file since that's how it is in gitlab-org/gitlab
|
63
|
+
next unless data.keys.count == 1
|
64
|
+
|
65
|
+
rule = data.keys[0]
|
66
|
+
violating_files = data[rule]['Exclude']
|
67
|
+
next unless violating_files&.count&.positive?
|
68
|
+
|
69
|
+
yield rule, file, violating_files
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def remove_first_exclusions(_rule, file, remove_count)
|
74
|
+
content = File.read(file)
|
75
|
+
skipped = 0
|
76
|
+
|
77
|
+
output = content.each_line.filter do |line|
|
78
|
+
if skipped < remove_count && line.match?(/\s+-\s+/)
|
79
|
+
skipped += 1
|
80
|
+
false
|
81
|
+
else
|
82
|
+
true
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
File.write(file, output.join)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module Housekeeper
|
5
|
+
module Keeps
|
6
|
+
class UpgradeDependency < Keep
|
7
|
+
def initialize(
|
8
|
+
upgrade_command: ENV.fetch('HOUSEKEEPER_DEPENDENCY_UPGRADE_COMMAND'), # bundle update haml
|
9
|
+
changed_files: ENV.fetch('HOUSEKEEPER_DEPENDENCY_UPGRADE_CHANGED_FILES').split(',') # Gemfile.lock
|
10
|
+
)
|
11
|
+
@upgrade_command = upgrade_command
|
12
|
+
@changed_files = changed_files
|
13
|
+
end
|
14
|
+
|
15
|
+
def each_change
|
16
|
+
::Gitlab::Housekeeper::Shell.execute(@upgrade_command)
|
17
|
+
|
18
|
+
change = Change.new
|
19
|
+
change.title = "Run #{@upgrade_command}"
|
20
|
+
change.identifiers = [self.class.name, @upgrade_command]
|
21
|
+
change.description = <<~MARKDOWN
|
22
|
+
Automatically upgrade a dependency using:
|
23
|
+
|
24
|
+
```
|
25
|
+
#{@upgrade_command}
|
26
|
+
```
|
27
|
+
MARKDOWN
|
28
|
+
|
29
|
+
|
30
|
+
change.changed_files = @changed_files
|
31
|
+
|
32
|
+
yield(change)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,185 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support'
|
4
|
+
require 'active_support/core_ext/string'
|
5
|
+
require 'gitlab/housekeeper/keep'
|
6
|
+
require 'gitlab/housekeeper/keeps/rubocop_fixer'
|
7
|
+
require 'gitlab/housekeeper/keeps/upgrade_dependency'
|
8
|
+
require 'gitlab/housekeeper/gitlab_client'
|
9
|
+
require 'gitlab/housekeeper/git'
|
10
|
+
require 'gitlab/housekeeper/change'
|
11
|
+
require 'gitlab/housekeeper/substitutor'
|
12
|
+
require 'awesome_print'
|
13
|
+
require 'digest'
|
14
|
+
|
15
|
+
module Gitlab
|
16
|
+
module Housekeeper
|
17
|
+
class Runner
|
18
|
+
def initialize(max_mrs: 1, dry_run: false, keeps: nil, filter_identifiers: [])
|
19
|
+
@max_mrs = max_mrs
|
20
|
+
@dry_run = dry_run
|
21
|
+
@logger = Logger.new($stdout)
|
22
|
+
require_keeps
|
23
|
+
|
24
|
+
@keeps = if keeps
|
25
|
+
keeps.map { |k| k.is_a?(String) ? k.constantize : k }
|
26
|
+
else
|
27
|
+
all_keeps
|
28
|
+
end
|
29
|
+
|
30
|
+
@filter_identifiers = filter_identifiers
|
31
|
+
end
|
32
|
+
|
33
|
+
def run
|
34
|
+
created = 0
|
35
|
+
|
36
|
+
git.with_clean_state do
|
37
|
+
@keeps.each do |keep_class|
|
38
|
+
keep = keep_class.new
|
39
|
+
keep.each_change do |change|
|
40
|
+
unless change.valid?
|
41
|
+
puts "Ignoring invalid change from: #{keep_class}"
|
42
|
+
next
|
43
|
+
end
|
44
|
+
|
45
|
+
change.keep_class ||= keep_class
|
46
|
+
|
47
|
+
branch_name = git.create_branch(change)
|
48
|
+
add_standard_change_data(change)
|
49
|
+
|
50
|
+
unless change.matches_filters?(@filter_identifiers)
|
51
|
+
# At this point the keep has already run and edited files so we need to
|
52
|
+
# restore the local working copy. We could simply checkout all
|
53
|
+
# changed_files but this is very risky as it could mean losing work that
|
54
|
+
# cannot be recovered. Instead we commit all the work to the branch and
|
55
|
+
# move on without pushing the branch.
|
56
|
+
git.in_branch(branch_name) do
|
57
|
+
git.create_commit(change)
|
58
|
+
end
|
59
|
+
|
60
|
+
puts "Skipping change: #{change.identifiers} due to not matching filter."
|
61
|
+
puts "Modified files have been committed to branch #{branch_name.yellowish}, but will not be pushed."
|
62
|
+
puts
|
63
|
+
|
64
|
+
next
|
65
|
+
end
|
66
|
+
|
67
|
+
# If no merge request exists yet, create an empty one to allow keeps to use the web URL.
|
68
|
+
unless @dry_run
|
69
|
+
merge_reqeust = get_existing_merge_request(branch_name) || create(change, branch_name)
|
70
|
+
|
71
|
+
change.mr_web_url = merge_reqeust['web_url']
|
72
|
+
end
|
73
|
+
|
74
|
+
git.in_branch(branch_name) do
|
75
|
+
Gitlab::Housekeeper::Substitutor.perform(change)
|
76
|
+
|
77
|
+
git.create_commit(change)
|
78
|
+
end
|
79
|
+
|
80
|
+
print_change_details(change, branch_name)
|
81
|
+
|
82
|
+
create(change, branch_name) unless @dry_run
|
83
|
+
|
84
|
+
created += 1
|
85
|
+
break if created >= @max_mrs
|
86
|
+
end
|
87
|
+
break if created >= @max_mrs
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
puts "Housekeeper created #{created} MRs"
|
92
|
+
end
|
93
|
+
|
94
|
+
def add_standard_change_data(change)
|
95
|
+
change.labels ||= []
|
96
|
+
change.labels << 'automation:gitlab-housekeeper-authored'
|
97
|
+
end
|
98
|
+
|
99
|
+
def git
|
100
|
+
@git ||= ::Gitlab::Housekeeper::Git.new(logger: @logger)
|
101
|
+
end
|
102
|
+
|
103
|
+
def require_keeps
|
104
|
+
Dir.glob("keeps/*.rb").each do |f|
|
105
|
+
require(Pathname(f).expand_path.to_s)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def print_change_details(change, branch_name)
|
110
|
+
puts "Merge request URL: #{change.mr_web_url || '(known after create)'}, on branch #{branch_name}".yellowish
|
111
|
+
puts "=> #{change.identifiers.join(': ')}".purple
|
112
|
+
|
113
|
+
puts '=> Title:'.purple
|
114
|
+
puts change.title.purple
|
115
|
+
puts
|
116
|
+
|
117
|
+
puts '=> Description:'
|
118
|
+
puts change.description
|
119
|
+
puts
|
120
|
+
|
121
|
+
if change.labels.present? || change.reviewers.present?
|
122
|
+
puts '=> Attributes:'
|
123
|
+
puts "Labels: #{change.labels.join(', ')}"
|
124
|
+
puts "Reviewers: #{change.reviewers.join(', ')}"
|
125
|
+
puts
|
126
|
+
end
|
127
|
+
|
128
|
+
puts '=> Diff:'
|
129
|
+
puts Shell.execute('git', '--no-pager', 'diff', '--color=always', 'master', branch_name, '--',
|
130
|
+
*change.changed_files)
|
131
|
+
puts
|
132
|
+
end
|
133
|
+
|
134
|
+
def create(change, branch_name)
|
135
|
+
non_housekeeper_changes = gitlab_client.non_housekeeper_changes(
|
136
|
+
source_project_id: housekeeper_fork_project_id,
|
137
|
+
source_branch: branch_name,
|
138
|
+
target_branch: 'master',
|
139
|
+
target_project_id: housekeeper_target_project_id
|
140
|
+
)
|
141
|
+
|
142
|
+
unless non_housekeeper_changes.include?(:code)
|
143
|
+
Shell.execute('git', 'push', '-f', 'housekeeper', "#{branch_name}:#{branch_name}")
|
144
|
+
end
|
145
|
+
|
146
|
+
gitlab_client.create_or_update_merge_request(
|
147
|
+
change: change,
|
148
|
+
source_project_id: housekeeper_fork_project_id,
|
149
|
+
source_branch: branch_name,
|
150
|
+
target_branch: 'master',
|
151
|
+
target_project_id: housekeeper_target_project_id,
|
152
|
+
update_title: !non_housekeeper_changes.include?(:title),
|
153
|
+
update_description: !non_housekeeper_changes.include?(:description),
|
154
|
+
update_labels: !non_housekeeper_changes.include?(:labels),
|
155
|
+
update_reviewers: !non_housekeeper_changes.include?(:reviewers)
|
156
|
+
)
|
157
|
+
end
|
158
|
+
|
159
|
+
def get_existing_merge_request(branch_name)
|
160
|
+
gitlab_client.get_existing_merge_request(
|
161
|
+
source_project_id: housekeeper_fork_project_id,
|
162
|
+
source_branch: branch_name,
|
163
|
+
target_branch: 'master',
|
164
|
+
target_project_id: housekeeper_target_project_id
|
165
|
+
)
|
166
|
+
end
|
167
|
+
|
168
|
+
def housekeeper_fork_project_id
|
169
|
+
ENV.fetch('HOUSEKEEPER_FORK_PROJECT_ID')
|
170
|
+
end
|
171
|
+
|
172
|
+
def housekeeper_target_project_id
|
173
|
+
ENV.fetch('HOUSEKEEPER_TARGET_PROJECT_ID')
|
174
|
+
end
|
175
|
+
|
176
|
+
def gitlab_client
|
177
|
+
@gitlab_client ||= GitlabClient.new
|
178
|
+
end
|
179
|
+
|
180
|
+
def all_keeps
|
181
|
+
@all_keeps ||= ObjectSpace.each_object(Class).select { |klass| klass < Keep }
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'open3'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
module Housekeeper
|
7
|
+
class Shell
|
8
|
+
Error = Class.new(StandardError)
|
9
|
+
|
10
|
+
def self.execute(*cmd)
|
11
|
+
stdin, stdout, stderr, wait_thr = Open3.popen3(*cmd)
|
12
|
+
|
13
|
+
stdin.close
|
14
|
+
out = stdout.read
|
15
|
+
stdout.close
|
16
|
+
err = stderr.read
|
17
|
+
stderr.close
|
18
|
+
|
19
|
+
exit_status = wait_thr.value
|
20
|
+
|
21
|
+
raise Error, "Failed with #{exit_status}\n#{out}\n#{err}\n" unless exit_status.success?
|
22
|
+
|
23
|
+
out + err
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module Housekeeper
|
5
|
+
class Substitutor
|
6
|
+
MR_WEB_URL_PLACEHOLDER = '<HOUSEKEEPER_MR_WEB_URL>'
|
7
|
+
|
8
|
+
def self.perform(change)
|
9
|
+
return unless change.mr_web_url.present?
|
10
|
+
|
11
|
+
change.changed_files.each do |file|
|
12
|
+
next unless File.file?(file)
|
13
|
+
|
14
|
+
content = File.read(file)
|
15
|
+
content.gsub!(MR_WEB_URL_PLACEHOLDER, change.mr_web_url)
|
16
|
+
|
17
|
+
File.write(file, content)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
metadata
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: gitlab-housekeeper
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0.codechallenge5e4fde.pre
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- group::tenant-scale
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-03-13 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: awesome_print
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: httparty
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rubocop
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: gitlab-styles
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rspec-rails
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rubocop-rspec
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: webmock
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description: Housekeeping following https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134487
|
126
|
+
email:
|
127
|
+
- engineering@gitlab.com
|
128
|
+
executables:
|
129
|
+
- gitlab-housekeeper
|
130
|
+
extensions: []
|
131
|
+
extra_rdoc_files: []
|
132
|
+
files:
|
133
|
+
- bin/gitlab-housekeeper
|
134
|
+
- lib/gitlab/housekeeper.rb
|
135
|
+
- lib/gitlab/housekeeper/change.rb
|
136
|
+
- lib/gitlab/housekeeper/git.rb
|
137
|
+
- lib/gitlab/housekeeper/gitlab_client.rb
|
138
|
+
- lib/gitlab/housekeeper/keep.rb
|
139
|
+
- lib/gitlab/housekeeper/keeps/rubocop_fixer.rb
|
140
|
+
- lib/gitlab/housekeeper/keeps/upgrade_dependency.rb
|
141
|
+
- lib/gitlab/housekeeper/runner.rb
|
142
|
+
- lib/gitlab/housekeeper/shell.rb
|
143
|
+
- lib/gitlab/housekeeper/substitutor.rb
|
144
|
+
- lib/gitlab/housekeeper/version.rb
|
145
|
+
homepage: https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper
|
146
|
+
licenses:
|
147
|
+
- MIT
|
148
|
+
metadata:
|
149
|
+
rubygems_mfa_required: 'true'
|
150
|
+
post_install_message:
|
151
|
+
rdoc_options: []
|
152
|
+
require_paths:
|
153
|
+
- lib
|
154
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - ">="
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '3.0'
|
159
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
requirements: []
|
165
|
+
rubygems_version: 3.5.6
|
166
|
+
signing_key:
|
167
|
+
specification_version: 4
|
168
|
+
summary: Gem summary
|
169
|
+
test_files: []
|