gitlab-housekeeper 0.1.0.codechallenge.4eada1e.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/gitlab-housekeeper +32 -0
- data/lib/gitlab/housekeeper/change.rb +61 -0
- data/lib/gitlab/housekeeper/git.rb +89 -0
- data/lib/gitlab/housekeeper/gitlab_client.rb +206 -0
- data/lib/gitlab/housekeeper/keep.rb +29 -0
- data/lib/gitlab/housekeeper/keeps/rubocop_fixer.rb +91 -0
- data/lib/gitlab/housekeeper/keeps/upgrade_dependency.rb +37 -0
- data/lib/gitlab/housekeeper/runner.rb +185 -0
- data/lib/gitlab/housekeeper/shell.rb +27 -0
- data/lib/gitlab/housekeeper/substitutor.rb +22 -0
- data/lib/gitlab/housekeeper/version.rb +7 -0
- data/lib/gitlab/housekeeper.rb +11 -0
- metadata +169 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 020c21d5518ddc149e309829db1620ca0714e6b95f111d8f704b6854c6d6fa76
|
4
|
+
data.tar.gz: 597930e33694bef637116a8bbe547c8ff31f46380c12e9d064c5f77a787296c0
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 662890ec680ec7df000f3389d13df32f709713a5e96a8d14b7bcf30b72de9e6dabf2c42e7eab1175ed48b7261a6b593044f53de0c626418ccf9486d4c42a8d73
|
7
|
+
data.tar.gz: fa9337b123899e797e35da8e7457a4b1d5ee97c89b402e03a69100d6fd57cceec12cff1b48374cde412cd40878a0172230cd865c36d50d0c5f2bf60006fdbea9
|
@@ -0,0 +1,32 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "optparse"
|
4
|
+
require 'gitlab/housekeeper'
|
5
|
+
|
6
|
+
options = {}
|
7
|
+
|
8
|
+
OptionParser.new do |opts|
|
9
|
+
opts.banner = 'Creates merge requests that can be inferred from the current state of the codebase'
|
10
|
+
|
11
|
+
opts.on('-m=M', '--max-mrs=M', Integer, 'Limit of MRs to create. Defaults to 1.') do |m|
|
12
|
+
options[:max_mrs] = m
|
13
|
+
end
|
14
|
+
|
15
|
+
opts.on('-d', '--dry-run', 'Dry-run only. Print the MR titles, descriptions and diffs') do
|
16
|
+
options[:dry_run] = true
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.on('-k OverdueFinalizeBackgroundMigration,AnotherKeep', '--keeps OverdueFinalizeBackgroundMigration,AnotherKeep', Array, 'Require keeps specified') do |k|
|
20
|
+
options[:keeps] = k
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on('--filter-identifiers some-identifier-regex,another-regex', Array, 'Skip any changes where none of the identifiers match these regexes. The identifiers is an array, so at least one element must match at least one regex.') do |filters|
|
24
|
+
options[:filter_identifiers] = filters.map { |f| Regexp.new(f) }
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on('-h', '--help', 'Prints this help') do
|
28
|
+
abort opts.to_s
|
29
|
+
end
|
30
|
+
end.parse!
|
31
|
+
|
32
|
+
Gitlab::Housekeeper::Runner.new(**options).run
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module Housekeeper
|
5
|
+
class Change
|
6
|
+
attr_accessor :identifiers,
|
7
|
+
:title,
|
8
|
+
:description,
|
9
|
+
:changed_files,
|
10
|
+
:labels,
|
11
|
+
:keep_class,
|
12
|
+
:changelog_type,
|
13
|
+
:mr_web_url
|
14
|
+
attr_reader :reviewers
|
15
|
+
|
16
|
+
def initialize
|
17
|
+
@labels = []
|
18
|
+
@reviewers = []
|
19
|
+
end
|
20
|
+
|
21
|
+
def reviewers=(reviewers)
|
22
|
+
@reviewers = Array(reviewers)
|
23
|
+
end
|
24
|
+
|
25
|
+
def mr_description
|
26
|
+
<<~MARKDOWN
|
27
|
+
#{description}
|
28
|
+
|
29
|
+
This change was generated by
|
30
|
+
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper)
|
31
|
+
using the #{keep_class} keep.
|
32
|
+
|
33
|
+
To provide feedback on your experience with `gitlab-housekeeper` please comment in
|
34
|
+
<https://gitlab.com/gitlab-org/gitlab/-/issues/442003>.
|
35
|
+
MARKDOWN
|
36
|
+
end
|
37
|
+
|
38
|
+
def commit_message
|
39
|
+
<<~MARKDOWN
|
40
|
+
#{title}
|
41
|
+
|
42
|
+
#{mr_description}
|
43
|
+
|
44
|
+
Changelog: #{changelog_type || 'other'}
|
45
|
+
MARKDOWN
|
46
|
+
end
|
47
|
+
|
48
|
+
def matches_filters?(filters)
|
49
|
+
filters.all? do |filter|
|
50
|
+
identifiers.any? do |identifier|
|
51
|
+
identifier.match?(filter)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def valid?
|
57
|
+
@identifiers && @title && @description && @changed_files
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'logger'
|
4
|
+
require 'gitlab/housekeeper/shell'
|
5
|
+
|
6
|
+
module Gitlab
|
7
|
+
module Housekeeper
|
8
|
+
class Git
|
9
|
+
def initialize(logger:, branch_from: 'master')
|
10
|
+
@logger = logger
|
11
|
+
@branch_from = branch_from
|
12
|
+
end
|
13
|
+
|
14
|
+
def with_clean_state
|
15
|
+
result = Shell.execute('git', 'stash')
|
16
|
+
stashed = !result.include?('No local changes to save')
|
17
|
+
|
18
|
+
with_return_to_current_branch(stashed: stashed) do
|
19
|
+
checkout_branch(@branch_from)
|
20
|
+
|
21
|
+
yield
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def create_branch(change)
|
26
|
+
branch_name = branch_name(change.identifiers)
|
27
|
+
|
28
|
+
Shell.execute("git", "branch", "-f", branch_name)
|
29
|
+
|
30
|
+
branch_name
|
31
|
+
end
|
32
|
+
|
33
|
+
def in_branch(branch_name)
|
34
|
+
with_return_to_current_branch do
|
35
|
+
checkout_branch(branch_name)
|
36
|
+
|
37
|
+
yield
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def create_commit(change)
|
42
|
+
Shell.execute("git", "add", *change.changed_files)
|
43
|
+
Shell.execute("git", "commit", "-m", change.commit_message)
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def create_and_checkout_branch(branch_name)
|
49
|
+
begin
|
50
|
+
Shell.execute("git", "branch", '-D', branch_name)
|
51
|
+
rescue Shell::Error # Might not exist yet
|
52
|
+
end
|
53
|
+
|
54
|
+
Shell.execute("git", "checkout", "-b", branch_name)
|
55
|
+
end
|
56
|
+
|
57
|
+
def checkout_branch(branch_name)
|
58
|
+
Shell.execute("git", "checkout", branch_name)
|
59
|
+
end
|
60
|
+
|
61
|
+
def branch_name(identifiers)
|
62
|
+
# Hyphen-case each identifier then join together with hyphens.
|
63
|
+
branch_name = identifiers
|
64
|
+
.map { |i| i.gsub(/[^\w]+/, '-') }
|
65
|
+
.map { |i| i.gsub(/[[:upper:]]/) { |w| "-#{w.downcase}" } }
|
66
|
+
.join('-')
|
67
|
+
.delete_prefix("-")
|
68
|
+
|
69
|
+
# Truncate if it's too long and add a digest
|
70
|
+
if branch_name.length > 240
|
71
|
+
branch_name = branch_name[0...200] + OpenSSL::Digest::SHA256.hexdigest(branch_name)[0...15]
|
72
|
+
end
|
73
|
+
|
74
|
+
branch_name
|
75
|
+
end
|
76
|
+
|
77
|
+
def with_return_to_current_branch(stashed: false)
|
78
|
+
current_branch = Shell.execute('git', 'branch', '--show-current').chomp
|
79
|
+
|
80
|
+
yield
|
81
|
+
ensure
|
82
|
+
# The `current_branch` won't be set in CI due to how the repo is cloned. Therefore we should only checkout
|
83
|
+
# `current_branch` if we actually have one.
|
84
|
+
checkout_branch(current_branch) if current_branch.present?
|
85
|
+
Shell.execute('git', 'stash', 'pop') if stashed
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,206 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'httparty'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
module Gitlab
|
7
|
+
module Housekeeper
|
8
|
+
class GitlabClient
|
9
|
+
Error = Class.new(StandardError)
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@token = ENV.fetch("HOUSEKEEPER_GITLAB_API_TOKEN")
|
13
|
+
@base_uri = 'https://gitlab.com/api/v4'
|
14
|
+
end
|
15
|
+
|
16
|
+
# This looks at the system notes of the merge request to detect if it has been updated by anyone other than the
|
17
|
+
# current housekeeper user. If it has then it assumes that they did this for a reason and we can skip updating
|
18
|
+
# this detail of the merge request. Otherwise we assume we should generate it again using the latest output.
|
19
|
+
def non_housekeeper_changes(
|
20
|
+
source_project_id:,
|
21
|
+
source_branch:,
|
22
|
+
target_branch:,
|
23
|
+
target_project_id:
|
24
|
+
)
|
25
|
+
|
26
|
+
existing_merge_request = get_existing_merge_request(
|
27
|
+
source_project_id: source_project_id,
|
28
|
+
source_branch: source_branch,
|
29
|
+
target_branch: target_branch,
|
30
|
+
target_project_id: target_project_id
|
31
|
+
)
|
32
|
+
|
33
|
+
return [] if existing_merge_request.nil?
|
34
|
+
|
35
|
+
merge_request_notes = get_merge_request_notes(
|
36
|
+
target_project_id: target_project_id,
|
37
|
+
iid: existing_merge_request['iid']
|
38
|
+
)
|
39
|
+
|
40
|
+
changes = Set.new
|
41
|
+
|
42
|
+
merge_request_notes.each do |note|
|
43
|
+
next false unless note["system"]
|
44
|
+
next false if note["author"]["id"] == current_user_id
|
45
|
+
|
46
|
+
changes << :title if note['body'].start_with?("changed title from")
|
47
|
+
changes << :description if note['body'] == "changed the description"
|
48
|
+
changes << :code if note['body'].match?(/added \d+ commit/)
|
49
|
+
|
50
|
+
changes << :reviewers if note['body'].include?('requested review from')
|
51
|
+
changes << :reviewers if note['body'].include?('removed review request for')
|
52
|
+
end
|
53
|
+
|
54
|
+
resource_label_events = get_merge_request_resource_label_events(
|
55
|
+
target_project_id: target_project_id,
|
56
|
+
iid: existing_merge_request['iid']
|
57
|
+
)
|
58
|
+
|
59
|
+
resource_label_events.each do |event|
|
60
|
+
next if event["user"]["id"] == current_user_id
|
61
|
+
|
62
|
+
# Labels are routinely added by both humans and bots, so addition events aren't cause for concern.
|
63
|
+
# However, if labels have been removed it may mean housekeeper added an incorrect label, and we shouldn't
|
64
|
+
# re-add them.
|
65
|
+
#
|
66
|
+
# TODO: Inspect the actual labels housekeeper wants to add, and add if they haven't previously been removed.
|
67
|
+
changes << :labels if event["action"] == "remove"
|
68
|
+
end
|
69
|
+
|
70
|
+
changes.to_a
|
71
|
+
end
|
72
|
+
|
73
|
+
# rubocop:disable Metrics/ParameterLists
|
74
|
+
def create_or_update_merge_request(
|
75
|
+
change:,
|
76
|
+
source_project_id:,
|
77
|
+
source_branch:,
|
78
|
+
target_branch:,
|
79
|
+
target_project_id:,
|
80
|
+
update_title:,
|
81
|
+
update_description:,
|
82
|
+
update_labels:,
|
83
|
+
update_reviewers:
|
84
|
+
)
|
85
|
+
existing_merge_request = get_existing_merge_request(
|
86
|
+
source_project_id: source_project_id,
|
87
|
+
source_branch: source_branch,
|
88
|
+
target_branch: target_branch,
|
89
|
+
target_project_id: target_project_id
|
90
|
+
)
|
91
|
+
|
92
|
+
if existing_merge_request
|
93
|
+
update_existing_merge_request(
|
94
|
+
change: change,
|
95
|
+
existing_iid: existing_merge_request['iid'],
|
96
|
+
target_project_id: target_project_id,
|
97
|
+
update_title: update_title,
|
98
|
+
update_description: update_description,
|
99
|
+
update_labels: update_labels,
|
100
|
+
update_reviewers: update_reviewers
|
101
|
+
)
|
102
|
+
else
|
103
|
+
create_merge_request(
|
104
|
+
change: change,
|
105
|
+
source_project_id: source_project_id,
|
106
|
+
source_branch: source_branch,
|
107
|
+
target_branch: target_branch,
|
108
|
+
target_project_id: target_project_id
|
109
|
+
)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
# rubocop:enable Metrics/ParameterLists
|
113
|
+
|
114
|
+
def get_existing_merge_request(source_project_id:, source_branch:, target_branch:, target_project_id:)
|
115
|
+
data = request(:get, "/projects/#{target_project_id}/merge_requests", query: {
|
116
|
+
state: :opened,
|
117
|
+
source_branch: source_branch,
|
118
|
+
target_branch: target_branch,
|
119
|
+
source_project_id: source_project_id
|
120
|
+
})
|
121
|
+
|
122
|
+
return nil if data.empty?
|
123
|
+
|
124
|
+
raise Error, "More than one matching MR exists: iids: #{data.pluck('iid').join(',')}" unless data.size == 1
|
125
|
+
|
126
|
+
data.first
|
127
|
+
end
|
128
|
+
|
129
|
+
private
|
130
|
+
|
131
|
+
def get_merge_request_notes(target_project_id:, iid:)
|
132
|
+
request(:get, "/projects/#{target_project_id}/merge_requests/#{iid}/notes", query: { per_page: 100 })
|
133
|
+
end
|
134
|
+
|
135
|
+
def get_merge_request_resource_label_events(target_project_id:, iid:)
|
136
|
+
request(:get, "/projects/#{target_project_id}/merge_requests/#{iid}/resource_label_events",
|
137
|
+
query: { per_page: 100 })
|
138
|
+
end
|
139
|
+
|
140
|
+
def current_user_id
|
141
|
+
@current_user_id ||= request(:get, "/user")['id']
|
142
|
+
end
|
143
|
+
|
144
|
+
def create_merge_request(
|
145
|
+
change:,
|
146
|
+
source_project_id:,
|
147
|
+
source_branch:,
|
148
|
+
target_branch:,
|
149
|
+
target_project_id:
|
150
|
+
)
|
151
|
+
request(:post, "/projects/#{source_project_id}/merge_requests", body: {
|
152
|
+
title: change.title,
|
153
|
+
description: change.mr_description,
|
154
|
+
labels: Array(change.labels).join(','),
|
155
|
+
source_branch: source_branch,
|
156
|
+
target_branch: target_branch,
|
157
|
+
target_project_id: target_project_id,
|
158
|
+
remove_source_branch: true,
|
159
|
+
reviewer_ids: usernames_to_ids(change.reviewers)
|
160
|
+
})
|
161
|
+
end
|
162
|
+
|
163
|
+
def update_existing_merge_request(
|
164
|
+
change:,
|
165
|
+
existing_iid:,
|
166
|
+
target_project_id:,
|
167
|
+
update_title:,
|
168
|
+
update_description:,
|
169
|
+
update_labels:,
|
170
|
+
update_reviewers:
|
171
|
+
)
|
172
|
+
body = {}
|
173
|
+
|
174
|
+
body[:title] = change.title if update_title
|
175
|
+
body[:description] = change.mr_description if update_description
|
176
|
+
body[:add_labels] = Array(change.labels).join(',') if update_labels
|
177
|
+
body[:reviewer_ids] = usernames_to_ids(change.reviewers) if update_reviewers
|
178
|
+
|
179
|
+
return if body.empty?
|
180
|
+
|
181
|
+
request(:put, "/projects/#{target_project_id}/merge_requests/#{existing_iid}", body: body)
|
182
|
+
end
|
183
|
+
|
184
|
+
def usernames_to_ids(usernames)
|
185
|
+
Array(usernames).map do |username|
|
186
|
+
data = request(:get, "/users", query: { username: username })
|
187
|
+
data[0]['id']
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
def request(method, path, query: {}, body: {})
|
192
|
+
response = HTTParty.public_send(method, "#{@base_uri}#{path}", query: query, body: body.to_json, headers: { # rubocop:disable GitlabSecurity/PublicSend
|
193
|
+
'Private-Token' => @token,
|
194
|
+
'Content-Type' => 'application/json'
|
195
|
+
})
|
196
|
+
|
197
|
+
unless (200..299).cover?(response.code)
|
198
|
+
raise Error,
|
199
|
+
"Failed with response code: #{response.code} and body:\n#{response.body}"
|
200
|
+
end
|
201
|
+
|
202
|
+
JSON.parse(response.body)
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module Housekeeper
|
5
|
+
# A Keep is analogous to a Cop in RuboCop. The Keep is responsible for:
|
6
|
+
# - Detecting a specific code change that should be made (eg. removing an old feature flag)
|
7
|
+
# - Making the code change (eg. delete the feature flag YML file)
|
8
|
+
# - Yielding a Change object that describes this change, For example:
|
9
|
+
# ```
|
10
|
+
# yield Gitlab::Housekeeper::Change.new(
|
11
|
+
# identifiers: ['remove-old-ff', 'old_ff_name'], # Unique and stable identifier for branch name
|
12
|
+
# title: "Remove old feature flag old_ff_name as it has been enabled since %15.0",
|
13
|
+
# description: "This feature flag was enabled in 15.0 and is not needed anymore ...",
|
14
|
+
# changed_files: ["config/feature_flags/ops/old_ff_name.yml", "app/models/user.rb"]
|
15
|
+
# )
|
16
|
+
# ```
|
17
|
+
class Keep
|
18
|
+
# The each_change method must update local working copy files and yield a Change object which describes the
|
19
|
+
# specific changed files and other data that will be used to generate a merge request. This is the core
|
20
|
+
# implementation details for a specific housekeeper keep. This does not need to commit the changes or create the
|
21
|
+
# merge request as that is handled by the gitlab-housekeeper gem.
|
22
|
+
#
|
23
|
+
# @yieldparam [Gitlab::Housekeeper::Change]
|
24
|
+
def each_change
|
25
|
+
raise NotImplementedError, "A Keep must implement each_change method"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
module Housekeeper
|
7
|
+
module Keeps
|
8
|
+
class RubocopFixer < Keep
|
9
|
+
LIMIT_FIXES = 20
|
10
|
+
RUBOCOP_TODO_DIR_PATTERN = ".rubocop_todo/**/*.yml"
|
11
|
+
|
12
|
+
def initialize(todo_dir_pattern: RUBOCOP_TODO_DIR_PATTERN, limit_fixes: LIMIT_FIXES)
|
13
|
+
@todo_dir_pattern = todo_dir_pattern
|
14
|
+
@limit_fixes = limit_fixes
|
15
|
+
end
|
16
|
+
|
17
|
+
def each_change
|
18
|
+
each_allowed_rubocop_rule do |rule, rule_file_path, violating_files|
|
19
|
+
remove_allow_rule = true
|
20
|
+
|
21
|
+
if violating_files.count > @limit_fixes
|
22
|
+
violating_files = violating_files.first(@limit_fixes)
|
23
|
+
remove_allow_rule = false
|
24
|
+
end
|
25
|
+
|
26
|
+
change = Change.new
|
27
|
+
change.title = "Fix #{violating_files.count} rubocop violations for #{rule}"
|
28
|
+
change.identifiers = [self.class.name, rule, violating_files.last]
|
29
|
+
change.description = <<~MARKDOWN
|
30
|
+
Fixes the #{violating_files.count} violations for the rubocop rule `#{rule}`
|
31
|
+
that were previously excluded in `#{rule_file_path}`.
|
32
|
+
The exclusions have now been removed.
|
33
|
+
MARKDOWN
|
34
|
+
|
35
|
+
if remove_allow_rule
|
36
|
+
FileUtils.rm(rule_file_path)
|
37
|
+
else
|
38
|
+
remove_first_exclusions(rule, rule_file_path, violating_files.count)
|
39
|
+
end
|
40
|
+
|
41
|
+
begin
|
42
|
+
::Gitlab::Housekeeper::Shell.execute('rubocop', '--autocorrect', *violating_files)
|
43
|
+
rescue ::Gitlab::Housekeeper::Shell::Error
|
44
|
+
# Ignore when it cannot be automatically fixed. But we need to checkout any files we might have updated.
|
45
|
+
::Gitlab::Housekeeper::Shell.execute('git', 'checkout', rule_file_path, *violating_files)
|
46
|
+
next
|
47
|
+
end
|
48
|
+
|
49
|
+
change.changed_files = [rule_file_path, *violating_files]
|
50
|
+
|
51
|
+
yield(change)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def each_allowed_rubocop_rule
|
56
|
+
Dir.glob(@todo_dir_pattern).each do |file|
|
57
|
+
content = File.read(file)
|
58
|
+
next unless content.include?('Cop supports --autocorrect')
|
59
|
+
|
60
|
+
data = YAML.safe_load(content)
|
61
|
+
|
62
|
+
# Assume one per file since that's how it is in gitlab-org/gitlab
|
63
|
+
next unless data.keys.count == 1
|
64
|
+
|
65
|
+
rule = data.keys[0]
|
66
|
+
violating_files = data[rule]['Exclude']
|
67
|
+
next unless violating_files&.count&.positive?
|
68
|
+
|
69
|
+
yield rule, file, violating_files
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def remove_first_exclusions(_rule, file, remove_count)
|
74
|
+
content = File.read(file)
|
75
|
+
skipped = 0
|
76
|
+
|
77
|
+
output = content.each_line.filter do |line|
|
78
|
+
if skipped < remove_count && line.match?(/\s+-\s+/)
|
79
|
+
skipped += 1
|
80
|
+
false
|
81
|
+
else
|
82
|
+
true
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
File.write(file, output.join)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module Housekeeper
|
5
|
+
module Keeps
|
6
|
+
class UpgradeDependency < Keep
|
7
|
+
def initialize(
|
8
|
+
upgrade_command: ENV.fetch('HOUSEKEEPER_DEPENDENCY_UPGRADE_COMMAND'), # bundle update haml
|
9
|
+
changed_files: ENV.fetch('HOUSEKEEPER_DEPENDENCY_UPGRADE_CHANGED_FILES').split(',') # Gemfile.lock
|
10
|
+
)
|
11
|
+
@upgrade_command = upgrade_command
|
12
|
+
@changed_files = changed_files
|
13
|
+
end
|
14
|
+
|
15
|
+
def each_change
|
16
|
+
::Gitlab::Housekeeper::Shell.execute(@upgrade_command)
|
17
|
+
|
18
|
+
change = Change.new
|
19
|
+
change.title = "Run #{@upgrade_command}"
|
20
|
+
change.identifiers = [self.class.name, @upgrade_command]
|
21
|
+
change.description = <<~MARKDOWN
|
22
|
+
Automatically upgrade a dependency using:
|
23
|
+
|
24
|
+
```
|
25
|
+
#{@upgrade_command}
|
26
|
+
```
|
27
|
+
MARKDOWN
|
28
|
+
|
29
|
+
|
30
|
+
change.changed_files = @changed_files
|
31
|
+
|
32
|
+
yield(change)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,185 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support'
|
4
|
+
require 'active_support/core_ext'
|
5
|
+
require 'gitlab/housekeeper/keep'
|
6
|
+
require 'gitlab/housekeeper/keeps/rubocop_fixer'
|
7
|
+
require 'gitlab/housekeeper/keeps/upgrade_dependency'
|
8
|
+
require 'gitlab/housekeeper/gitlab_client'
|
9
|
+
require 'gitlab/housekeeper/git'
|
10
|
+
require 'gitlab/housekeeper/change'
|
11
|
+
require 'gitlab/housekeeper/substitutor'
|
12
|
+
require 'awesome_print'
|
13
|
+
require 'digest'
|
14
|
+
|
15
|
+
module Gitlab
|
16
|
+
module Housekeeper
|
17
|
+
class Runner
|
18
|
+
def initialize(max_mrs: 1, dry_run: false, keeps: nil, filter_identifiers: [])
|
19
|
+
@max_mrs = max_mrs
|
20
|
+
@dry_run = dry_run
|
21
|
+
@logger = Logger.new($stdout)
|
22
|
+
require_keeps
|
23
|
+
|
24
|
+
@keeps = if keeps
|
25
|
+
keeps.map { |k| k.is_a?(String) ? k.constantize : k }
|
26
|
+
else
|
27
|
+
all_keeps
|
28
|
+
end
|
29
|
+
|
30
|
+
@filter_identifiers = filter_identifiers
|
31
|
+
end
|
32
|
+
|
33
|
+
def run
|
34
|
+
created = 0
|
35
|
+
|
36
|
+
git.with_clean_state do
|
37
|
+
@keeps.each do |keep_class|
|
38
|
+
keep = keep_class.new
|
39
|
+
keep.each_change do |change|
|
40
|
+
unless change.valid?
|
41
|
+
puts "Ignoring invalid change from: #{keep_class}"
|
42
|
+
next
|
43
|
+
end
|
44
|
+
|
45
|
+
change.keep_class ||= keep_class
|
46
|
+
|
47
|
+
branch_name = git.create_branch(change)
|
48
|
+
add_standard_change_data(change)
|
49
|
+
|
50
|
+
unless change.matches_filters?(@filter_identifiers)
|
51
|
+
# At this point the keep has already run and edited files so we need to
|
52
|
+
# restore the local working copy. We could simply checkout all
|
53
|
+
# changed_files but this is very risky as it could mean losing work that
|
54
|
+
# cannot be recovered. Instead we commit all the work to the branch and
|
55
|
+
# move on without pushing the branch.
|
56
|
+
git.in_branch(branch_name) do
|
57
|
+
git.create_commit(change)
|
58
|
+
end
|
59
|
+
|
60
|
+
puts "Skipping change: #{change.identifiers} due to not matching filter."
|
61
|
+
puts "Modified files have been committed to branch #{branch_name.yellowish}, but will not be pushed."
|
62
|
+
puts
|
63
|
+
|
64
|
+
next
|
65
|
+
end
|
66
|
+
|
67
|
+
# If no merge request exists yet, create an empty one to allow keeps to use the web URL.
|
68
|
+
unless @dry_run
|
69
|
+
merge_reqeust = get_existing_merge_request(branch_name) || create(change, branch_name)
|
70
|
+
|
71
|
+
change.mr_web_url = merge_reqeust['web_url']
|
72
|
+
end
|
73
|
+
|
74
|
+
git.in_branch(branch_name) do
|
75
|
+
Gitlab::Housekeeper::Substitutor.perform(change)
|
76
|
+
|
77
|
+
git.create_commit(change)
|
78
|
+
end
|
79
|
+
|
80
|
+
print_change_details(change, branch_name)
|
81
|
+
|
82
|
+
create(change, branch_name) unless @dry_run
|
83
|
+
|
84
|
+
created += 1
|
85
|
+
break if created >= @max_mrs
|
86
|
+
end
|
87
|
+
break if created >= @max_mrs
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
puts "Housekeeper created #{created} MRs"
|
92
|
+
end
|
93
|
+
|
94
|
+
def add_standard_change_data(change)
|
95
|
+
change.labels ||= []
|
96
|
+
change.labels << 'automation:gitlab-housekeeper-authored'
|
97
|
+
end
|
98
|
+
|
99
|
+
def git
|
100
|
+
@git ||= ::Gitlab::Housekeeper::Git.new(logger: @logger)
|
101
|
+
end
|
102
|
+
|
103
|
+
def require_keeps
|
104
|
+
Dir.glob("keeps/*.rb").each do |f|
|
105
|
+
require(Pathname(f).expand_path.to_s)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def print_change_details(change, branch_name)
|
110
|
+
puts "Merge request URL: #{change.mr_web_url || '(known after create)'}, on branch #{branch_name}".yellowish
|
111
|
+
puts "=> #{change.identifiers.join(': ')}".purple
|
112
|
+
|
113
|
+
puts '=> Title:'.purple
|
114
|
+
puts change.title.purple
|
115
|
+
puts
|
116
|
+
|
117
|
+
puts '=> Description:'
|
118
|
+
puts change.description
|
119
|
+
puts
|
120
|
+
|
121
|
+
if change.labels.present? || change.reviewers.present?
|
122
|
+
puts '=> Attributes:'
|
123
|
+
puts "Labels: #{change.labels.join(', ')}"
|
124
|
+
puts "Reviewers: #{change.reviewers.join(', ')}"
|
125
|
+
puts
|
126
|
+
end
|
127
|
+
|
128
|
+
puts '=> Diff:'
|
129
|
+
puts Shell.execute('git', '--no-pager', 'diff', '--color=always', 'master', branch_name, '--',
|
130
|
+
*change.changed_files)
|
131
|
+
puts
|
132
|
+
end
|
133
|
+
|
134
|
+
def create(change, branch_name)
|
135
|
+
non_housekeeper_changes = gitlab_client.non_housekeeper_changes(
|
136
|
+
source_project_id: housekeeper_fork_project_id,
|
137
|
+
source_branch: branch_name,
|
138
|
+
target_branch: 'master',
|
139
|
+
target_project_id: housekeeper_target_project_id
|
140
|
+
)
|
141
|
+
|
142
|
+
unless non_housekeeper_changes.include?(:code)
|
143
|
+
Shell.execute('git', 'push', '-f', 'housekeeper', "#{branch_name}:#{branch_name}")
|
144
|
+
end
|
145
|
+
|
146
|
+
gitlab_client.create_or_update_merge_request(
|
147
|
+
change: change,
|
148
|
+
source_project_id: housekeeper_fork_project_id,
|
149
|
+
source_branch: branch_name,
|
150
|
+
target_branch: 'master',
|
151
|
+
target_project_id: housekeeper_target_project_id,
|
152
|
+
update_title: !non_housekeeper_changes.include?(:title),
|
153
|
+
update_description: !non_housekeeper_changes.include?(:description),
|
154
|
+
update_labels: !non_housekeeper_changes.include?(:labels),
|
155
|
+
update_reviewers: !non_housekeeper_changes.include?(:reviewers)
|
156
|
+
)
|
157
|
+
end
|
158
|
+
|
159
|
+
def get_existing_merge_request(branch_name)
|
160
|
+
gitlab_client.get_existing_merge_request(
|
161
|
+
source_project_id: housekeeper_fork_project_id,
|
162
|
+
source_branch: branch_name,
|
163
|
+
target_branch: 'master',
|
164
|
+
target_project_id: housekeeper_target_project_id
|
165
|
+
)
|
166
|
+
end
|
167
|
+
|
168
|
+
def housekeeper_fork_project_id
|
169
|
+
ENV.fetch('HOUSEKEEPER_FORK_PROJECT_ID')
|
170
|
+
end
|
171
|
+
|
172
|
+
def housekeeper_target_project_id
|
173
|
+
ENV.fetch('HOUSEKEEPER_TARGET_PROJECT_ID')
|
174
|
+
end
|
175
|
+
|
176
|
+
def gitlab_client
|
177
|
+
@gitlab_client ||= GitlabClient.new
|
178
|
+
end
|
179
|
+
|
180
|
+
def all_keeps
|
181
|
+
@all_keeps ||= ObjectSpace.each_object(Class).select { |klass| klass < Keep }
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'open3'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
module Housekeeper
|
7
|
+
class Shell
|
8
|
+
Error = Class.new(StandardError)
|
9
|
+
|
10
|
+
def self.execute(*cmd)
|
11
|
+
stdin, stdout, stderr, wait_thr = Open3.popen3(*cmd)
|
12
|
+
|
13
|
+
stdin.close
|
14
|
+
out = stdout.read
|
15
|
+
stdout.close
|
16
|
+
err = stderr.read
|
17
|
+
stderr.close
|
18
|
+
|
19
|
+
exit_status = wait_thr.value
|
20
|
+
|
21
|
+
raise Error, "Failed with #{exit_status}\n#{out}\n#{err}\n" unless exit_status.success?
|
22
|
+
|
23
|
+
out + err
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module Housekeeper
|
5
|
+
class Substitutor
|
6
|
+
MR_WEB_URL_PLACEHOLDER = '<HOUSEKEEPER_MR_WEB_URL>'
|
7
|
+
|
8
|
+
def self.perform(change)
|
9
|
+
return unless change.mr_web_url.present?
|
10
|
+
|
11
|
+
change.changed_files.each do |file|
|
12
|
+
next unless File.file?(file)
|
13
|
+
|
14
|
+
content = File.read(file)
|
15
|
+
content.gsub!(MR_WEB_URL_PLACEHOLDER, change.mr_web_url)
|
16
|
+
|
17
|
+
File.write(file, content)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
metadata
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: gitlab-housekeeper
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0.codechallenge.4eada1e.pre
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- group::tenant-scale
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-03-13 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: awesome_print
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: httparty
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rubocop
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: gitlab-styles
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rspec-rails
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rubocop-rspec
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: webmock
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description: Housekeeping following https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134487
|
126
|
+
email:
|
127
|
+
- engineering@gitlab.com
|
128
|
+
executables:
|
129
|
+
- gitlab-housekeeper
|
130
|
+
extensions: []
|
131
|
+
extra_rdoc_files: []
|
132
|
+
files:
|
133
|
+
- bin/gitlab-housekeeper
|
134
|
+
- lib/gitlab/housekeeper.rb
|
135
|
+
- lib/gitlab/housekeeper/change.rb
|
136
|
+
- lib/gitlab/housekeeper/git.rb
|
137
|
+
- lib/gitlab/housekeeper/gitlab_client.rb
|
138
|
+
- lib/gitlab/housekeeper/keep.rb
|
139
|
+
- lib/gitlab/housekeeper/keeps/rubocop_fixer.rb
|
140
|
+
- lib/gitlab/housekeeper/keeps/upgrade_dependency.rb
|
141
|
+
- lib/gitlab/housekeeper/runner.rb
|
142
|
+
- lib/gitlab/housekeeper/shell.rb
|
143
|
+
- lib/gitlab/housekeeper/substitutor.rb
|
144
|
+
- lib/gitlab/housekeeper/version.rb
|
145
|
+
homepage: https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper
|
146
|
+
licenses:
|
147
|
+
- MIT
|
148
|
+
metadata:
|
149
|
+
rubygems_mfa_required: 'true'
|
150
|
+
post_install_message:
|
151
|
+
rdoc_options: []
|
152
|
+
require_paths:
|
153
|
+
- lib
|
154
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - ">="
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '3.0'
|
159
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
requirements: []
|
165
|
+
rubygems_version: 3.5.6
|
166
|
+
signing_key:
|
167
|
+
specification_version: 4
|
168
|
+
summary: Gem summary
|
169
|
+
test_files: []
|