gitlab_quality-test_tooling 1.11.0 → 1.13.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +19 -2
- data/exe/flaky-test-issues +5 -0
- data/exe/knapsack-report-issues +54 -0
- data/lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb +47 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/branches_client.rb +18 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/branches_dry_client.rb +15 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/commits_client.rb +18 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/commits_dry_client.rb +14 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb +14 -9
- data/lib/gitlab_quality/test_tooling/gitlab_client/repository_files_client.rb +23 -0
- data/lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time.rb +85 -0
- data/lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time_report.rb +60 -0
- data/lib/gitlab_quality/test_tooling/report/concerns/utils.rb +22 -0
- data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb +13 -3
- data/lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb +142 -0
- data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +2 -4
- data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb +11 -2
- data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb +1 -3
- data/lib/gitlab_quality/test_tooling/slack/post_to_slack_dry.rb +14 -0
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +18 -7
- data/lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb +0 -49
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0b5576ca28a8e8cbf76564017a53418fd059747789bbbb8146835285d8f5b37c
|
4
|
+
data.tar.gz: b1d7c6dd581dccabe91f64687b420a83f5561aa407e1e4583e7a5146183890f4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d245b9be5a9a127630d934096277c339e649e9e719727f871d8b794245b385e6f4dbfba3175c546f44b2ee833099ef5398738dcaa631bab39b44eb9734fe338f
|
7
|
+
data.tar.gz: 33a391d891853d45e75198462d504641d090e0f619e31a9fbf5c613b4fe5cef6443d6b465e9e2ffc27d643a3d8d8426030bb10c872a83ca0a2def1b0232ae0ae
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -141,15 +141,32 @@ Usage: exe/slow-test-issues [options]
|
|
141
141
|
-h, --help Show the usage
|
142
142
|
```
|
143
143
|
|
144
|
+
### `exe/knapsack-report-issues`
|
145
|
+
|
146
|
+
```shell
|
147
|
+
Purpose: Create spec run time issue when a spec file almost caused job timeout because it ran significantly longer than what Knapsack expected.
|
148
|
+
Usage: exe/knapsack-report-issues [options]
|
149
|
+
-i, --input-file INPUT_FILE Knapsack actual run time report file path glob
|
150
|
+
-e EXPECTED_REPORT, Knapsack expected report file path
|
151
|
+
--expected-report
|
152
|
+
-p, --project PROJECT Can be an integer or a group/project string
|
153
|
+
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
154
|
+
--dry-run Perform a dry-run (don't create issues)
|
155
|
+
-v, --version Show the version
|
156
|
+
-h, --help Show the usage
|
157
|
+
```
|
158
|
+
|
144
159
|
### `exe/flaky-test-issues`
|
145
160
|
|
146
161
|
```shell
|
147
|
-
Purpose:
|
162
|
+
Purpose: Create flaky test issues for any passed test coming from rspec-retry JSON report files.
|
148
163
|
Usage: exe/flaky-test-issues [options]
|
149
164
|
-i, --input-files INPUT_FILES JSON rspec-retry report files
|
150
165
|
-p, --project PROJECT Can be an integer or a group/project string
|
151
166
|
-m MERGE_REQUEST_IID, An integer merge request IID
|
152
167
|
--merge_request_iid
|
168
|
+
--base-issue-labels BASE_ISSUE_LABELS
|
169
|
+
Comma-separated labels (without tilde) to add to new flaky test issues
|
153
170
|
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
154
171
|
--dry-run Perform a dry-run (don't create issues)
|
155
172
|
-v, --version Show the version
|
@@ -217,7 +234,7 @@ If you forget to set the changelog entry in your commit messages, you can also e
|
|
217
234
|
|
218
235
|
### Steps to release
|
219
236
|
|
220
|
-
Use a `Release` merge request template and create a merge
|
237
|
+
Use a `Release` merge request template and create a merge request to update the version number in `version.rb`, and get the merge request merged by a maintainer.
|
221
238
|
|
222
239
|
This will then be packaged into a gem and pushed to [rubygems.org](https://rubygems.org) by the CI/CD.
|
223
240
|
|
data/exe/flaky-test-issues
CHANGED
@@ -23,6 +23,11 @@ options = OptionParser.new do |opts|
|
|
23
23
|
params[:merge_request_iid] = merge_request_iid
|
24
24
|
end
|
25
25
|
|
26
|
+
opts.on('--base-issue-labels BASE_ISSUE_LABELS', String,
|
27
|
+
'Comma-separated labels (without tilde) to add to new flaky test issues') do |base_issue_labels|
|
28
|
+
params[:base_issue_labels] = base_issue_labels.split(',')
|
29
|
+
end
|
30
|
+
|
26
31
|
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
|
27
32
|
params[:token] = token
|
28
33
|
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "optparse"
|
6
|
+
|
7
|
+
require_relative "../lib/gitlab_quality/test_tooling"
|
8
|
+
|
9
|
+
params = {}
|
10
|
+
|
11
|
+
options = OptionParser.new do |opts|
|
12
|
+
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
|
+
|
14
|
+
opts.on("-i", "--input-file INPUT_FILE", String, "Knapsack actual run time report file path glob") do |input_file|
|
15
|
+
params[:input_files] = input_file
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on("-e", "--expected-report EXPECTED_REPORT", String, "Knapsack expected report file path") do |report_path|
|
19
|
+
params[:expected_report] = report_path.strip
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
|
23
|
+
params[:project] = project
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
|
27
|
+
params[:token] = token
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on('--dry-run', "Perform a dry-run (don't create issues)") do
|
31
|
+
params[:dry_run] = true
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on_tail('-v', '--version', 'Show the version') do
|
35
|
+
require_relative "../lib/gitlab_quality/test_tooling/version"
|
36
|
+
puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
|
37
|
+
exit
|
38
|
+
end
|
39
|
+
|
40
|
+
opts.on_tail('-h', '--help', 'Show the usage') do
|
41
|
+
puts "Purpose: Create spec run time issue when a spec file almost caused job timeout because it ran significantly longer than what Knapsack expected."
|
42
|
+
puts opts
|
43
|
+
exit
|
44
|
+
end
|
45
|
+
|
46
|
+
opts.parse(ARGV)
|
47
|
+
end
|
48
|
+
|
49
|
+
if params.any?
|
50
|
+
GitlabQuality::TestTooling::Report::KnapsackReportIssue.new(**params).invoke!
|
51
|
+
else
|
52
|
+
puts options
|
53
|
+
exit 1
|
54
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module GitlabQuality
|
6
|
+
module TestTooling
|
7
|
+
module Concerns
|
8
|
+
module FindSetDri
|
9
|
+
def set_dri_via_group(product_group, stage)
|
10
|
+
parse_json_with_sets
|
11
|
+
fetch_stage_sets(stage)
|
12
|
+
|
13
|
+
return @sets.sample['username'] if @stage_sets.empty?
|
14
|
+
|
15
|
+
fetch_group_sets(product_group)
|
16
|
+
|
17
|
+
if @group_sets.empty?
|
18
|
+
@stage_sets.sample['username']
|
19
|
+
else
|
20
|
+
@group_sets.sample['username']
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def parse_json_with_sets
|
27
|
+
response = Support::HttpRequest.make_http_request(
|
28
|
+
url: 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json'
|
29
|
+
)
|
30
|
+
@sets = JSON.parse(response.body).select { |user| user['role'].include?('software-engineer-in-test') }
|
31
|
+
end
|
32
|
+
|
33
|
+
def fetch_stage_sets(stage)
|
34
|
+
@stage_sets = @sets.select do |user|
|
35
|
+
user['role'].include?(stage.split("_").map(&:capitalize).join(" "))
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def fetch_group_sets(product_group)
|
40
|
+
@group_sets = @stage_sets.select do |user|
|
41
|
+
user['role'].downcase.tr(' ', '_').include?(product_group)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class BranchesClient < GitlabClient
|
7
|
+
def create(branch_name, ref)
|
8
|
+
branch = handle_gitlab_client_exceptions do
|
9
|
+
client.create_branch(project, branch_name, ref)
|
10
|
+
end
|
11
|
+
|
12
|
+
Runtime::Logger.debug("Created branch #{branch['name']} (#{branch['web_url']})")
|
13
|
+
branch
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class BranchesDryClient < BranchesClient
|
7
|
+
def create(branch_name, ref)
|
8
|
+
branch = { 'name' => branch_name, 'web_url' => 'https://example.com/dummy/branch/url' }
|
9
|
+
puts "A branch would have been created with name: #{branch['name']}, web_url: #{branch['web_url']} and ref: #{ref}"
|
10
|
+
branch
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class CommitsClient < GitlabClient
|
7
|
+
def create(branch_name, file_path, new_content, message)
|
8
|
+
commit = client.create_commit(project, branch_name, message, [
|
9
|
+
{ action: :update, file_path: file_path, content: new_content }
|
10
|
+
])
|
11
|
+
|
12
|
+
Runtime::Logger.debug("Created commit #{commit['id']} (#{commit['web_url']}) on #{branch_name}")
|
13
|
+
commit
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class CommitsDryClient < CommitsClient
|
7
|
+
def create(branch_name, file_path, new_content, message)
|
8
|
+
puts "A commit would have been created on branch_name: #{branch_name}, file_path: #{file_path}, message: #{message} and content:"
|
9
|
+
puts new_content
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -1,7 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'gitlab'
|
4
|
-
|
5
3
|
module GitlabQuality
|
6
4
|
module TestTooling
|
7
5
|
module GitlabClient
|
@@ -10,19 +8,26 @@ module GitlabQuality
|
|
10
8
|
client.merge_request_changes(project, merge_request_iid)
|
11
9
|
end
|
12
10
|
|
13
|
-
def create_merge_request(title:, source_branch:, target_branch:, description:, labels:)
|
11
|
+
def create_merge_request(title:, source_branch:, target_branch:, description:, labels:, assignee_id: nil)
|
12
|
+
attrs = {
|
13
|
+
source_branch: source_branch,
|
14
|
+
target_branch: target_branch,
|
15
|
+
description: description,
|
16
|
+
labels: labels,
|
17
|
+
assignee_id: assignee_id,
|
18
|
+
squash: true,
|
19
|
+
remove_source_branch: true
|
20
|
+
}.compact
|
21
|
+
|
14
22
|
merge_request = handle_gitlab_client_exceptions do
|
15
23
|
client.create_merge_request(project,
|
16
24
|
title,
|
17
|
-
|
18
|
-
target_branch: target_branch,
|
19
|
-
description: description,
|
20
|
-
labels: labels,
|
21
|
-
squash: true,
|
22
|
-
remove_source_branch: true)
|
25
|
+
attrs)
|
23
26
|
end
|
24
27
|
|
25
28
|
Runtime::Logger.debug("Created merge request #{merge_request['iid']} (#{merge_request['web_url']})") if merge_request
|
29
|
+
|
30
|
+
merge_request
|
26
31
|
end
|
27
32
|
|
28
33
|
def find(iid: nil, options: {}, &select)
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class RepositoryFilesClient < GitlabClient
|
7
|
+
attr_reader :file_path
|
8
|
+
|
9
|
+
def initialize(file_path:, **kwargs)
|
10
|
+
@file_path = file_path
|
11
|
+
|
12
|
+
super
|
13
|
+
end
|
14
|
+
|
15
|
+
def file_contents
|
16
|
+
handle_gitlab_client_exceptions do
|
17
|
+
client.file_contents(project, file_path.gsub(%r{^/}, ""))
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module KnapsackReports
|
6
|
+
class SpecRunTime
|
7
|
+
attr_reader :file, :expected, :actual, :expected_suite_duration, :actual_suite_duration
|
8
|
+
|
9
|
+
ACTUAL_TO_EXPECTED_SPEC_RUN_TIME_RATIO_THRESHOLD = 1.5 # actual run time is longer than expected by 50% +
|
10
|
+
SPEC_WEIGHT_PERCENTAGE_TRESHOLD = 15 # a spec file takes 15%+ of the total test suite run time
|
11
|
+
SUITE_DURATION_THRESHOLD = 70 * 60 # if test suite takes more than 70 minutes, job risks timing out
|
12
|
+
|
13
|
+
def initialize(file:, expected:, actual:, expected_suite_duration:, actual_suite_duration:)
|
14
|
+
@file = file
|
15
|
+
@expected = expected.to_f
|
16
|
+
@actual = actual.to_f
|
17
|
+
@expected_suite_duration = expected_suite_duration.to_f
|
18
|
+
@actual_suite_duration = actual_suite_duration.to_f
|
19
|
+
end
|
20
|
+
|
21
|
+
def should_report?
|
22
|
+
# guideline proposed in https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/354
|
23
|
+
exceed_actual_to_expected_ratio_threshold? && test_suite_bottleneck?
|
24
|
+
end
|
25
|
+
|
26
|
+
def ci_pipeline_url_markdown
|
27
|
+
"[#{ci_pipeline_id}](#{ci_pipeline_url})"
|
28
|
+
end
|
29
|
+
|
30
|
+
def ci_pipeline_created_at
|
31
|
+
ENV.fetch('CI_PIPELINE_CREATED_AT', nil)
|
32
|
+
end
|
33
|
+
|
34
|
+
def ci_job_link_markdown
|
35
|
+
"[#{ci_job_name}](#{ci_job_url})"
|
36
|
+
end
|
37
|
+
|
38
|
+
def file_link_markdown
|
39
|
+
"[#{file}](#{file_link})"
|
40
|
+
end
|
41
|
+
|
42
|
+
def actual_percentage
|
43
|
+
(actual / actual_suite_duration * 100).round(2)
|
44
|
+
end
|
45
|
+
|
46
|
+
def name
|
47
|
+
nil
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def exceed_actual_to_expected_ratio_threshold?
|
53
|
+
actual / expected >= ACTUAL_TO_EXPECTED_SPEC_RUN_TIME_RATIO_THRESHOLD
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_suite_bottleneck?
|
57
|
+
# now we only report bottlenecks when they risk causing job timeouts
|
58
|
+
return unless actual_suite_duration > SUITE_DURATION_THRESHOLD
|
59
|
+
|
60
|
+
actual_percentage > SPEC_WEIGHT_PERCENTAGE_TRESHOLD
|
61
|
+
end
|
62
|
+
|
63
|
+
def ci_job_url
|
64
|
+
ENV.fetch('CI_JOB_URL', nil)
|
65
|
+
end
|
66
|
+
|
67
|
+
def ci_job_name
|
68
|
+
ENV.fetch('CI_JOB_NAME_SLUG', nil)
|
69
|
+
end
|
70
|
+
|
71
|
+
def ci_pipeline_id
|
72
|
+
ENV.fetch('CI_PIPELINE_IID', nil)
|
73
|
+
end
|
74
|
+
|
75
|
+
def ci_pipeline_url
|
76
|
+
ENV.fetch('CI_PIPELINE_URL', nil)
|
77
|
+
end
|
78
|
+
|
79
|
+
def file_link
|
80
|
+
"#{Runtime::Env.file_base_url}#{file}"
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module GitlabQuality
|
6
|
+
module TestTooling
|
7
|
+
module KnapsackReports
|
8
|
+
class SpecRunTimeReport
|
9
|
+
attr_reader :expected_report, :actual_report
|
10
|
+
|
11
|
+
def initialize(expected_report_path:, actual_report_path:)
|
12
|
+
@expected_report = parse(expected_report_path)
|
13
|
+
@actual_report = parse(actual_report_path)
|
14
|
+
end
|
15
|
+
|
16
|
+
def filtered_report
|
17
|
+
@filtered_report = actual_report.keys.filter_map do |spec_file|
|
18
|
+
expected_run_time = expected_report[spec_file]
|
19
|
+
actual_run_time = actual_report[spec_file]
|
20
|
+
|
21
|
+
if expected_run_time.nil?
|
22
|
+
puts "#{spec_file} missing from the expected Knapsack report, skipping."
|
23
|
+
next
|
24
|
+
end
|
25
|
+
|
26
|
+
spec_run_time = SpecRunTime.new(
|
27
|
+
file: spec_file,
|
28
|
+
expected: expected_run_time,
|
29
|
+
actual: actual_run_time,
|
30
|
+
expected_suite_duration: expected_test_suite_run_time_total,
|
31
|
+
actual_suite_duration: actual_test_suite_run_time_total
|
32
|
+
)
|
33
|
+
|
34
|
+
spec_run_time if spec_run_time.should_report?
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def parse(report_path)
|
41
|
+
JSON.parse(File.read(report_path))
|
42
|
+
end
|
43
|
+
|
44
|
+
def expected_test_suite_run_time_total
|
45
|
+
@expected_test_suite_run_time_total ||=
|
46
|
+
expected_report.reduce(0) do |total_run_time, (_spec_file, run_time)|
|
47
|
+
total_run_time + run_time
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def actual_test_suite_run_time_total
|
52
|
+
@actual_test_suite_run_time_total ||=
|
53
|
+
actual_report.reduce(0) do |total_run_time, (_spec_file, run_time)|
|
54
|
+
total_run_time + run_time
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -45,6 +45,28 @@ module GitlabQuality
|
|
45
45
|
|
46
46
|
@pipeline ||= Runtime::Env.pipeline_from_project_name
|
47
47
|
end
|
48
|
+
|
49
|
+
def readable_duration(duration_in_seconds)
|
50
|
+
minutes = (duration_in_seconds / 60).to_i
|
51
|
+
seconds = (duration_in_seconds % 60).round(2)
|
52
|
+
|
53
|
+
min_output = normalize_duration_output(minutes, 'minute')
|
54
|
+
sec_output = normalize_duration_output(seconds, 'second')
|
55
|
+
|
56
|
+
"#{min_output} #{sec_output}".strip
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
def normalize_duration_output(number, unit)
|
62
|
+
if number <= 0
|
63
|
+
""
|
64
|
+
elsif number <= 1
|
65
|
+
"#{number} #{unit}"
|
66
|
+
else
|
67
|
+
"#{number} #{unit}s"
|
68
|
+
end
|
69
|
+
end
|
48
70
|
end
|
49
71
|
end
|
50
72
|
end
|
@@ -19,14 +19,24 @@ module GitlabQuality
|
|
19
19
|
FOUND_IN_MR_LABEL = 'found:in MR'
|
20
20
|
FOUND_IN_MASTER_LABEL = 'found:master'
|
21
21
|
|
22
|
-
def initialize(
|
22
|
+
def initialize(
|
23
|
+
token:,
|
24
|
+
input_files:,
|
25
|
+
base_issue_labels: nil,
|
26
|
+
confidential: false,
|
27
|
+
dry_run: false,
|
28
|
+
merge_request_iid: nil,
|
29
|
+
project: nil,
|
30
|
+
**_kwargs)
|
23
31
|
super(token: token, input_files: input_files, project: project, confidential: confidential, dry_run: dry_run)
|
32
|
+
|
33
|
+
@base_issue_labels = Set.new(base_issue_labels)
|
24
34
|
@merge_request_iid = merge_request_iid
|
25
35
|
end
|
26
36
|
|
27
37
|
private
|
28
38
|
|
29
|
-
attr_reader :merge_request_iid
|
39
|
+
attr_reader :base_issue_labels, :merge_request_iid
|
30
40
|
|
31
41
|
def run!
|
32
42
|
puts "Reporting flaky tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
@@ -54,7 +64,7 @@ module GitlabQuality
|
|
54
64
|
FOUND_IN_MR_LABEL
|
55
65
|
end
|
56
66
|
|
57
|
-
NEW_ISSUE_LABELS + [found_label]
|
67
|
+
NEW_ISSUE_LABELS + base_issue_labels + [found_label]
|
58
68
|
end
|
59
69
|
|
60
70
|
def new_issue_description(test)
|
@@ -0,0 +1,142 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module Report
|
6
|
+
# Uses the API to create GitLab issues for spec run time exceeding Knapsack expectation
|
7
|
+
#
|
8
|
+
# - Takes the expected and actual Knapsack JSON reports from the knapsack output
|
9
|
+
# - Takes a project where issues should be created
|
10
|
+
# - For every test file reported with unexpectedly long run time:
|
11
|
+
# - Find issue by test file name, and if found:
|
12
|
+
# - Reopen issue if it already exists, but is closed
|
13
|
+
# - Update the issue with the new run time data
|
14
|
+
# - If not found:
|
15
|
+
# - Create a new issue with the run time data
|
16
|
+
class KnapsackReportIssue < ReportAsIssue
|
17
|
+
NEW_ISSUE_LABELS = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'knapsack_report']).freeze
|
18
|
+
SEARCH_LABELS = %w[test maintenance::performance knapsack_report].freeze
|
19
|
+
JOB_TIMEOUT_EPIC_URL = 'https://gitlab.com/groups/gitlab-org/quality/engineering-productivity/-/epics/19'
|
20
|
+
|
21
|
+
def initialize(token:, input_files:, expected_report:, project: nil, dry_run: false)
|
22
|
+
super
|
23
|
+
|
24
|
+
@expected_report = expected_report
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
attr_reader :expected_report
|
30
|
+
|
31
|
+
def run!
|
32
|
+
puts "Reporting spec file exceeding Knapsack expectaton issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
33
|
+
|
34
|
+
search_and_create_issue
|
35
|
+
end
|
36
|
+
|
37
|
+
def search_and_create_issue
|
38
|
+
filtered_report = KnapsackReports::SpecRunTimeReport.new(
|
39
|
+
expected_report_path: expected_report_path,
|
40
|
+
actual_report_path: actual_report_path
|
41
|
+
).filtered_report
|
42
|
+
|
43
|
+
puts "=> Reporting #{filtered_report.count} spec files exceeding Knapsack expectation."
|
44
|
+
|
45
|
+
filtered_report.each do |spec_with_run_time|
|
46
|
+
existing_issues = find_issues_for_test(spec_with_run_time, labels: SEARCH_LABELS)
|
47
|
+
|
48
|
+
if existing_issues.empty?
|
49
|
+
puts "Creating issue for #{spec_with_run_time.file}"
|
50
|
+
create_issue(spec_with_run_time)
|
51
|
+
else
|
52
|
+
update_issue(issue: existing_issues.last, spec_run_time: spec_with_run_time)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def expected_report_path
|
58
|
+
return if expected_report.nil? || !File.exist?(expected_report)
|
59
|
+
|
60
|
+
expected_report
|
61
|
+
end
|
62
|
+
|
63
|
+
def actual_report_path
|
64
|
+
return if files.nil? || !File.exist?(files.first)
|
65
|
+
|
66
|
+
files.first
|
67
|
+
end
|
68
|
+
|
69
|
+
def new_issue_labels(_spec_run_time)
|
70
|
+
NEW_ISSUE_LABELS
|
71
|
+
end
|
72
|
+
|
73
|
+
def new_issue_title(spec_run_time)
|
74
|
+
"Job timeout risk: #{spec_run_time.file} ran much longer than expected"
|
75
|
+
end
|
76
|
+
|
77
|
+
def new_issue_description(spec_run_time)
|
78
|
+
<<~MARKDOWN.chomp
|
79
|
+
/epic #{JOB_TIMEOUT_EPIC_URL}
|
80
|
+
|
81
|
+
### Why was this issue created?
|
82
|
+
|
83
|
+
#{spec_run_time.file_link_markdown} was reported to have:
|
84
|
+
|
85
|
+
1. exceeded Knapsack's expected runtime by at least 50%, and
|
86
|
+
2. been identified as a notable pipeline bottleneck and a job timeout risk
|
87
|
+
|
88
|
+
### Suggested steps for investigation
|
89
|
+
|
90
|
+
1. To reproduce in CI by running test files in the same order, you can follow the steps listed [here](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#recreate-job-failure-in-ci-by-forcing-the-job-to-run-the-same-set-of-test-files).
|
91
|
+
1. Identify if a specific test case is stalling the run time. Hint: You can search the job's log for `Starting example group #{spec_run_time.file}` and view the elapsed time after each test case in the proceeding lines starting with `[RSpecRunTime]`.
|
92
|
+
1. If the test file is large, consider refactoring it into multiple files to allow better test parallelization across runners.
|
93
|
+
1. If the run time cannot be fixed in time, consider quarantine the spec(s) to restore performance.
|
94
|
+
|
95
|
+
### Run time details
|
96
|
+
|
97
|
+
#{run_time_detail(spec_run_time)}
|
98
|
+
MARKDOWN
|
99
|
+
end
|
100
|
+
|
101
|
+
def update_issue(issue:, spec_run_time:)
|
102
|
+
state_event = issue.state == 'closed' ? 'reopen' : nil
|
103
|
+
updated_description = <<~MARKDOWN.chomp
|
104
|
+
#{issue.description}
|
105
|
+
|
106
|
+
#{run_time_detail(spec_run_time)}
|
107
|
+
MARKDOWN
|
108
|
+
|
109
|
+
issue_attrs = {
|
110
|
+
description: updated_description
|
111
|
+
}
|
112
|
+
|
113
|
+
issue_attrs[:state_event] = state_event if state_event
|
114
|
+
|
115
|
+
gitlab.edit_issue(iid: issue.iid, options: issue_attrs)
|
116
|
+
puts " => Added a report in #{issue.web_url}!"
|
117
|
+
end
|
118
|
+
|
119
|
+
def run_time_detail(spec_run_time)
|
120
|
+
<<~MARKDOWN.chomp
|
121
|
+
- Reported from pipeline #{spec_run_time.ci_pipeline_url_markdown} created at `#{spec_run_time.ci_pipeline_created_at}`
|
122
|
+
|
123
|
+
| Field | Value |
|
124
|
+
| ------ | ------ |
|
125
|
+
| Job URL| #{spec_run_time.ci_job_link_markdown} |
|
126
|
+
| Job total RSpec suite run time | expected: `#{readable_duration(spec_run_time.expected_suite_duration)}`, actual: `#{readable_duration(spec_run_time.actual_suite_duration)}` |
|
127
|
+
| Spec file run time | expected: `#{readable_duration(spec_run_time.expected)}`, actual: `#{readable_duration(spec_run_time.actual)}` |
|
128
|
+
| Spec file weight | `#{spec_run_time.actual_percentage}%` of total suite run time |
|
129
|
+
MARKDOWN
|
130
|
+
end
|
131
|
+
|
132
|
+
def assert_input_files!(_files)
|
133
|
+
missing_expected_report_msg = "Missing a valid expected Knapsack report."
|
134
|
+
missing_actual_report_msg = "Missing a valid actual Knapsack report."
|
135
|
+
|
136
|
+
abort missing_expected_report_msg if expected_report_path.nil?
|
137
|
+
abort missing_actual_report_msg if actual_report_path.nil?
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
@@ -16,7 +16,7 @@ module GitlabQuality
|
|
16
16
|
# - Find issue by title (with test description or test file), then further filter by stack trace, then pick the better-matching one
|
17
17
|
# - Add the failed job to the issue description, and update labels
|
18
18
|
class RelateFailureIssue < ReportAsIssue
|
19
|
-
include Concerns::FindSetDri
|
19
|
+
include TestTooling::Concerns::FindSetDri
|
20
20
|
include Concerns::GroupAndCategoryLabels
|
21
21
|
include Concerns::IssueReports
|
22
22
|
include Amatch
|
@@ -64,8 +64,6 @@ module GitlabQuality
|
|
64
64
|
puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
|
65
65
|
process_test_results(test_results)
|
66
66
|
end
|
67
|
-
|
68
|
-
write_issues_log_file
|
69
67
|
end
|
70
68
|
|
71
69
|
def test_metric_collections
|
@@ -332,7 +330,7 @@ module GitlabQuality
|
|
332
330
|
def new_issue_assignee_id(test)
|
333
331
|
return unless test.product_group?
|
334
332
|
|
335
|
-
dri = set_dri_via_group(test.product_group, test)
|
333
|
+
dri = set_dri_via_group(test.product_group, test.stage)
|
336
334
|
puts " => Assigning #{dri} as DRI for the issue."
|
337
335
|
|
338
336
|
gitlab.find_user_id(username: dri)
|
@@ -19,7 +19,11 @@ module GitlabQuality
|
|
19
19
|
def invoke!
|
20
20
|
validate_input!
|
21
21
|
|
22
|
-
run!
|
22
|
+
issue_url = run!
|
23
|
+
|
24
|
+
write_issues_log_file
|
25
|
+
|
26
|
+
issue_url
|
23
27
|
end
|
24
28
|
|
25
29
|
private
|
@@ -162,7 +166,12 @@ module GitlabQuality
|
|
162
166
|
def issue_match_test?(issue, test)
|
163
167
|
issue_title = issue.title.strip
|
164
168
|
test_file_path_found = !test.file.to_s.empty? && issue_title.include?(partial_file_path(test.file))
|
165
|
-
|
169
|
+
|
170
|
+
if test.name
|
171
|
+
issue_title.include?(test.name) || test_file_path_found
|
172
|
+
else
|
173
|
+
test_file_path_found
|
174
|
+
end
|
166
175
|
end
|
167
176
|
|
168
177
|
def pipeline_name_label
|
@@ -10,7 +10,7 @@ module GitlabQuality
|
|
10
10
|
# - Find issue by title (with test description or test file)
|
11
11
|
# - Add test metadata, duration to the issue with group and category labels
|
12
12
|
class SlowTestIssue < ReportAsIssue
|
13
|
-
include Concerns::FindSetDri
|
13
|
+
include TestTooling::Concerns::FindSetDri
|
14
14
|
include Concerns::GroupAndCategoryLabels
|
15
15
|
|
16
16
|
NEW_ISSUE_LABELS = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'rspec profiling', 'rspec:slow test']).freeze
|
@@ -33,8 +33,6 @@ module GitlabQuality
|
|
33
33
|
create_slow_issue(test) if test.slow_test?
|
34
34
|
end
|
35
35
|
end
|
36
|
-
|
37
|
-
write_issues_log_file
|
38
36
|
end
|
39
37
|
|
40
38
|
def new_issue_title(test)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab_quality-test_tooling
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.13.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-01-
|
11
|
+
date: 2024-01-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -348,6 +348,7 @@ email:
|
|
348
348
|
executables:
|
349
349
|
- flaky-test-issues
|
350
350
|
- generate-test-session
|
351
|
+
- knapsack-report-issues
|
351
352
|
- post-to-slack
|
352
353
|
- prepare-stage-reports
|
353
354
|
- relate-failure-issue
|
@@ -374,6 +375,7 @@ files:
|
|
374
375
|
- Rakefile
|
375
376
|
- exe/flaky-test-issues
|
376
377
|
- exe/generate-test-session
|
378
|
+
- exe/knapsack-report-issues
|
377
379
|
- exe/post-to-slack
|
378
380
|
- exe/prepare-stage-reports
|
379
381
|
- exe/relate-failure-issue
|
@@ -383,15 +385,22 @@ files:
|
|
383
385
|
- exe/update-screenshot-paths
|
384
386
|
- lefthook.yml
|
385
387
|
- lib/gitlab_quality/test_tooling.rb
|
388
|
+
- lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb
|
386
389
|
- lib/gitlab_quality/test_tooling/failed_jobs_table.rb
|
390
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/branches_client.rb
|
391
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/branches_dry_client.rb
|
392
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/commits_client.rb
|
393
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/commits_dry_client.rb
|
387
394
|
- lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb
|
388
395
|
- lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb
|
389
396
|
- lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb
|
390
397
|
- lib/gitlab_quality/test_tooling/gitlab_client/jobs_client.rb
|
391
398
|
- lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb
|
392
399
|
- lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_dry_client.rb
|
400
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/repository_files_client.rb
|
401
|
+
- lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time.rb
|
402
|
+
- lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time_report.rb
|
393
403
|
- lib/gitlab_quality/test_tooling/labels_inference.rb
|
394
|
-
- lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb
|
395
404
|
- lib/gitlab_quality/test_tooling/report/concerns/group_and_category_labels.rb
|
396
405
|
- lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb
|
397
406
|
- lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
|
@@ -399,6 +408,7 @@ files:
|
|
399
408
|
- lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb
|
400
409
|
- lib/gitlab_quality/test_tooling/report/generate_test_session.rb
|
401
410
|
- lib/gitlab_quality/test_tooling/report/issue_logger.rb
|
411
|
+
- lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb
|
402
412
|
- lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb
|
403
413
|
- lib/gitlab_quality/test_tooling/report/prepare_stage_reports.rb
|
404
414
|
- lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb
|
@@ -411,6 +421,7 @@ files:
|
|
411
421
|
- lib/gitlab_quality/test_tooling/runtime/env.rb
|
412
422
|
- lib/gitlab_quality/test_tooling/runtime/logger.rb
|
413
423
|
- lib/gitlab_quality/test_tooling/slack/post_to_slack.rb
|
424
|
+
- lib/gitlab_quality/test_tooling/slack/post_to_slack_dry.rb
|
414
425
|
- lib/gitlab_quality/test_tooling/summary_table.rb
|
415
426
|
- lib/gitlab_quality/test_tooling/support/http_request.rb
|
416
427
|
- lib/gitlab_quality/test_tooling/system_logs/finders/json_log_finder.rb
|
@@ -443,7 +454,7 @@ metadata:
|
|
443
454
|
homepage_uri: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling
|
444
455
|
source_code_uri: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling
|
445
456
|
changelog_uri: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/releases
|
446
|
-
post_install_message:
|
457
|
+
post_install_message:
|
447
458
|
rdoc_options: []
|
448
459
|
require_paths:
|
449
460
|
- lib
|
@@ -458,8 +469,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
458
469
|
- !ruby/object:Gem::Version
|
459
470
|
version: '0'
|
460
471
|
requirements: []
|
461
|
-
rubygems_version: 3.
|
462
|
-
signing_key:
|
472
|
+
rubygems_version: 3.3.26
|
473
|
+
signing_key:
|
463
474
|
specification_version: 4
|
464
475
|
summary: A collection of test-related tools.
|
465
476
|
test_files: []
|
@@ -1,49 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'json'
|
4
|
-
|
5
|
-
module GitlabQuality
|
6
|
-
module TestTooling
|
7
|
-
module Report
|
8
|
-
module Concerns
|
9
|
-
module FindSetDri
|
10
|
-
def set_dri_via_group(product_group, test)
|
11
|
-
parse_json_with_sets
|
12
|
-
fetch_stage_sets(test)
|
13
|
-
|
14
|
-
return @sets.sample['username'] if @stage_sets.empty?
|
15
|
-
|
16
|
-
fetch_group_sets(product_group)
|
17
|
-
|
18
|
-
if @group_sets.empty?
|
19
|
-
@stage_sets.sample['username']
|
20
|
-
else
|
21
|
-
@group_sets.sample['username']
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
private
|
26
|
-
|
27
|
-
def parse_json_with_sets
|
28
|
-
response = Support::HttpRequest.make_http_request(
|
29
|
-
url: 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json'
|
30
|
-
)
|
31
|
-
@sets = JSON.parse(response.body).select { |user| user['role'].include?('software-engineer-in-test') }
|
32
|
-
end
|
33
|
-
|
34
|
-
def fetch_stage_sets(test)
|
35
|
-
@stage_sets = @sets.select do |user|
|
36
|
-
user['role'].include?(test.stage.split("_").map(&:capitalize).join(" "))
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def fetch_group_sets(product_group)
|
41
|
-
@group_sets = @stage_sets.select do |user|
|
42
|
-
user['role'].downcase.tr(' ', '_').include?(product_group)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|