gitlab_quality-test_tooling 1.11.0 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +19 -2
- data/exe/flaky-test-issues +5 -0
- data/exe/knapsack-report-issues +54 -0
- data/lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb +47 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/branches_client.rb +18 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/branches_dry_client.rb +15 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/commits_client.rb +18 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/commits_dry_client.rb +14 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb +14 -9
- data/lib/gitlab_quality/test_tooling/gitlab_client/repository_files_client.rb +23 -0
- data/lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time.rb +85 -0
- data/lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time_report.rb +60 -0
- data/lib/gitlab_quality/test_tooling/report/concerns/utils.rb +22 -0
- data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb +13 -3
- data/lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb +142 -0
- data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +2 -4
- data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb +11 -2
- data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb +1 -3
- data/lib/gitlab_quality/test_tooling/slack/post_to_slack_dry.rb +14 -0
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +18 -7
- data/lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb +0 -49
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0b5576ca28a8e8cbf76564017a53418fd059747789bbbb8146835285d8f5b37c
|
4
|
+
data.tar.gz: b1d7c6dd581dccabe91f64687b420a83f5561aa407e1e4583e7a5146183890f4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d245b9be5a9a127630d934096277c339e649e9e719727f871d8b794245b385e6f4dbfba3175c546f44b2ee833099ef5398738dcaa631bab39b44eb9734fe338f
|
7
|
+
data.tar.gz: 33a391d891853d45e75198462d504641d090e0f619e31a9fbf5c613b4fe5cef6443d6b465e9e2ffc27d643a3d8d8426030bb10c872a83ca0a2def1b0232ae0ae
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -141,15 +141,32 @@ Usage: exe/slow-test-issues [options]
|
|
141
141
|
-h, --help Show the usage
|
142
142
|
```
|
143
143
|
|
144
|
+
### `exe/knapsack-report-issues`
|
145
|
+
|
146
|
+
```shell
|
147
|
+
Purpose: Create spec run time issue when a spec file almost caused job timeout because it ran significantly longer than what Knapsack expected.
|
148
|
+
Usage: exe/knapsack-report-issues [options]
|
149
|
+
-i, --input-file INPUT_FILE Knapsack actual run time report file path glob
|
150
|
+
-e EXPECTED_REPORT, Knapsack expected report file path
|
151
|
+
--expected-report
|
152
|
+
-p, --project PROJECT Can be an integer or a group/project string
|
153
|
+
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
154
|
+
--dry-run Perform a dry-run (don't create issues)
|
155
|
+
-v, --version Show the version
|
156
|
+
-h, --help Show the usage
|
157
|
+
```
|
158
|
+
|
144
159
|
### `exe/flaky-test-issues`
|
145
160
|
|
146
161
|
```shell
|
147
|
-
Purpose:
|
162
|
+
Purpose: Create flaky test issues for any passed test coming from rspec-retry JSON report files.
|
148
163
|
Usage: exe/flaky-test-issues [options]
|
149
164
|
-i, --input-files INPUT_FILES JSON rspec-retry report files
|
150
165
|
-p, --project PROJECT Can be an integer or a group/project string
|
151
166
|
-m MERGE_REQUEST_IID, An integer merge request IID
|
152
167
|
--merge_request_iid
|
168
|
+
--base-issue-labels BASE_ISSUE_LABELS
|
169
|
+
Comma-separated labels (without tilde) to add to new flaky test issues
|
153
170
|
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
154
171
|
--dry-run Perform a dry-run (don't create issues)
|
155
172
|
-v, --version Show the version
|
@@ -217,7 +234,7 @@ If you forget to set the changelog entry in your commit messages, you can also e
|
|
217
234
|
|
218
235
|
### Steps to release
|
219
236
|
|
220
|
-
Use a `Release` merge request template and create a merge
|
237
|
+
Use a `Release` merge request template and create a merge request to update the version number in `version.rb`, and get the merge request merged by a maintainer.
|
221
238
|
|
222
239
|
This will then be packaged into a gem and pushed to [rubygems.org](https://rubygems.org) by the CI/CD.
|
223
240
|
|
data/exe/flaky-test-issues
CHANGED
@@ -23,6 +23,11 @@ options = OptionParser.new do |opts|
|
|
23
23
|
params[:merge_request_iid] = merge_request_iid
|
24
24
|
end
|
25
25
|
|
26
|
+
opts.on('--base-issue-labels BASE_ISSUE_LABELS', String,
|
27
|
+
'Comma-separated labels (without tilde) to add to new flaky test issues') do |base_issue_labels|
|
28
|
+
params[:base_issue_labels] = base_issue_labels.split(',')
|
29
|
+
end
|
30
|
+
|
26
31
|
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
|
27
32
|
params[:token] = token
|
28
33
|
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "optparse"
|
6
|
+
|
7
|
+
require_relative "../lib/gitlab_quality/test_tooling"
|
8
|
+
|
9
|
+
params = {}
|
10
|
+
|
11
|
+
options = OptionParser.new do |opts|
|
12
|
+
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
|
+
|
14
|
+
opts.on("-i", "--input-file INPUT_FILE", String, "Knapsack actual run time report file path glob") do |input_file|
|
15
|
+
params[:input_files] = input_file
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on("-e", "--expected-report EXPECTED_REPORT", String, "Knapsack expected report file path") do |report_path|
|
19
|
+
params[:expected_report] = report_path.strip
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
|
23
|
+
params[:project] = project
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
|
27
|
+
params[:token] = token
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on('--dry-run', "Perform a dry-run (don't create issues)") do
|
31
|
+
params[:dry_run] = true
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on_tail('-v', '--version', 'Show the version') do
|
35
|
+
require_relative "../lib/gitlab_quality/test_tooling/version"
|
36
|
+
puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
|
37
|
+
exit
|
38
|
+
end
|
39
|
+
|
40
|
+
opts.on_tail('-h', '--help', 'Show the usage') do
|
41
|
+
puts "Purpose: Create spec run time issue when a spec file almost caused job timeout because it ran significantly longer than what Knapsack expected."
|
42
|
+
puts opts
|
43
|
+
exit
|
44
|
+
end
|
45
|
+
|
46
|
+
opts.parse(ARGV)
|
47
|
+
end
|
48
|
+
|
49
|
+
if params.any?
|
50
|
+
GitlabQuality::TestTooling::Report::KnapsackReportIssue.new(**params).invoke!
|
51
|
+
else
|
52
|
+
puts options
|
53
|
+
exit 1
|
54
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module GitlabQuality
|
6
|
+
module TestTooling
|
7
|
+
module Concerns
|
8
|
+
module FindSetDri
|
9
|
+
def set_dri_via_group(product_group, stage)
|
10
|
+
parse_json_with_sets
|
11
|
+
fetch_stage_sets(stage)
|
12
|
+
|
13
|
+
return @sets.sample['username'] if @stage_sets.empty?
|
14
|
+
|
15
|
+
fetch_group_sets(product_group)
|
16
|
+
|
17
|
+
if @group_sets.empty?
|
18
|
+
@stage_sets.sample['username']
|
19
|
+
else
|
20
|
+
@group_sets.sample['username']
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def parse_json_with_sets
|
27
|
+
response = Support::HttpRequest.make_http_request(
|
28
|
+
url: 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json'
|
29
|
+
)
|
30
|
+
@sets = JSON.parse(response.body).select { |user| user['role'].include?('software-engineer-in-test') }
|
31
|
+
end
|
32
|
+
|
33
|
+
def fetch_stage_sets(stage)
|
34
|
+
@stage_sets = @sets.select do |user|
|
35
|
+
user['role'].include?(stage.split("_").map(&:capitalize).join(" "))
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def fetch_group_sets(product_group)
|
40
|
+
@group_sets = @stage_sets.select do |user|
|
41
|
+
user['role'].downcase.tr(' ', '_').include?(product_group)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class BranchesClient < GitlabClient
|
7
|
+
def create(branch_name, ref)
|
8
|
+
branch = handle_gitlab_client_exceptions do
|
9
|
+
client.create_branch(project, branch_name, ref)
|
10
|
+
end
|
11
|
+
|
12
|
+
Runtime::Logger.debug("Created branch #{branch['name']} (#{branch['web_url']})")
|
13
|
+
branch
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class BranchesDryClient < BranchesClient
|
7
|
+
def create(branch_name, ref)
|
8
|
+
branch = { 'name' => branch_name, 'web_url' => 'https://example.com/dummy/branch/url' }
|
9
|
+
puts "A branch would have been created with name: #{branch['name']}, web_url: #{branch['web_url']} and ref: #{ref}"
|
10
|
+
branch
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class CommitsClient < GitlabClient
|
7
|
+
def create(branch_name, file_path, new_content, message)
|
8
|
+
commit = client.create_commit(project, branch_name, message, [
|
9
|
+
{ action: :update, file_path: file_path, content: new_content }
|
10
|
+
])
|
11
|
+
|
12
|
+
Runtime::Logger.debug("Created commit #{commit['id']} (#{commit['web_url']}) on #{branch_name}")
|
13
|
+
commit
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class CommitsDryClient < CommitsClient
|
7
|
+
def create(branch_name, file_path, new_content, message)
|
8
|
+
puts "A commit would have been created on branch_name: #{branch_name}, file_path: #{file_path}, message: #{message} and content:"
|
9
|
+
puts new_content
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -1,7 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'gitlab'
|
4
|
-
|
5
3
|
module GitlabQuality
|
6
4
|
module TestTooling
|
7
5
|
module GitlabClient
|
@@ -10,19 +8,26 @@ module GitlabQuality
|
|
10
8
|
client.merge_request_changes(project, merge_request_iid)
|
11
9
|
end
|
12
10
|
|
13
|
-
def create_merge_request(title:, source_branch:, target_branch:, description:, labels:)
|
11
|
+
def create_merge_request(title:, source_branch:, target_branch:, description:, labels:, assignee_id: nil)
|
12
|
+
attrs = {
|
13
|
+
source_branch: source_branch,
|
14
|
+
target_branch: target_branch,
|
15
|
+
description: description,
|
16
|
+
labels: labels,
|
17
|
+
assignee_id: assignee_id,
|
18
|
+
squash: true,
|
19
|
+
remove_source_branch: true
|
20
|
+
}.compact
|
21
|
+
|
14
22
|
merge_request = handle_gitlab_client_exceptions do
|
15
23
|
client.create_merge_request(project,
|
16
24
|
title,
|
17
|
-
|
18
|
-
target_branch: target_branch,
|
19
|
-
description: description,
|
20
|
-
labels: labels,
|
21
|
-
squash: true,
|
22
|
-
remove_source_branch: true)
|
25
|
+
attrs)
|
23
26
|
end
|
24
27
|
|
25
28
|
Runtime::Logger.debug("Created merge request #{merge_request['iid']} (#{merge_request['web_url']})") if merge_request
|
29
|
+
|
30
|
+
merge_request
|
26
31
|
end
|
27
32
|
|
28
33
|
def find(iid: nil, options: {}, &select)
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module GitlabClient
|
6
|
+
class RepositoryFilesClient < GitlabClient
|
7
|
+
attr_reader :file_path
|
8
|
+
|
9
|
+
def initialize(file_path:, **kwargs)
|
10
|
+
@file_path = file_path
|
11
|
+
|
12
|
+
super
|
13
|
+
end
|
14
|
+
|
15
|
+
def file_contents
|
16
|
+
handle_gitlab_client_exceptions do
|
17
|
+
client.file_contents(project, file_path.gsub(%r{^/}, ""))
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module KnapsackReports
|
6
|
+
class SpecRunTime
|
7
|
+
attr_reader :file, :expected, :actual, :expected_suite_duration, :actual_suite_duration
|
8
|
+
|
9
|
+
ACTUAL_TO_EXPECTED_SPEC_RUN_TIME_RATIO_THRESHOLD = 1.5 # actual run time is longer than expected by 50% +
|
10
|
+
SPEC_WEIGHT_PERCENTAGE_TRESHOLD = 15 # a spec file takes 15%+ of the total test suite run time
|
11
|
+
SUITE_DURATION_THRESHOLD = 70 * 60 # if test suite takes more than 70 minutes, job risks timing out
|
12
|
+
|
13
|
+
def initialize(file:, expected:, actual:, expected_suite_duration:, actual_suite_duration:)
|
14
|
+
@file = file
|
15
|
+
@expected = expected.to_f
|
16
|
+
@actual = actual.to_f
|
17
|
+
@expected_suite_duration = expected_suite_duration.to_f
|
18
|
+
@actual_suite_duration = actual_suite_duration.to_f
|
19
|
+
end
|
20
|
+
|
21
|
+
def should_report?
|
22
|
+
# guideline proposed in https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/354
|
23
|
+
exceed_actual_to_expected_ratio_threshold? && test_suite_bottleneck?
|
24
|
+
end
|
25
|
+
|
26
|
+
def ci_pipeline_url_markdown
|
27
|
+
"[#{ci_pipeline_id}](#{ci_pipeline_url})"
|
28
|
+
end
|
29
|
+
|
30
|
+
def ci_pipeline_created_at
|
31
|
+
ENV.fetch('CI_PIPELINE_CREATED_AT', nil)
|
32
|
+
end
|
33
|
+
|
34
|
+
def ci_job_link_markdown
|
35
|
+
"[#{ci_job_name}](#{ci_job_url})"
|
36
|
+
end
|
37
|
+
|
38
|
+
def file_link_markdown
|
39
|
+
"[#{file}](#{file_link})"
|
40
|
+
end
|
41
|
+
|
42
|
+
def actual_percentage
|
43
|
+
(actual / actual_suite_duration * 100).round(2)
|
44
|
+
end
|
45
|
+
|
46
|
+
def name
|
47
|
+
nil
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def exceed_actual_to_expected_ratio_threshold?
|
53
|
+
actual / expected >= ACTUAL_TO_EXPECTED_SPEC_RUN_TIME_RATIO_THRESHOLD
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_suite_bottleneck?
|
57
|
+
# now we only report bottlenecks when they risk causing job timeouts
|
58
|
+
return unless actual_suite_duration > SUITE_DURATION_THRESHOLD
|
59
|
+
|
60
|
+
actual_percentage > SPEC_WEIGHT_PERCENTAGE_TRESHOLD
|
61
|
+
end
|
62
|
+
|
63
|
+
def ci_job_url
|
64
|
+
ENV.fetch('CI_JOB_URL', nil)
|
65
|
+
end
|
66
|
+
|
67
|
+
def ci_job_name
|
68
|
+
ENV.fetch('CI_JOB_NAME_SLUG', nil)
|
69
|
+
end
|
70
|
+
|
71
|
+
def ci_pipeline_id
|
72
|
+
ENV.fetch('CI_PIPELINE_IID', nil)
|
73
|
+
end
|
74
|
+
|
75
|
+
def ci_pipeline_url
|
76
|
+
ENV.fetch('CI_PIPELINE_URL', nil)
|
77
|
+
end
|
78
|
+
|
79
|
+
def file_link
|
80
|
+
"#{Runtime::Env.file_base_url}#{file}"
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module GitlabQuality
|
6
|
+
module TestTooling
|
7
|
+
module KnapsackReports
|
8
|
+
class SpecRunTimeReport
|
9
|
+
attr_reader :expected_report, :actual_report
|
10
|
+
|
11
|
+
def initialize(expected_report_path:, actual_report_path:)
|
12
|
+
@expected_report = parse(expected_report_path)
|
13
|
+
@actual_report = parse(actual_report_path)
|
14
|
+
end
|
15
|
+
|
16
|
+
def filtered_report
|
17
|
+
@filtered_report = actual_report.keys.filter_map do |spec_file|
|
18
|
+
expected_run_time = expected_report[spec_file]
|
19
|
+
actual_run_time = actual_report[spec_file]
|
20
|
+
|
21
|
+
if expected_run_time.nil?
|
22
|
+
puts "#{spec_file} missing from the expected Knapsack report, skipping."
|
23
|
+
next
|
24
|
+
end
|
25
|
+
|
26
|
+
spec_run_time = SpecRunTime.new(
|
27
|
+
file: spec_file,
|
28
|
+
expected: expected_run_time,
|
29
|
+
actual: actual_run_time,
|
30
|
+
expected_suite_duration: expected_test_suite_run_time_total,
|
31
|
+
actual_suite_duration: actual_test_suite_run_time_total
|
32
|
+
)
|
33
|
+
|
34
|
+
spec_run_time if spec_run_time.should_report?
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def parse(report_path)
|
41
|
+
JSON.parse(File.read(report_path))
|
42
|
+
end
|
43
|
+
|
44
|
+
def expected_test_suite_run_time_total
|
45
|
+
@expected_test_suite_run_time_total ||=
|
46
|
+
expected_report.reduce(0) do |total_run_time, (_spec_file, run_time)|
|
47
|
+
total_run_time + run_time
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def actual_test_suite_run_time_total
|
52
|
+
@actual_test_suite_run_time_total ||=
|
53
|
+
actual_report.reduce(0) do |total_run_time, (_spec_file, run_time)|
|
54
|
+
total_run_time + run_time
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -45,6 +45,28 @@ module GitlabQuality
|
|
45
45
|
|
46
46
|
@pipeline ||= Runtime::Env.pipeline_from_project_name
|
47
47
|
end
|
48
|
+
|
49
|
+
def readable_duration(duration_in_seconds)
|
50
|
+
minutes = (duration_in_seconds / 60).to_i
|
51
|
+
seconds = (duration_in_seconds % 60).round(2)
|
52
|
+
|
53
|
+
min_output = normalize_duration_output(minutes, 'minute')
|
54
|
+
sec_output = normalize_duration_output(seconds, 'second')
|
55
|
+
|
56
|
+
"#{min_output} #{sec_output}".strip
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
def normalize_duration_output(number, unit)
|
62
|
+
if number <= 0
|
63
|
+
""
|
64
|
+
elsif number <= 1
|
65
|
+
"#{number} #{unit}"
|
66
|
+
else
|
67
|
+
"#{number} #{unit}s"
|
68
|
+
end
|
69
|
+
end
|
48
70
|
end
|
49
71
|
end
|
50
72
|
end
|
@@ -19,14 +19,24 @@ module GitlabQuality
|
|
19
19
|
FOUND_IN_MR_LABEL = 'found:in MR'
|
20
20
|
FOUND_IN_MASTER_LABEL = 'found:master'
|
21
21
|
|
22
|
-
def initialize(
|
22
|
+
def initialize(
|
23
|
+
token:,
|
24
|
+
input_files:,
|
25
|
+
base_issue_labels: nil,
|
26
|
+
confidential: false,
|
27
|
+
dry_run: false,
|
28
|
+
merge_request_iid: nil,
|
29
|
+
project: nil,
|
30
|
+
**_kwargs)
|
23
31
|
super(token: token, input_files: input_files, project: project, confidential: confidential, dry_run: dry_run)
|
32
|
+
|
33
|
+
@base_issue_labels = Set.new(base_issue_labels)
|
24
34
|
@merge_request_iid = merge_request_iid
|
25
35
|
end
|
26
36
|
|
27
37
|
private
|
28
38
|
|
29
|
-
attr_reader :merge_request_iid
|
39
|
+
attr_reader :base_issue_labels, :merge_request_iid
|
30
40
|
|
31
41
|
def run!
|
32
42
|
puts "Reporting flaky tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
@@ -54,7 +64,7 @@ module GitlabQuality
|
|
54
64
|
FOUND_IN_MR_LABEL
|
55
65
|
end
|
56
66
|
|
57
|
-
NEW_ISSUE_LABELS + [found_label]
|
67
|
+
NEW_ISSUE_LABELS + base_issue_labels + [found_label]
|
58
68
|
end
|
59
69
|
|
60
70
|
def new_issue_description(test)
|
@@ -0,0 +1,142 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module Report
|
6
|
+
# Uses the API to create GitLab issues for spec run time exceeding Knapsack expectation
|
7
|
+
#
|
8
|
+
# - Takes the expected and actual Knapsack JSON reports from the knapsack output
|
9
|
+
# - Takes a project where issues should be created
|
10
|
+
# - For every test file reported with unexpectedly long run time:
|
11
|
+
# - Find issue by test file name, and if found:
|
12
|
+
# - Reopen issue if it already exists, but is closed
|
13
|
+
# - Update the issue with the new run time data
|
14
|
+
# - If not found:
|
15
|
+
# - Create a new issue with the run time data
|
16
|
+
class KnapsackReportIssue < ReportAsIssue
|
17
|
+
NEW_ISSUE_LABELS = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'knapsack_report']).freeze
|
18
|
+
SEARCH_LABELS = %w[test maintenance::performance knapsack_report].freeze
|
19
|
+
JOB_TIMEOUT_EPIC_URL = 'https://gitlab.com/groups/gitlab-org/quality/engineering-productivity/-/epics/19'
|
20
|
+
|
21
|
+
def initialize(token:, input_files:, expected_report:, project: nil, dry_run: false)
|
22
|
+
super
|
23
|
+
|
24
|
+
@expected_report = expected_report
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
attr_reader :expected_report
|
30
|
+
|
31
|
+
def run!
|
32
|
+
puts "Reporting spec file exceeding Knapsack expectaton issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
33
|
+
|
34
|
+
search_and_create_issue
|
35
|
+
end
|
36
|
+
|
37
|
+
def search_and_create_issue
|
38
|
+
filtered_report = KnapsackReports::SpecRunTimeReport.new(
|
39
|
+
expected_report_path: expected_report_path,
|
40
|
+
actual_report_path: actual_report_path
|
41
|
+
).filtered_report
|
42
|
+
|
43
|
+
puts "=> Reporting #{filtered_report.count} spec files exceeding Knapsack expectation."
|
44
|
+
|
45
|
+
filtered_report.each do |spec_with_run_time|
|
46
|
+
existing_issues = find_issues_for_test(spec_with_run_time, labels: SEARCH_LABELS)
|
47
|
+
|
48
|
+
if existing_issues.empty?
|
49
|
+
puts "Creating issue for #{spec_with_run_time.file}"
|
50
|
+
create_issue(spec_with_run_time)
|
51
|
+
else
|
52
|
+
update_issue(issue: existing_issues.last, spec_run_time: spec_with_run_time)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def expected_report_path
|
58
|
+
return if expected_report.nil? || !File.exist?(expected_report)
|
59
|
+
|
60
|
+
expected_report
|
61
|
+
end
|
62
|
+
|
63
|
+
def actual_report_path
|
64
|
+
return if files.nil? || !File.exist?(files.first)
|
65
|
+
|
66
|
+
files.first
|
67
|
+
end
|
68
|
+
|
69
|
+
def new_issue_labels(_spec_run_time)
|
70
|
+
NEW_ISSUE_LABELS
|
71
|
+
end
|
72
|
+
|
73
|
+
def new_issue_title(spec_run_time)
|
74
|
+
"Job timeout risk: #{spec_run_time.file} ran much longer than expected"
|
75
|
+
end
|
76
|
+
|
77
|
+
def new_issue_description(spec_run_time)
|
78
|
+
<<~MARKDOWN.chomp
|
79
|
+
/epic #{JOB_TIMEOUT_EPIC_URL}
|
80
|
+
|
81
|
+
### Why was this issue created?
|
82
|
+
|
83
|
+
#{spec_run_time.file_link_markdown} was reported to have:
|
84
|
+
|
85
|
+
1. exceeded Knapsack's expected runtime by at least 50%, and
|
86
|
+
2. been identified as a notable pipeline bottleneck and a job timeout risk
|
87
|
+
|
88
|
+
### Suggested steps for investigation
|
89
|
+
|
90
|
+
1. To reproduce in CI by running test files in the same order, you can follow the steps listed [here](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#recreate-job-failure-in-ci-by-forcing-the-job-to-run-the-same-set-of-test-files).
|
91
|
+
1. Identify if a specific test case is stalling the run time. Hint: You can search the job's log for `Starting example group #{spec_run_time.file}` and view the elapsed time after each test case in the proceeding lines starting with `[RSpecRunTime]`.
|
92
|
+
1. If the test file is large, consider refactoring it into multiple files to allow better test parallelization across runners.
|
93
|
+
1. If the run time cannot be fixed in time, consider quarantine the spec(s) to restore performance.
|
94
|
+
|
95
|
+
### Run time details
|
96
|
+
|
97
|
+
#{run_time_detail(spec_run_time)}
|
98
|
+
MARKDOWN
|
99
|
+
end
|
100
|
+
|
101
|
+
def update_issue(issue:, spec_run_time:)
|
102
|
+
state_event = issue.state == 'closed' ? 'reopen' : nil
|
103
|
+
updated_description = <<~MARKDOWN.chomp
|
104
|
+
#{issue.description}
|
105
|
+
|
106
|
+
#{run_time_detail(spec_run_time)}
|
107
|
+
MARKDOWN
|
108
|
+
|
109
|
+
issue_attrs = {
|
110
|
+
description: updated_description
|
111
|
+
}
|
112
|
+
|
113
|
+
issue_attrs[:state_event] = state_event if state_event
|
114
|
+
|
115
|
+
gitlab.edit_issue(iid: issue.iid, options: issue_attrs)
|
116
|
+
puts " => Added a report in #{issue.web_url}!"
|
117
|
+
end
|
118
|
+
|
119
|
+
def run_time_detail(spec_run_time)
|
120
|
+
<<~MARKDOWN.chomp
|
121
|
+
- Reported from pipeline #{spec_run_time.ci_pipeline_url_markdown} created at `#{spec_run_time.ci_pipeline_created_at}`
|
122
|
+
|
123
|
+
| Field | Value |
|
124
|
+
| ------ | ------ |
|
125
|
+
| Job URL| #{spec_run_time.ci_job_link_markdown} |
|
126
|
+
| Job total RSpec suite run time | expected: `#{readable_duration(spec_run_time.expected_suite_duration)}`, actual: `#{readable_duration(spec_run_time.actual_suite_duration)}` |
|
127
|
+
| Spec file run time | expected: `#{readable_duration(spec_run_time.expected)}`, actual: `#{readable_duration(spec_run_time.actual)}` |
|
128
|
+
| Spec file weight | `#{spec_run_time.actual_percentage}%` of total suite run time |
|
129
|
+
MARKDOWN
|
130
|
+
end
|
131
|
+
|
132
|
+
def assert_input_files!(_files)
|
133
|
+
missing_expected_report_msg = "Missing a valid expected Knapsack report."
|
134
|
+
missing_actual_report_msg = "Missing a valid actual Knapsack report."
|
135
|
+
|
136
|
+
abort missing_expected_report_msg if expected_report_path.nil?
|
137
|
+
abort missing_actual_report_msg if actual_report_path.nil?
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
@@ -16,7 +16,7 @@ module GitlabQuality
|
|
16
16
|
# - Find issue by title (with test description or test file), then further filter by stack trace, then pick the better-matching one
|
17
17
|
# - Add the failed job to the issue description, and update labels
|
18
18
|
class RelateFailureIssue < ReportAsIssue
|
19
|
-
include Concerns::FindSetDri
|
19
|
+
include TestTooling::Concerns::FindSetDri
|
20
20
|
include Concerns::GroupAndCategoryLabels
|
21
21
|
include Concerns::IssueReports
|
22
22
|
include Amatch
|
@@ -64,8 +64,6 @@ module GitlabQuality
|
|
64
64
|
puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
|
65
65
|
process_test_results(test_results)
|
66
66
|
end
|
67
|
-
|
68
|
-
write_issues_log_file
|
69
67
|
end
|
70
68
|
|
71
69
|
def test_metric_collections
|
@@ -332,7 +330,7 @@ module GitlabQuality
|
|
332
330
|
def new_issue_assignee_id(test)
|
333
331
|
return unless test.product_group?
|
334
332
|
|
335
|
-
dri = set_dri_via_group(test.product_group, test)
|
333
|
+
dri = set_dri_via_group(test.product_group, test.stage)
|
336
334
|
puts " => Assigning #{dri} as DRI for the issue."
|
337
335
|
|
338
336
|
gitlab.find_user_id(username: dri)
|
@@ -19,7 +19,11 @@ module GitlabQuality
|
|
19
19
|
def invoke!
|
20
20
|
validate_input!
|
21
21
|
|
22
|
-
run!
|
22
|
+
issue_url = run!
|
23
|
+
|
24
|
+
write_issues_log_file
|
25
|
+
|
26
|
+
issue_url
|
23
27
|
end
|
24
28
|
|
25
29
|
private
|
@@ -162,7 +166,12 @@ module GitlabQuality
|
|
162
166
|
def issue_match_test?(issue, test)
|
163
167
|
issue_title = issue.title.strip
|
164
168
|
test_file_path_found = !test.file.to_s.empty? && issue_title.include?(partial_file_path(test.file))
|
165
|
-
|
169
|
+
|
170
|
+
if test.name
|
171
|
+
issue_title.include?(test.name) || test_file_path_found
|
172
|
+
else
|
173
|
+
test_file_path_found
|
174
|
+
end
|
166
175
|
end
|
167
176
|
|
168
177
|
def pipeline_name_label
|
@@ -10,7 +10,7 @@ module GitlabQuality
|
|
10
10
|
# - Find issue by title (with test description or test file)
|
11
11
|
# - Add test metadata, duration to the issue with group and category labels
|
12
12
|
class SlowTestIssue < ReportAsIssue
|
13
|
-
include Concerns::FindSetDri
|
13
|
+
include TestTooling::Concerns::FindSetDri
|
14
14
|
include Concerns::GroupAndCategoryLabels
|
15
15
|
|
16
16
|
NEW_ISSUE_LABELS = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'rspec profiling', 'rspec:slow test']).freeze
|
@@ -33,8 +33,6 @@ module GitlabQuality
|
|
33
33
|
create_slow_issue(test) if test.slow_test?
|
34
34
|
end
|
35
35
|
end
|
36
|
-
|
37
|
-
write_issues_log_file
|
38
36
|
end
|
39
37
|
|
40
38
|
def new_issue_title(test)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab_quality-test_tooling
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.13.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-01-
|
11
|
+
date: 2024-01-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -348,6 +348,7 @@ email:
|
|
348
348
|
executables:
|
349
349
|
- flaky-test-issues
|
350
350
|
- generate-test-session
|
351
|
+
- knapsack-report-issues
|
351
352
|
- post-to-slack
|
352
353
|
- prepare-stage-reports
|
353
354
|
- relate-failure-issue
|
@@ -374,6 +375,7 @@ files:
|
|
374
375
|
- Rakefile
|
375
376
|
- exe/flaky-test-issues
|
376
377
|
- exe/generate-test-session
|
378
|
+
- exe/knapsack-report-issues
|
377
379
|
- exe/post-to-slack
|
378
380
|
- exe/prepare-stage-reports
|
379
381
|
- exe/relate-failure-issue
|
@@ -383,15 +385,22 @@ files:
|
|
383
385
|
- exe/update-screenshot-paths
|
384
386
|
- lefthook.yml
|
385
387
|
- lib/gitlab_quality/test_tooling.rb
|
388
|
+
- lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb
|
386
389
|
- lib/gitlab_quality/test_tooling/failed_jobs_table.rb
|
390
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/branches_client.rb
|
391
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/branches_dry_client.rb
|
392
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/commits_client.rb
|
393
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/commits_dry_client.rb
|
387
394
|
- lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb
|
388
395
|
- lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb
|
389
396
|
- lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb
|
390
397
|
- lib/gitlab_quality/test_tooling/gitlab_client/jobs_client.rb
|
391
398
|
- lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb
|
392
399
|
- lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_dry_client.rb
|
400
|
+
- lib/gitlab_quality/test_tooling/gitlab_client/repository_files_client.rb
|
401
|
+
- lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time.rb
|
402
|
+
- lib/gitlab_quality/test_tooling/knapsack_reports/spec_run_time_report.rb
|
393
403
|
- lib/gitlab_quality/test_tooling/labels_inference.rb
|
394
|
-
- lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb
|
395
404
|
- lib/gitlab_quality/test_tooling/report/concerns/group_and_category_labels.rb
|
396
405
|
- lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb
|
397
406
|
- lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
|
@@ -399,6 +408,7 @@ files:
|
|
399
408
|
- lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb
|
400
409
|
- lib/gitlab_quality/test_tooling/report/generate_test_session.rb
|
401
410
|
- lib/gitlab_quality/test_tooling/report/issue_logger.rb
|
411
|
+
- lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb
|
402
412
|
- lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb
|
403
413
|
- lib/gitlab_quality/test_tooling/report/prepare_stage_reports.rb
|
404
414
|
- lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb
|
@@ -411,6 +421,7 @@ files:
|
|
411
421
|
- lib/gitlab_quality/test_tooling/runtime/env.rb
|
412
422
|
- lib/gitlab_quality/test_tooling/runtime/logger.rb
|
413
423
|
- lib/gitlab_quality/test_tooling/slack/post_to_slack.rb
|
424
|
+
- lib/gitlab_quality/test_tooling/slack/post_to_slack_dry.rb
|
414
425
|
- lib/gitlab_quality/test_tooling/summary_table.rb
|
415
426
|
- lib/gitlab_quality/test_tooling/support/http_request.rb
|
416
427
|
- lib/gitlab_quality/test_tooling/system_logs/finders/json_log_finder.rb
|
@@ -443,7 +454,7 @@ metadata:
|
|
443
454
|
homepage_uri: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling
|
444
455
|
source_code_uri: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling
|
445
456
|
changelog_uri: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/releases
|
446
|
-
post_install_message:
|
457
|
+
post_install_message:
|
447
458
|
rdoc_options: []
|
448
459
|
require_paths:
|
449
460
|
- lib
|
@@ -458,8 +469,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
458
469
|
- !ruby/object:Gem::Version
|
459
470
|
version: '0'
|
460
471
|
requirements: []
|
461
|
-
rubygems_version: 3.
|
462
|
-
signing_key:
|
472
|
+
rubygems_version: 3.3.26
|
473
|
+
signing_key:
|
463
474
|
specification_version: 4
|
464
475
|
summary: A collection of test-related tools.
|
465
476
|
test_files: []
|
@@ -1,49 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'json'
|
4
|
-
|
5
|
-
module GitlabQuality
|
6
|
-
module TestTooling
|
7
|
-
module Report
|
8
|
-
module Concerns
|
9
|
-
module FindSetDri
|
10
|
-
def set_dri_via_group(product_group, test)
|
11
|
-
parse_json_with_sets
|
12
|
-
fetch_stage_sets(test)
|
13
|
-
|
14
|
-
return @sets.sample['username'] if @stage_sets.empty?
|
15
|
-
|
16
|
-
fetch_group_sets(product_group)
|
17
|
-
|
18
|
-
if @group_sets.empty?
|
19
|
-
@stage_sets.sample['username']
|
20
|
-
else
|
21
|
-
@group_sets.sample['username']
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
private
|
26
|
-
|
27
|
-
def parse_json_with_sets
|
28
|
-
response = Support::HttpRequest.make_http_request(
|
29
|
-
url: 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json'
|
30
|
-
)
|
31
|
-
@sets = JSON.parse(response.body).select { |user| user['role'].include?('software-engineer-in-test') }
|
32
|
-
end
|
33
|
-
|
34
|
-
def fetch_stage_sets(test)
|
35
|
-
@stage_sets = @sets.select do |user|
|
36
|
-
user['role'].include?(test.stage.split("_").map(&:capitalize).join(" "))
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def fetch_group_sets(product_group)
|
41
|
-
@group_sets = @stage_sets.select do |user|
|
42
|
-
user['role'].downcase.tr(' ', '_').include?(product_group)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|