gitlab_quality-test_tooling 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.tool-versions +1 -0
- data/Gemfile.lock +4 -1
- data/README.md +14 -0
- data/exe/existing-test-health-issue +59 -0
- data/lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb +38 -11
- data/lib/gitlab_quality/test_tooling/report/test_health_issue_finder.rb +79 -0
- data/lib/gitlab_quality/test_tooling/test_results/json_test_results.rb +5 -0
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +19 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7437a045ff423ca3c6585be2aa13b11d97fd94abb5d8ffd95a14d11f48970b68
|
4
|
+
data.tar.gz: fd91b826d48c661a7b05f196857ec95c7fc59d478e8718ffa9ca5d534f38ffe5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4de31f3a187af7c92df5b015c631e5fa1278fa6a3d62f8111294ce07c245236b9e30e78a6760859eee4d024ee0de62dd544201e2d3e4beec73ef7be00f443def
|
7
|
+
data.tar.gz: 1d44821f07f2d61ac2d1b31b0837149bcf015070ffa1c4e6791d4685c3be322ab0991a28bf1551130eaf274045d58855a405b680948673f78aa208c9a34cab77
|
data/.tool-versions
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
gitlab_quality-test_tooling (2.
|
4
|
+
gitlab_quality-test_tooling (2.2.0)
|
5
5
|
activesupport (>= 7.0, < 7.2)
|
6
6
|
amatch (~> 0.4.1)
|
7
7
|
fog-google (~> 1.24, >= 1.24.1)
|
@@ -302,6 +302,8 @@ GEM
|
|
302
302
|
binding_of_caller
|
303
303
|
rspec-parameterized-core (< 2)
|
304
304
|
rspec-support (3.13.1)
|
305
|
+
rspec_junit_formatter (0.6.0)
|
306
|
+
rspec-core (>= 2, < 4, != 2.12.0)
|
305
307
|
rubocop (1.62.1)
|
306
308
|
json (~> 2.3)
|
307
309
|
language_server-protocol (>= 3.17.0)
|
@@ -407,6 +409,7 @@ DEPENDENCIES
|
|
407
409
|
pry-byebug (= 3.10.1)
|
408
410
|
rake (~> 13.0)
|
409
411
|
rspec (~> 3.12)
|
412
|
+
rspec_junit_formatter (~> 0.6.0)
|
410
413
|
simplecov (~> 0.22)
|
411
414
|
simplecov-cobertura (~> 2.1)
|
412
415
|
solargraph (~> 0.41)
|
data/README.md
CHANGED
@@ -177,6 +177,20 @@ Usage: exe/failed-test-issues [options]
|
|
177
177
|
-h, --help Show the usage
|
178
178
|
```
|
179
179
|
|
180
|
+
### `exe/existing-test-health-issue`
|
181
|
+
|
182
|
+
```shell
|
183
|
+
Purpose: Checks whether tests coming from the rspec JSON report files has an existing test health issue opened.
|
184
|
+
Usage: exe/existing-test-health-issue [options]
|
185
|
+
-i, --input-files INPUT_FILES JSON rspec-retry report files
|
186
|
+
-p, --project PROJECT Can be an integer or a group/project string
|
187
|
+
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
188
|
+
--health-problem-type PROBLEM_TYPE
|
189
|
+
Look for the given health problem type (failures, pass-after-retry, slow)
|
190
|
+
-v, --version Show the version
|
191
|
+
-h, --help Show the usage
|
192
|
+
```
|
193
|
+
|
180
194
|
### `exe/flaky-test-issues`
|
181
195
|
|
182
196
|
```shell
|
@@ -0,0 +1,59 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "optparse"
|
6
|
+
|
7
|
+
require_relative "../lib/gitlab_quality/test_tooling"
|
8
|
+
|
9
|
+
params = {}
|
10
|
+
HEALTH_PROBLEM_TYPES = GitlabQuality::TestTooling::Report::TestHealthIssueFinder::HEALTH_PROBLEM_TYPE_TO_LABEL.keys
|
11
|
+
|
12
|
+
options = OptionParser.new do |opts|
|
13
|
+
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
14
|
+
|
15
|
+
opts.on('-i', '--input-files INPUT_FILES', String, 'JSON rspec-retry report files') do |input_files|
|
16
|
+
params[:input_files] = input_files
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
|
20
|
+
params[:project] = project
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
|
24
|
+
params[:token] = token
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on("--health-problem-type PROBLEM_TYPE", String, "Look for the given health problem type (#{HEALTH_PROBLEM_TYPES.join(', ')})") do |value|
|
28
|
+
raise ArgumentError, "Invalid health problem type: #{value}. Valid options are: #{HEALTH_PROBLEM_TYPES.join(', ')}" unless HEALTH_PROBLEM_TYPES.include?(value)
|
29
|
+
|
30
|
+
params[:health_problem_type] = value
|
31
|
+
end
|
32
|
+
|
33
|
+
opts.on_tail('-v', '--version', 'Show the version') do
|
34
|
+
require_relative "../lib/gitlab_quality/test_tooling/version"
|
35
|
+
puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
|
36
|
+
exit
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on_tail('-h', '--help', 'Show the usage') do
|
40
|
+
puts "Purpose: Checks whether tests coming from the rspec JSON report files has an existing test health issue opened."
|
41
|
+
puts opts
|
42
|
+
exit
|
43
|
+
end
|
44
|
+
|
45
|
+
opts.parse(ARGV)
|
46
|
+
end
|
47
|
+
|
48
|
+
if params.any?
|
49
|
+
raise ArgumentError, "No health problem type given. Valid options are: #{HEALTH_PROBLEM_TYPES.join(', ')}" unless params.key?(:health_problem_type)
|
50
|
+
|
51
|
+
if GitlabQuality::TestTooling::Report::TestHealthIssueFinder.new(**params).found_existing_unhealthy_test_issue?
|
52
|
+
exit 0
|
53
|
+
else
|
54
|
+
exit 1
|
55
|
+
end
|
56
|
+
else
|
57
|
+
puts options
|
58
|
+
exit 1
|
59
|
+
end
|
@@ -11,13 +11,15 @@ module GitlabQuality
|
|
11
11
|
FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
|
12
12
|
REPORT_ITEM_REGEX = /^1\. (?<report_date>\d{4}-\d{2}-\d{2}): #{JOB_URL_REGEX} \((?<pipeline_url>\S+)\) ?(?<extra_content>.*)$/
|
13
13
|
LATEST_REPORTS_TO_SHOW = 10
|
14
|
+
DISPLAYED_HISTORY_REPORTS_THRESHOLD = 510 # https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/587
|
14
15
|
DAILY_REPORTS_THRESHOLDS = 10
|
15
16
|
|
16
17
|
class ReportsList
|
17
|
-
def initialize(preserved_content:, section_header:, reports:, extra_content:)
|
18
|
+
def initialize(preserved_content:, section_header:, reports:, total_reports_count:, extra_content:)
|
18
19
|
@preserved_content = preserved_content
|
19
20
|
@section_header = section_header
|
20
21
|
@reports = reports
|
22
|
+
@total_reports_count = total_reports_count
|
21
23
|
@extra_content = extra_content
|
22
24
|
end
|
23
25
|
|
@@ -31,14 +33,14 @@ module GitlabQuality
|
|
31
33
|
end
|
32
34
|
|
33
35
|
def reports_count
|
34
|
-
|
36
|
+
total_reports_count
|
35
37
|
end
|
36
38
|
|
37
39
|
def to_s
|
38
40
|
[
|
39
41
|
preserved_content,
|
40
42
|
"#{section_header} (#{reports_count})",
|
41
|
-
reports_list,
|
43
|
+
reports_list(total_reports_count),
|
42
44
|
extra_content
|
43
45
|
].reject(&:blank?).compact.join("\n\n")
|
44
46
|
end
|
@@ -53,15 +55,16 @@ module GitlabQuality
|
|
53
55
|
|
54
56
|
private
|
55
57
|
|
56
|
-
attr_reader :preserved_content, :section_header, :reports, :extra_content
|
58
|
+
attr_reader :preserved_content, :section_header, :reports, :total_reports_count, :extra_content
|
57
59
|
|
58
|
-
def reports_list
|
60
|
+
def reports_list(total_reports_count)
|
59
61
|
if sorted_reports.size > LATEST_REPORTS_TO_SHOW
|
62
|
+
max_displayed_reports = DISPLAYED_HISTORY_REPORTS_THRESHOLD - LATEST_REPORTS_TO_SHOW
|
60
63
|
[
|
61
64
|
"Last #{LATEST_REPORTS_TO_SHOW} reports:",
|
62
|
-
|
63
|
-
"<details><summary>
|
64
|
-
|
65
|
+
displayed_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
|
66
|
+
"<details><summary>With #{total_reports_count - LATEST_REPORTS_TO_SHOW} more reports (displaying up to #{max_displayed_reports} reports) </summary>",
|
67
|
+
displayed_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
|
65
68
|
"</details>"
|
66
69
|
].join("\n\n")
|
67
70
|
else
|
@@ -70,7 +73,12 @@ module GitlabQuality
|
|
70
73
|
end
|
71
74
|
|
72
75
|
def sorted_reports
|
73
|
-
@sorted_reports ||=
|
76
|
+
@sorted_reports ||=
|
77
|
+
reports.sort_by { |report| [report.report_date, report.job_id, report.to_s] }.reverse
|
78
|
+
end
|
79
|
+
|
80
|
+
def displayed_reports
|
81
|
+
sorted_reports[0...DISPLAYED_HISTORY_REPORTS_THRESHOLD]
|
74
82
|
end
|
75
83
|
end
|
76
84
|
|
@@ -88,8 +96,8 @@ module GitlabQuality
|
|
88
96
|
"1. #{report_date}: #{job_url} (#{pipeline_url}) #{extra_content}".strip
|
89
97
|
end
|
90
98
|
|
91
|
-
def
|
92
|
-
|
99
|
+
def job_id
|
100
|
+
job_url.split('/').last
|
93
101
|
end
|
94
102
|
|
95
103
|
private
|
@@ -122,14 +130,33 @@ module GitlabQuality
|
|
122
130
|
preserved_content = current_reports_content.split(reports_section_header).first&.strip
|
123
131
|
reports = report_lines(current_reports_content) + [ReportsList.report_list_item(test, item_extra_content: item_extra_content)]
|
124
132
|
|
133
|
+
total_reports_count = increment_total_reports_count(
|
134
|
+
reports_section_header: reports_section_header,
|
135
|
+
content: current_reports_content,
|
136
|
+
reports: reports
|
137
|
+
)
|
138
|
+
|
125
139
|
ReportsList.new(
|
126
140
|
preserved_content: preserved_content,
|
127
141
|
section_header: reports_section_header,
|
128
142
|
reports: reports,
|
143
|
+
total_reports_count: total_reports_count,
|
129
144
|
extra_content: reports_extra_content
|
130
145
|
)
|
131
146
|
end
|
132
147
|
|
148
|
+
def increment_total_reports_count(reports_section_header:, content:, reports:)
|
149
|
+
reports_count = reports.count
|
150
|
+
|
151
|
+
return reports_count if reports_count < DISPLAYED_HISTORY_REPORTS_THRESHOLD
|
152
|
+
|
153
|
+
count_match = content.match(/#{Regexp.escape(reports_section_header)} \((\d+)\)/)
|
154
|
+
|
155
|
+
return reports_count unless count_match
|
156
|
+
|
157
|
+
count_match[1].to_i + 1
|
158
|
+
end
|
159
|
+
|
133
160
|
def failed_issue_job_url(issue)
|
134
161
|
job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
|
135
162
|
# Legacy format
|
@@ -0,0 +1,79 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'http'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
module GitlabQuality
|
7
|
+
module TestTooling
|
8
|
+
module Report
|
9
|
+
class TestHealthIssueFinder < ReportAsIssue
|
10
|
+
HEALTH_PROBLEM_TYPE_TO_LABEL = {
|
11
|
+
'pass-after-retry' => 'test-health:pass-after-retry',
|
12
|
+
'slow' => 'test-health:slow',
|
13
|
+
'failures' => 'test-health:failures'
|
14
|
+
}.freeze
|
15
|
+
|
16
|
+
def initialize(health_problem_type: [], **kwargs)
|
17
|
+
super(**kwargs)
|
18
|
+
|
19
|
+
@health_problem_type = health_problem_type
|
20
|
+
end
|
21
|
+
|
22
|
+
def found_existing_unhealthy_test_issue?
|
23
|
+
issue_url = invoke!
|
24
|
+
|
25
|
+
!issue_url.nil? && !issue_url.empty?
|
26
|
+
end
|
27
|
+
|
28
|
+
def run!
|
29
|
+
existing_issue_found = nil
|
30
|
+
|
31
|
+
applicable_tests.each do |test|
|
32
|
+
issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: search_labels)
|
33
|
+
next if issues.empty?
|
34
|
+
|
35
|
+
existing_issue_found = issues.first.web_url
|
36
|
+
puts "Found an existing test health issue of type #{health_problem_type} for test #{test.file}:#{test.line_number}: #{existing_issue_found}."
|
37
|
+
break
|
38
|
+
end
|
39
|
+
|
40
|
+
puts "Did not find an existing test health issue of type #{health_problem_type}." unless existing_issue_found
|
41
|
+
|
42
|
+
existing_issue_found
|
43
|
+
end
|
44
|
+
|
45
|
+
def applicable_tests
|
46
|
+
applicable_tests = []
|
47
|
+
|
48
|
+
TestResults::Builder.new(file_glob: files, token: token, project: project).test_results_per_file do |test_results|
|
49
|
+
applicable_tests = test_results.select { |test| test_is_applicable?(test) }
|
50
|
+
end
|
51
|
+
|
52
|
+
applicable_tests
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
attr_reader :health_problem_type
|
58
|
+
|
59
|
+
# Be mindful about the number of tests this method would return,
|
60
|
+
# as we will make at least one API request per test.
|
61
|
+
def test_is_applicable?(test)
|
62
|
+
expected_test_status =
|
63
|
+
case health_problem_type
|
64
|
+
when 'failures'
|
65
|
+
'failed'
|
66
|
+
else
|
67
|
+
'passed'
|
68
|
+
end
|
69
|
+
|
70
|
+
test.status == expected_test_status
|
71
|
+
end
|
72
|
+
|
73
|
+
def search_labels
|
74
|
+
['test', HEALTH_PROBLEM_TYPE_TO_LABEL[health_problem_type]]
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -16,9 +16,14 @@ module GitlabQuality
|
|
16
16
|
|
17
17
|
def parse
|
18
18
|
JSON.parse(File.read(path))
|
19
|
+
rescue JSON::ParserError
|
20
|
+
Runtime::Logger.debug("#{self.class.name}##{__method__} attempted to parse invalid JSON at path: #{path}")
|
21
|
+
{}
|
19
22
|
end
|
20
23
|
|
21
24
|
def process
|
25
|
+
return [] if results.empty?
|
26
|
+
|
22
27
|
results['examples'].map do |test|
|
23
28
|
GitlabQuality::TestTooling::TestResult::JsonTestResult.new(report: test, project: project, token: token)
|
24
29
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab_quality-test_tooling
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-11-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -192,6 +192,20 @@ dependencies:
|
|
192
192
|
- - '='
|
193
193
|
- !ruby/object:Gem::Version
|
194
194
|
version: 3.7.0
|
195
|
+
- !ruby/object:Gem::Dependency
|
196
|
+
name: rspec_junit_formatter
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - "~>"
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: 0.6.0
|
202
|
+
type: :development
|
203
|
+
prerelease: false
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
205
|
+
requirements:
|
206
|
+
- - "~>"
|
207
|
+
- !ruby/object:Gem::Version
|
208
|
+
version: 0.6.0
|
195
209
|
- !ruby/object:Gem::Dependency
|
196
210
|
name: activesupport
|
197
211
|
requirement: !ruby/object:Gem::Requirement
|
@@ -400,6 +414,7 @@ description: A collection of test-related tools.
|
|
400
414
|
email:
|
401
415
|
- quality@gitlab.com
|
402
416
|
executables:
|
417
|
+
- existing-test-health-issue
|
403
418
|
- failed-test-issues
|
404
419
|
- flaky-test-issues
|
405
420
|
- generate-test-session
|
@@ -429,6 +444,7 @@ files:
|
|
429
444
|
- LICENSE.txt
|
430
445
|
- README.md
|
431
446
|
- Rakefile
|
447
|
+
- exe/existing-test-health-issue
|
432
448
|
- exe/failed-test-issues
|
433
449
|
- exe/flaky-test-issues
|
434
450
|
- exe/generate-test-session
|
@@ -477,6 +493,7 @@ files:
|
|
477
493
|
- lib/gitlab_quality/test_tooling/report/results_in_issues.rb
|
478
494
|
- lib/gitlab_quality/test_tooling/report/results_in_test_cases.rb
|
479
495
|
- lib/gitlab_quality/test_tooling/report/slow_test_issue.rb
|
496
|
+
- lib/gitlab_quality/test_tooling/report/test_health_issue_finder.rb
|
480
497
|
- lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb
|
481
498
|
- lib/gitlab_quality/test_tooling/runtime/env.rb
|
482
499
|
- lib/gitlab_quality/test_tooling/runtime/logger.rb
|