gitlab_quality-test_tooling 1.5.3 → 1.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +6 -1
- data/README.md +19 -0
- data/exe/flaky-test-issues +54 -0
- data/exe/relate-failure-issue +5 -0
- data/lib/gitlab_quality/test_tooling/gitlab_issue_client.rb +10 -0
- data/lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb +83 -0
- data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb +97 -0
- data/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb +0 -2
- data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +25 -66
- data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb +20 -10
- data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb +1 -1
- data/lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb +5 -5
- data/lib/gitlab_quality/test_tooling/test_result/json_test_result.rb +19 -1
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +20 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6924fc592c0dbb5e1ae0aed8710f7f7714c336bdd1d1d31b343e6edddd358ead
|
4
|
+
data.tar.gz: 0ebb5fbe0aeff0417193d4521cf52a78d849b83d116e08118b9cf87ee2655a71
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f9a64cbae7e3a34ae66450ae7fe20181560dc6860950a92e53c5aaf61ab10343372044968b5a726bef51b082a45dd9150d6f161d93a45a4ccf75d550cf981be1
|
7
|
+
data.tar.gz: a77ea69469a02b3cbd089cade6be20347abc03cada1cf93fec8380456f000d083a914296e0b35faff573e60d73e4759abd1b5d224cb04e64b38862fe720cf65c
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
gitlab_quality-test_tooling (1.
|
4
|
+
gitlab_quality-test_tooling (1.7.0)
|
5
5
|
activesupport (>= 6.1, < 7.2)
|
6
6
|
amatch (~> 0.4.1)
|
7
7
|
gitlab (~> 4.19)
|
@@ -29,6 +29,7 @@ GEM
|
|
29
29
|
ast (2.4.2)
|
30
30
|
backport (1.2.0)
|
31
31
|
benchmark (0.2.1)
|
32
|
+
byebug (11.1.3)
|
32
33
|
claide (1.1.0)
|
33
34
|
claide-plugins (0.9.2)
|
34
35
|
cork
|
@@ -160,6 +161,9 @@ GEM
|
|
160
161
|
pry (0.14.2)
|
161
162
|
coderay (~> 1.1)
|
162
163
|
method_source (~> 1.0)
|
164
|
+
pry-byebug (3.10.1)
|
165
|
+
byebug (~> 11.0)
|
166
|
+
pry (>= 0.13, < 0.15)
|
163
167
|
public_suffix (5.0.1)
|
164
168
|
racc (1.6.2)
|
165
169
|
rack (3.0.7)
|
@@ -279,6 +283,7 @@ DEPENDENCIES
|
|
279
283
|
gitlab_quality-test_tooling!
|
280
284
|
guard-rspec (~> 4.7)
|
281
285
|
lefthook (~> 1.3)
|
286
|
+
pry-byebug (= 3.10.1)
|
282
287
|
rake (~> 13.0)
|
283
288
|
rspec (~> 3.12)
|
284
289
|
simplecov (~> 0.22)
|
data/README.md
CHANGED
@@ -81,10 +81,14 @@ Usage: exe/relate-failure-issue [options]
|
|
81
81
|
Max stacktrace diff ratio for failure issues detection
|
82
82
|
-p, --project PROJECT Can be an integer or a group/project string
|
83
83
|
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
84
|
+
-r RELATED_ISSUES_FILE, The file path for the related issues
|
85
|
+
--related-issues-file
|
84
86
|
--system-log-files SYSTEM_LOG_FILES
|
85
87
|
Include errors from system logs in failure issues
|
86
88
|
--base-issue-labels BASE_ISSUE_LABELS
|
87
89
|
Labels to add to new failure issues
|
90
|
+
--exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH
|
91
|
+
Labels to exclude when searching for existing issues
|
88
92
|
--confidential Makes created new issues confidential
|
89
93
|
--dry-run Perform a dry-run (don't create or update issues)
|
90
94
|
-v, --version Show the version
|
@@ -133,6 +137,21 @@ Usage: exe/slow-test-issue [options]
|
|
133
137
|
-h, --help Show the usage
|
134
138
|
```
|
135
139
|
|
140
|
+
### `exe/flaky-test-issues`
|
141
|
+
|
142
|
+
```shell
|
143
|
+
Purpose: Purpose: Create flaky test issues for any passed test coming from rspec-retry JSON report files
|
144
|
+
Usage: exe/flaky-test-issue [options]
|
145
|
+
-i, --input-files INPUT_FILES JSON rspec-retry report files
|
146
|
+
-p, --project PROJECT Can be an integer or a group/project string
|
147
|
+
-m MERGE_REQUEST_IID, An integer merge request IID
|
148
|
+
--merge_request_iid
|
149
|
+
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
150
|
+
--dry-run Perform a dry-run (don't create note)
|
151
|
+
-v, --version Show the version
|
152
|
+
-h, --help Show the usage
|
153
|
+
```
|
154
|
+
|
136
155
|
### `exe/slow-test-merge-request-report-note`
|
137
156
|
|
138
157
|
```shell
|
@@ -0,0 +1,54 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "optparse"
|
6
|
+
|
7
|
+
require_relative "../lib/gitlab_quality/test_tooling"
|
8
|
+
|
9
|
+
params = {}
|
10
|
+
|
11
|
+
options = OptionParser.new do |opts|
|
12
|
+
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
|
+
|
14
|
+
opts.on('-i', '--input-files INPUT_FILES', String, 'JSON rspec-retry report files') do |input_files|
|
15
|
+
params[:input_files] = input_files
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
|
19
|
+
params[:project] = project
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on('-m', '--merge_request_iid MERGE_REQUEST_IID', String, 'An integer merge request IID') do |merge_request_iid|
|
23
|
+
params[:merge_request_iid] = merge_request_iid
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
|
27
|
+
params[:token] = token
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on('--dry-run', "Perform a dry-run (don't create issues)") do
|
31
|
+
params[:dry_run] = true
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on_tail('-v', '--version', 'Show the version') do
|
35
|
+
require_relative "../lib/gitlab_quality/test_tooling/version"
|
36
|
+
puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
|
37
|
+
exit
|
38
|
+
end
|
39
|
+
|
40
|
+
opts.on_tail('-h', '--help', 'Show the usage') do
|
41
|
+
puts "Purpose: Create flaky test issues for any passed test coming from rspec-retry JSON report files."
|
42
|
+
puts opts
|
43
|
+
exit
|
44
|
+
end
|
45
|
+
|
46
|
+
opts.parse(ARGV)
|
47
|
+
end
|
48
|
+
|
49
|
+
if params.any?
|
50
|
+
GitlabQuality::TestTooling::Report::FlakyTestIssue.new(**params).invoke!
|
51
|
+
else
|
52
|
+
puts options
|
53
|
+
exit 1
|
54
|
+
end
|
data/exe/relate-failure-issue
CHANGED
@@ -41,6 +41,11 @@ options = OptionParser.new do |opts|
|
|
41
41
|
params[:base_issue_labels] = base_issue_labels.split(',')
|
42
42
|
end
|
43
43
|
|
44
|
+
opts.on('--exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH', String,
|
45
|
+
'Labels to exclude when searching for existing issues') do |exclude_labels_for_search|
|
46
|
+
params[:exclude_labels_for_search] = exclude_labels_for_search.split(',')
|
47
|
+
end
|
48
|
+
|
44
49
|
opts.on('--confidential', "Makes created new issues confidential") do
|
45
50
|
params[:confidential] = true
|
46
51
|
end
|
@@ -55,6 +55,16 @@ module GitlabQuality
|
|
55
55
|
end
|
56
56
|
end
|
57
57
|
|
58
|
+
def find_issues_by_hash(test_hash, &select)
|
59
|
+
select ||= :itself
|
60
|
+
|
61
|
+
handle_gitlab_client_exceptions do
|
62
|
+
client.search_in_project(project, 'issues', test_hash)
|
63
|
+
.auto_paginate
|
64
|
+
.select(&select)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
58
68
|
def find_issue_discussions(iid:)
|
59
69
|
handle_gitlab_client_exceptions do
|
60
70
|
client.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module Report
|
6
|
+
module Concerns
|
7
|
+
module IssueReports
|
8
|
+
JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
|
9
|
+
FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
|
10
|
+
REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
|
11
|
+
|
12
|
+
def initial_reports_section(test)
|
13
|
+
<<~REPORTS
|
14
|
+
### Reports (1)
|
15
|
+
|
16
|
+
#{report_list_item(test)}
|
17
|
+
REPORTS
|
18
|
+
end
|
19
|
+
|
20
|
+
def add_report_to_issue_description(issue, test)
|
21
|
+
issue_description = issue.description
|
22
|
+
|
23
|
+
# We include the number of reports in the header, for visibility.
|
24
|
+
new_issue_description =
|
25
|
+
if issue_description.include?('### Reports')
|
26
|
+
# We count the number of existing reports.
|
27
|
+
reports_count = issue_description
|
28
|
+
.scan(REPORT_ITEM_REGEX)
|
29
|
+
.size.to_i + 1
|
30
|
+
issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
|
31
|
+
else # For issue with the legacy format, we add the Reports section
|
32
|
+
update_legacy_issue_description(issue_description)
|
33
|
+
end
|
34
|
+
|
35
|
+
[new_issue_description, report_list_item(test)].join("\n")
|
36
|
+
end
|
37
|
+
|
38
|
+
def failed_issue_job_url(issue)
|
39
|
+
job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
|
40
|
+
# Legacy format
|
41
|
+
job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX).last
|
42
|
+
end
|
43
|
+
|
44
|
+
def failed_issue_job_urls(issue)
|
45
|
+
job_urls_from_description(issue.description, REPORT_ITEM_REGEX) +
|
46
|
+
# Legacy format
|
47
|
+
job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX)
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def report_list_item(test)
|
53
|
+
"1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
|
54
|
+
end
|
55
|
+
|
56
|
+
def job_urls_from_description(issue_description, regex)
|
57
|
+
issue_description.lines.filter_map do |line|
|
58
|
+
match = line.match(regex)
|
59
|
+
match[:job_url] if match
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def update_legacy_issue_description(issue_description)
|
64
|
+
test_captures = issue_description.scan(JOB_URL_REGEX)
|
65
|
+
reports_count = test_captures.size.to_i + 1
|
66
|
+
|
67
|
+
updated_description = "#{issue_description}\n\n### Reports (#{reports_count})\n"
|
68
|
+
updated_description = [updated_description, *test_captures_to_report_items(test_captures)].join("\n") unless test_captures.empty?
|
69
|
+
updated_description
|
70
|
+
end
|
71
|
+
|
72
|
+
def test_captures_to_report_items(test_captures)
|
73
|
+
test_captures.map do |ci_job_url, _, _|
|
74
|
+
report_list_item(GitlabQuality::TestTooling::TestResult::JsonTestResult.new(
|
75
|
+
'ci_job_url' => ci_job_url
|
76
|
+
))
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module Report
|
6
|
+
# Uses the API to create GitLab issues for any passed test coming from JSON test reports.
|
7
|
+
# We expect the test reports to come from rspec-retry, or from a new RSpec process where
|
8
|
+
# we retried failing specs.
|
9
|
+
#
|
10
|
+
# - Takes the JSON test reports like rspec-*.json (typically from rspec-retry gem)`
|
11
|
+
# - Takes a project where flaky test issues should be created
|
12
|
+
# - For every passed test in the report:
|
13
|
+
# - Find issue by test hash
|
14
|
+
# - Reopen issue if it already exists, but is closed
|
15
|
+
class FlakyTestIssue < ReportAsIssue
|
16
|
+
include Concerns::IssueReports
|
17
|
+
|
18
|
+
NEW_ISSUE_LABELS = Set.new(['type::maintenance', 'priority::3', 'severity::3', 'failure::flaky-test']).freeze
|
19
|
+
FOUND_IN_MR_LABEL = 'found:in MR'
|
20
|
+
FOUND_IN_MASTER_LABEL = 'found:master'
|
21
|
+
|
22
|
+
def initialize(token:, input_files:, project: nil, merge_request_iid: nil, confidential: false, dry_run: false, **_kwargs)
|
23
|
+
super(token: token, input_files: input_files, project: project, confidential: confidential, dry_run: dry_run)
|
24
|
+
@merge_request_iid = merge_request_iid
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
attr_reader :merge_request_iid
|
30
|
+
|
31
|
+
def run!
|
32
|
+
puts "Reporting flaky tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
33
|
+
|
34
|
+
TestResults::Builder.new(files).test_results_per_file do |test_results|
|
35
|
+
puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
|
36
|
+
|
37
|
+
test_results.each do |test|
|
38
|
+
next if test.status != 'passed' # We only want failed tests that passed in the end
|
39
|
+
|
40
|
+
create_flaky_issue(test)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def new_issue_title(test)
|
46
|
+
"Flaky test in #{super}"
|
47
|
+
end
|
48
|
+
|
49
|
+
def new_issue_labels(_test)
|
50
|
+
found_label =
|
51
|
+
if !merge_request_iid || merge_request_iid.empty?
|
52
|
+
FOUND_IN_MASTER_LABEL
|
53
|
+
else
|
54
|
+
FOUND_IN_MR_LABEL
|
55
|
+
end
|
56
|
+
|
57
|
+
NEW_ISSUE_LABELS + [found_label]
|
58
|
+
end
|
59
|
+
|
60
|
+
def new_issue_description(test)
|
61
|
+
super + [
|
62
|
+
"Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html) " \
|
63
|
+
"to learn more about how to reproduce them.",
|
64
|
+
initial_reports_section(test)
|
65
|
+
].compact.join("\n\n")
|
66
|
+
end
|
67
|
+
|
68
|
+
def create_flaky_issue(test)
|
69
|
+
puts " => Finding existing issues for flaky test '#{test.name}' (run time: #{test.run_time} seconds)..."
|
70
|
+
|
71
|
+
issues = find_issues_by_hash(test_hash(test))
|
72
|
+
issues.each do |issue|
|
73
|
+
puts " => Existing issue link #{issue.web_url}."
|
74
|
+
|
75
|
+
puts " => Adding the flaky test to the existing issue..."
|
76
|
+
add_report_to_issue(issue, test)
|
77
|
+
|
78
|
+
if issue.state == 'closed'
|
79
|
+
puts " => Issue is closed. Reopening it."
|
80
|
+
reopen_issue(issue)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
create_issue(test) unless issues.any?
|
85
|
+
end
|
86
|
+
|
87
|
+
def add_report_to_issue(issue, test)
|
88
|
+
gitlab.edit_issue(iid: issue.iid, options: { description: add_report_to_issue_description(issue, test) })
|
89
|
+
end
|
90
|
+
|
91
|
+
def reopen_issue(issue)
|
92
|
+
gitlab.edit_issue(iid: issue.iid, options: { state_event: 'reopen' })
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -7,7 +7,6 @@ module GitlabQuality
|
|
7
7
|
SLOW_TEST_MESSAGE = '<!-- slow-test -->'
|
8
8
|
SLOW_TEST_LABEL = '/label ~"rspec:slow test detected"'
|
9
9
|
SLOW_TEST_NOTE_SOURCE_CODE = 'Generated by [`gitlab_quality-test_tooling`](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/blob/main/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb).'
|
10
|
-
SLOW_TEST_NOTE_FEEDBACK = 'Please [share your feedback and suggestions](https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/289).'
|
11
10
|
|
12
11
|
def initialize(token:, input_files:, merge_request_iid:, project: nil, dry_run: false, **_kwargs)
|
13
12
|
@project = project
|
@@ -70,7 +69,6 @@ module GitlabQuality
|
|
70
69
|
SLOW_TEST_MESSAGE,
|
71
70
|
SLOW_TEST_LABEL,
|
72
71
|
":tools: #{SLOW_TEST_NOTE_SOURCE_CODE}\n",
|
73
|
-
":recycle: #{SLOW_TEST_NOTE_FEEDBACK}\n",
|
74
72
|
"---\n",
|
75
73
|
":snail: Slow tests detected in this merge request. These slow tests might be related to this merge request's changes.",
|
76
74
|
"<details><summary>Click to expand</summary>\n",
|
@@ -18,6 +18,7 @@ module GitlabQuality
|
|
18
18
|
class RelateFailureIssue < ReportAsIssue
|
19
19
|
include Concerns::FindSetDri
|
20
20
|
include Concerns::GroupAndCategoryLabels
|
21
|
+
include Concerns::IssueReports
|
21
22
|
include Amatch
|
22
23
|
|
23
24
|
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
|
@@ -25,9 +26,7 @@ module GitlabQuality
|
|
25
26
|
SPAM_THRESHOLD_FOR_FAILURE_ISSUES = 3
|
26
27
|
FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
|
27
28
|
ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m
|
28
|
-
|
29
|
-
FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
|
30
|
-
REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
|
29
|
+
|
31
30
|
NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
|
32
31
|
IGNORED_FAILURES = [
|
33
32
|
'Net::ReadTimeout',
|
@@ -37,18 +36,24 @@ module GitlabQuality
|
|
37
36
|
|
38
37
|
MultipleIssuesFound = Class.new(StandardError)
|
39
38
|
|
40
|
-
def initialize(
|
39
|
+
def initialize(
|
40
|
+
max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION,
|
41
|
+
system_logs: [],
|
42
|
+
base_issue_labels: nil,
|
43
|
+
exclude_labels_for_search: nil,
|
44
|
+
**kwargs)
|
41
45
|
super
|
42
46
|
@max_diff_ratio = max_diff_ratio.to_f
|
43
47
|
@system_logs = Dir.glob(system_logs)
|
44
48
|
@base_issue_labels = Set.new(base_issue_labels)
|
49
|
+
@exclude_labels_for_search = Set.new(exclude_labels_for_search)
|
45
50
|
@issue_type = 'issue'
|
46
51
|
@commented_issue_list = Set.new
|
47
52
|
end
|
48
53
|
|
49
54
|
private
|
50
55
|
|
51
|
-
attr_reader :max_diff_ratio, :system_logs, :base_issue_labels
|
56
|
+
attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search
|
52
57
|
|
53
58
|
def run!
|
54
59
|
puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
@@ -86,7 +91,9 @@ module GitlabQuality
|
|
86
91
|
begin
|
87
92
|
issue = find_issue_and_update_reports(test)
|
88
93
|
|
89
|
-
create_issue(test) unless issue || test.quarantine?
|
94
|
+
issue = create_issue(test) unless issue || test.quarantine?
|
95
|
+
|
96
|
+
issue
|
90
97
|
rescue MultipleIssuesFound => e
|
91
98
|
warn(e.message)
|
92
99
|
end
|
@@ -134,35 +141,15 @@ module GitlabQuality
|
|
134
141
|
|
135
142
|
def pipeline_issues_with_similar_stacktrace(test)
|
136
143
|
search_labels = (base_issue_labels + Set.new(%w[test failure::new])).to_a
|
144
|
+
not_labels = exclude_labels_for_search.to_a
|
137
145
|
gitlab.find_issues(options: { labels: search_labels,
|
146
|
+
not: { labels: not_labels },
|
138
147
|
created_after: past_timestamp(2) }).select do |issue|
|
139
148
|
job_url_from_issue = failed_issue_job_url(issue)
|
140
149
|
|
141
150
|
next if pipeline != pipeline_env_from_job_url(job_url_from_issue)
|
142
151
|
|
143
|
-
|
144
|
-
stack_trace_from_test = cleaned_stacktrace_from_test(test)
|
145
|
-
diff_ratio = compare_stack_traces(stack_trace_from_test, stack_trace_from_issue)
|
146
|
-
diff_ratio < max_diff_ratio
|
147
|
-
end
|
148
|
-
end
|
149
|
-
|
150
|
-
def failed_issue_job_url(issue)
|
151
|
-
job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
|
152
|
-
# Legacy format
|
153
|
-
job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX).last
|
154
|
-
end
|
155
|
-
|
156
|
-
def failed_issue_job_urls(issue)
|
157
|
-
job_urls_from_description(issue.description, REPORT_ITEM_REGEX) +
|
158
|
-
# Legacy format
|
159
|
-
job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX)
|
160
|
-
end
|
161
|
-
|
162
|
-
def job_urls_from_description(issue_description, regex)
|
163
|
-
issue_description.lines.filter_map do |line|
|
164
|
-
match = line.match(regex)
|
165
|
-
match[:job_url] if match
|
152
|
+
compare_stack_traces(cleaned_stack_trace_from_test(test), cleaned_stack_trace_from_issue(issue)) < max_diff_ratio
|
166
153
|
end
|
167
154
|
end
|
168
155
|
|
@@ -182,7 +169,11 @@ module GitlabQuality
|
|
182
169
|
end
|
183
170
|
|
184
171
|
def failure_issues(test)
|
185
|
-
|
172
|
+
find_issues_for_test(
|
173
|
+
test,
|
174
|
+
labels: base_issue_labels + Set.new(%w[test]),
|
175
|
+
not_labels: exclude_labels_for_search
|
176
|
+
)
|
186
177
|
end
|
187
178
|
|
188
179
|
def full_stacktrace(test)
|
@@ -203,7 +194,7 @@ module GitlabQuality
|
|
203
194
|
remove_unique_resource_names(relevant_issue_stacktrace)
|
204
195
|
end
|
205
196
|
|
206
|
-
def
|
197
|
+
def cleaned_stack_trace_from_test(test)
|
207
198
|
test_failure_stacktrace = sanitize_stacktrace(full_stacktrace(test),
|
208
199
|
FAILURE_STACKTRACE_REGEX) || full_stacktrace(test)
|
209
200
|
remove_unique_resource_names(test_failure_stacktrace)
|
@@ -219,7 +210,7 @@ module GitlabQuality
|
|
219
210
|
end
|
220
211
|
|
221
212
|
def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
|
222
|
-
clean_first_test_failure_stacktrace =
|
213
|
+
clean_first_test_failure_stacktrace = cleaned_stack_trace_from_test(test)
|
223
214
|
# Search with the `search` param returns 500 errors, so we filter by `base_issue_labels` and then filter further in Ruby
|
224
215
|
failure_issues(test).each_with_object({}) do |issue, memo|
|
225
216
|
clean_relevant_issue_stacktrace = cleaned_stack_trace_from_issue(issue)
|
@@ -287,7 +278,7 @@ module GitlabQuality
|
|
287
278
|
"```\n#{full_stacktrace(test)}\n```",
|
288
279
|
screenshot_section(test),
|
289
280
|
system_log_errors_section(test),
|
290
|
-
|
281
|
+
initial_reports_section(test)
|
291
282
|
].compact.join("\n\n")
|
292
283
|
end
|
293
284
|
|
@@ -310,18 +301,6 @@ module GitlabQuality
|
|
310
301
|
section
|
311
302
|
end
|
312
303
|
|
313
|
-
def reports_section(test)
|
314
|
-
<<~REPORTS
|
315
|
-
### Reports (1)
|
316
|
-
|
317
|
-
#{report_list_item(test)}
|
318
|
-
REPORTS
|
319
|
-
end
|
320
|
-
|
321
|
-
def report_list_item(test)
|
322
|
-
"1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
|
323
|
-
end
|
324
|
-
|
325
304
|
def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
|
326
305
|
(Set.new(base_issue_labels) + (super << pipeline_name_label)).to_a
|
327
306
|
end
|
@@ -346,7 +325,7 @@ module GitlabQuality
|
|
346
325
|
state_event = issue.state == 'closed' ? 'reopen' : nil
|
347
326
|
|
348
327
|
issue_attrs = {
|
349
|
-
description:
|
328
|
+
description: add_report_to_issue_description(issue, test),
|
350
329
|
labels: up_to_date_labels(test: test, issue: issue)
|
351
330
|
}
|
352
331
|
issue_attrs[:state_event] = state_event if state_event
|
@@ -355,26 +334,6 @@ module GitlabQuality
|
|
355
334
|
puts " => Added a report in '#{issue.title}': #{issue.web_url}!"
|
356
335
|
end
|
357
336
|
|
358
|
-
def up_to_date_issue_description(issue_description, test)
|
359
|
-
# We include the number of reports in the header, for visibility.
|
360
|
-
new_issue_description =
|
361
|
-
if issue_description.include?('### Reports')
|
362
|
-
# We count the number of existing reports.
|
363
|
-
reports_count = issue_description
|
364
|
-
.scan(REPORT_ITEM_REGEX)
|
365
|
-
.size.to_i + 1
|
366
|
-
issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
|
367
|
-
else # For issue with the legacy format, we add the Reports section
|
368
|
-
reports_count = issue_description
|
369
|
-
.scan(JOB_URL_REGEX)
|
370
|
-
.size.to_i + 1
|
371
|
-
|
372
|
-
"#{issue_description}\n\n### Reports (#{reports_count})"
|
373
|
-
end
|
374
|
-
|
375
|
-
"#{new_issue_description}\n#{report_list_item(test)}"
|
376
|
-
end
|
377
|
-
|
378
337
|
def new_issue_title(test)
|
379
338
|
"Failure in #{super}"
|
380
339
|
end
|
@@ -39,7 +39,10 @@ module GitlabQuality
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def test_hash(test)
|
42
|
-
|
42
|
+
# Should not be more than 50 characters if we want it indexed.
|
43
|
+
#
|
44
|
+
# See https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/issues/27#note_1607276486
|
45
|
+
OpenSSL::Digest.hexdigest('SHA256', "#{test.file}#{test.name}")[..40]
|
43
46
|
end
|
44
47
|
|
45
48
|
def new_issue_description(test)
|
@@ -48,7 +51,8 @@ module GitlabQuality
|
|
48
51
|
|
49
52
|
| Field | Value |
|
50
53
|
| ------ | ------ |
|
51
|
-
| File | #{test.test_file_link} |
|
54
|
+
| File URL | #{test.test_file_link} |
|
55
|
+
| Filename | `#{test.file}` |
|
52
56
|
| Description | `#{test.name}` |
|
53
57
|
| Test level | #{test.level} |
|
54
58
|
| Hash | `#{test_hash(test)}` |
|
@@ -134,15 +138,21 @@ module GitlabQuality
|
|
134
138
|
labels
|
135
139
|
end
|
136
140
|
|
137
|
-
def
|
138
|
-
|
141
|
+
def find_issues_by_hash(test_hash)
|
142
|
+
gitlab.find_issues_by_hash(test_hash)
|
143
|
+
end
|
144
|
+
|
145
|
+
def find_issues_for_test(test, labels:, not_labels: Set.new, state: nil)
|
146
|
+
search_options = { labels: labels.to_a, not: { labels: not_labels.to_a } }
|
139
147
|
search_options[:state] = state if state
|
140
148
|
|
141
|
-
gitlab.find_issues(options: search_options).find_all
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
149
|
+
gitlab.find_issues(options: search_options).find_all { |issue| issue_match_test?(issue, test) }
|
150
|
+
end
|
151
|
+
|
152
|
+
def issue_match_test?(issue, test)
|
153
|
+
issue_title = issue.title.strip
|
154
|
+
test_file_path_found = !test.file.to_s.empty? && issue_title.include?(partial_file_path(test.file))
|
155
|
+
issue_title.include?(test.name) || test_file_path_found
|
146
156
|
end
|
147
157
|
|
148
158
|
def pipeline_name_label
|
@@ -160,7 +170,7 @@ module GitlabQuality
|
|
160
170
|
when 'customers-gitlab-com'
|
161
171
|
'found:customers.stg.gitlab.com'
|
162
172
|
else
|
163
|
-
|
173
|
+
puts " => [DEBUG] No `found:*` label for the `#{pipeline}` pipeline!"
|
164
174
|
end
|
165
175
|
end
|
166
176
|
|
@@ -68,7 +68,7 @@ module GitlabQuality
|
|
68
68
|
def create_slow_issue(test)
|
69
69
|
puts " => Finding existing issues for slow test '#{test.name}' (run time: #{test.run_time} seconds)..."
|
70
70
|
|
71
|
-
issues =
|
71
|
+
issues = find_issues_for_test(test, labels: SEARCH_LABELS, state: 'opened')
|
72
72
|
|
73
73
|
if issues.blank?
|
74
74
|
issues << create_issue(test)
|
@@ -16,8 +16,8 @@ module GitlabQuality
|
|
16
16
|
|
17
17
|
def invoke!
|
18
18
|
Dir.glob(input_files).each do |input_file|
|
19
|
-
|
20
|
-
|
19
|
+
rewrite_screenshot_paths_in_junit_file(input_file)
|
20
|
+
rewrite_screenshot_paths_in_json_file(input_file.gsub('.xml', '.json'))
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
@@ -25,7 +25,7 @@ module GitlabQuality
|
|
25
25
|
|
26
26
|
attr_reader :input_files
|
27
27
|
|
28
|
-
def
|
28
|
+
def rewrite_screenshot_paths_in_junit_file(junit_file)
|
29
29
|
File.write(
|
30
30
|
junit_file,
|
31
31
|
rewrite_each_junit_screenshot_path(junit_file).to_s
|
@@ -34,7 +34,7 @@ module GitlabQuality
|
|
34
34
|
puts "Saved #{junit_file}"
|
35
35
|
end
|
36
36
|
|
37
|
-
def
|
37
|
+
def rewrite_screenshot_paths_in_json_file(json_file)
|
38
38
|
File.write(
|
39
39
|
json_file,
|
40
40
|
JSON.pretty_generate(
|
@@ -58,7 +58,7 @@ module GitlabQuality
|
|
58
58
|
examples = report['examples']
|
59
59
|
|
60
60
|
examples.each do |example|
|
61
|
-
next unless example['screenshot'].present?
|
61
|
+
next unless example['screenshot'].present? && example['screenshot']['image'].present?
|
62
62
|
|
63
63
|
example['screenshot']['image'] =
|
64
64
|
remove_container_absolute_path_prefix(example.dig('screenshot', 'image'), test_artifacts_directory(json_file))
|
@@ -18,7 +18,25 @@ module GitlabQuality
|
|
18
18
|
].freeze
|
19
19
|
|
20
20
|
def name
|
21
|
-
|
21
|
+
# If we see a string representation of an object in a test full_description, we discard it.
|
22
|
+
#
|
23
|
+
# This is to ensure that tests would have a reproducible name, in case they don't have a name.
|
24
|
+
#
|
25
|
+
# Test example:
|
26
|
+
#
|
27
|
+
# it { is_expected.to eq(secondary_node) }
|
28
|
+
#
|
29
|
+
# Would have its full_description as follows:
|
30
|
+
#
|
31
|
+
# Gitlab::Geo.proxied_site on a primary for a proxied request with a proxy extra data header
|
32
|
+
# for an existing site is expected to eq #<GeoNode id: 116, primary: false, oauth_application_id: 97
|
33
|
+
# , enabled: true, access_key: [FILTERED], e...pdated_at: "2023-10-10 08:49:49.797128469 +0000",
|
34
|
+
# sync_object_storage: true, secret_access_key: nil>
|
35
|
+
#
|
36
|
+
# Which would change for every test run due to the timestamps.
|
37
|
+
#
|
38
|
+
# See https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/merge_requests/77#note_1608793804
|
39
|
+
report.fetch('full_description').split('#<').first
|
22
40
|
end
|
23
41
|
|
24
42
|
def file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab_quality-test_tooling
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-11-
|
11
|
+
date: 2023-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '1.3'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: pry-byebug
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - '='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 3.10.1
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - '='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 3.10.1
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: rake
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -332,6 +346,7 @@ description: A collection of test-related tools.
|
|
332
346
|
email:
|
333
347
|
- quality@gitlab.com
|
334
348
|
executables:
|
349
|
+
- flaky-test-issues
|
335
350
|
- generate-test-session
|
336
351
|
- post-to-slack
|
337
352
|
- prepare-stage-reports
|
@@ -357,6 +372,7 @@ files:
|
|
357
372
|
- LICENSE.txt
|
358
373
|
- README.md
|
359
374
|
- Rakefile
|
375
|
+
- exe/flaky-test-issues
|
360
376
|
- exe/generate-test-session
|
361
377
|
- exe/post-to-slack
|
362
378
|
- exe/prepare-stage-reports
|
@@ -374,8 +390,10 @@ files:
|
|
374
390
|
- lib/gitlab_quality/test_tooling/labels_inference.rb
|
375
391
|
- lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb
|
376
392
|
- lib/gitlab_quality/test_tooling/report/concerns/group_and_category_labels.rb
|
393
|
+
- lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb
|
377
394
|
- lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
|
378
395
|
- lib/gitlab_quality/test_tooling/report/concerns/utils.rb
|
396
|
+
- lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb
|
379
397
|
- lib/gitlab_quality/test_tooling/report/generate_test_session.rb
|
380
398
|
- lib/gitlab_quality/test_tooling/report/issue_logger.rb
|
381
399
|
- lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb
|