gitlab_quality-test_tooling 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +9 -4
- data/Gemfile.lock +1 -1
- data/Guardfile +0 -22
- data/README.md +150 -9
- data/exe/generate-test-session +50 -0
- data/exe/post-to-slack +58 -0
- data/exe/prepare-stage-reports +38 -0
- data/exe/relate-failure-issue +59 -0
- data/exe/report-results +56 -0
- data/exe/update-screenshot-paths +38 -0
- data/lib/gitlab_quality/test_tooling/gitlab_issue_client.rb +194 -0
- data/lib/gitlab_quality/test_tooling/gitlab_issue_dry_client.rb +26 -0
- data/lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb +51 -0
- data/lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb +75 -0
- data/lib/gitlab_quality/test_tooling/report/concerns/utils.rb +49 -0
- data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb +275 -0
- data/lib/gitlab_quality/test_tooling/report/prepare_stage_reports.rb +78 -0
- data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +377 -0
- data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb +134 -0
- data/lib/gitlab_quality/test_tooling/report/report_results.rb +83 -0
- data/lib/gitlab_quality/test_tooling/report/results_in_issues.rb +130 -0
- data/lib/gitlab_quality/test_tooling/report/results_in_testcases.rb +113 -0
- data/lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb +81 -0
- data/lib/gitlab_quality/test_tooling/runtime/env.rb +113 -0
- data/lib/gitlab_quality/test_tooling/runtime/logger.rb +92 -0
- data/lib/gitlab_quality/test_tooling/runtime/token_finder.rb +44 -0
- data/lib/gitlab_quality/test_tooling/slack/post_to_slack.rb +36 -0
- data/lib/gitlab_quality/test_tooling/summary_table.rb +41 -0
- data/lib/gitlab_quality/test_tooling/support/http_request.rb +34 -0
- data/lib/gitlab_quality/test_tooling/system_logs/finders/json_log_finder.rb +65 -0
- data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/api_log_finder.rb +21 -0
- data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/application_log_finder.rb +21 -0
- data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/exception_log_finder.rb +21 -0
- data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/graphql_log_finder.rb +21 -0
- data/lib/gitlab_quality/test_tooling/system_logs/log_types/log.rb +38 -0
- data/lib/gitlab_quality/test_tooling/system_logs/log_types/rails/api_log.rb +34 -0
- data/lib/gitlab_quality/test_tooling/system_logs/log_types/rails/application_log.rb +27 -0
- data/lib/gitlab_quality/test_tooling/system_logs/log_types/rails/exception_log.rb +23 -0
- data/lib/gitlab_quality/test_tooling/system_logs/log_types/rails/graphql_log.rb +30 -0
- data/lib/gitlab_quality/test_tooling/system_logs/shared_fields.rb +29 -0
- data/lib/gitlab_quality/test_tooling/system_logs/system_logs_formatter.rb +65 -0
- data/lib/gitlab_quality/test_tooling/test_results/base_test_results.rb +39 -0
- data/lib/gitlab_quality/test_tooling/test_results/builder.rb +35 -0
- data/lib/gitlab_quality/test_tooling/test_results/j_unit_test_results.rb +27 -0
- data/lib/gitlab_quality/test_tooling/test_results/json_test_results.rb +29 -0
- data/lib/gitlab_quality/test_tooling/test_results/test_result.rb +184 -0
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- data/lib/gitlab_quality/test_tooling.rb +11 -2
- metadata +51 -3
@@ -0,0 +1,194 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'gitlab'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
# Monkey patch the Gitlab client to use the correct API path and add required methods
|
7
|
+
class Client
|
8
|
+
def team_member(project, id)
|
9
|
+
get("/projects/#{url_encode(project)}/members/all/#{id}")
|
10
|
+
end
|
11
|
+
|
12
|
+
def issue_discussions(project, issue_id, options = {})
|
13
|
+
get("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions", query: options)
|
14
|
+
end
|
15
|
+
|
16
|
+
def add_note_to_issue_discussion_as_thread(project, issue_id, discussion_id, options = {})
|
17
|
+
post("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions/#{discussion_id}/notes", query: options)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
module GitlabQuality
|
23
|
+
module TestTooling
|
24
|
+
# The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
|
25
|
+
class GitlabIssueClient
|
26
|
+
MAINTAINER_ACCESS_LEVEL = 40
|
27
|
+
RETRY_BACK_OFF_DELAY = 60
|
28
|
+
MAX_RETRY_ATTEMPTS = 3
|
29
|
+
|
30
|
+
def initialize(token:, project:)
|
31
|
+
@token = token
|
32
|
+
@project = project
|
33
|
+
@retry_backoff = 0
|
34
|
+
|
35
|
+
configure_gitlab_client
|
36
|
+
end
|
37
|
+
|
38
|
+
def assert_user_permission!
|
39
|
+
handle_gitlab_client_exceptions do
|
40
|
+
user = Gitlab.user
|
41
|
+
member = Gitlab.team_member(project, user.id)
|
42
|
+
|
43
|
+
abort_not_permitted if member.access_level < MAINTAINER_ACCESS_LEVEL
|
44
|
+
end
|
45
|
+
rescue Gitlab::Error::NotFound
|
46
|
+
abort_not_permitted
|
47
|
+
end
|
48
|
+
|
49
|
+
def find_issues(iid: nil, options: {}, &select)
|
50
|
+
select ||= :itself
|
51
|
+
|
52
|
+
handle_gitlab_client_exceptions do
|
53
|
+
break [Gitlab.issue(project, iid)].select(&select) if iid
|
54
|
+
|
55
|
+
Gitlab.issues(project, options)
|
56
|
+
.auto_paginate
|
57
|
+
.select(&select)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def find_issue_discussions(iid:)
|
62
|
+
handle_gitlab_client_exceptions do
|
63
|
+
Gitlab.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def create_issue(title:, description:, labels:, issue_type: 'issue')
|
68
|
+
attrs = { issue_type: issue_type, description: description, labels: labels }
|
69
|
+
|
70
|
+
handle_gitlab_client_exceptions do
|
71
|
+
Gitlab.create_issue(project, title, attrs)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def edit_issue(iid:, options: {})
|
76
|
+
handle_gitlab_client_exceptions do
|
77
|
+
Gitlab.edit_issue(project, iid, options)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def find_issue_notes(iid:)
|
82
|
+
handle_gitlab_client_exceptions do
|
83
|
+
Gitlab.issue_notes(project, iid, order_by: 'created_at', sort: 'asc')&.auto_paginate
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def create_issue_note(iid:, note:)
|
88
|
+
handle_gitlab_client_exceptions do
|
89
|
+
Gitlab.create_issue_note(project, iid, note)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def edit_issue_note(issue_iid:, note_id:, note:)
|
94
|
+
handle_gitlab_client_exceptions do
|
95
|
+
Gitlab.edit_issue_note(project, issue_iid, note_id, note)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
|
100
|
+
handle_gitlab_client_exceptions do
|
101
|
+
Gitlab.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: body)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def find_user_id(username:)
|
106
|
+
handle_gitlab_client_exceptions do
|
107
|
+
user = Gitlab.users(username: username)&.first
|
108
|
+
user['id'] unless user.nil?
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def upload_file(file_fullpath:)
|
113
|
+
ignore_gitlab_client_exceptions do
|
114
|
+
Gitlab.upload_file(project, file_fullpath)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def ignore_gitlab_client_exceptions
|
119
|
+
yield
|
120
|
+
rescue StandardError, SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout,
|
121
|
+
Gitlab::Error::Error => e
|
122
|
+
puts "Ignoring the following error: #{e}"
|
123
|
+
end
|
124
|
+
|
125
|
+
def handle_gitlab_client_exceptions # rubocop:disable Metrics/AbcSize
|
126
|
+
yield
|
127
|
+
rescue Gitlab::Error::NotFound
|
128
|
+
# This error could be raised in assert_user_permission!
|
129
|
+
# If so, we want it to terminate at that point
|
130
|
+
raise
|
131
|
+
rescue SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout,
|
132
|
+
Gitlab::Error::InternalServerError, Gitlab::Error::Parsing => e
|
133
|
+
@retry_backoff += RETRY_BACK_OFF_DELAY
|
134
|
+
|
135
|
+
raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS
|
136
|
+
|
137
|
+
warn_exception(e)
|
138
|
+
warn("Sleeping for #{@retry_backoff} seconds before retrying...")
|
139
|
+
sleep @retry_backoff
|
140
|
+
|
141
|
+
retry
|
142
|
+
rescue StandardError => e
|
143
|
+
pipeline = Runtime::Env.pipeline_from_project_name
|
144
|
+
channel = case pipeline
|
145
|
+
when "canary"
|
146
|
+
"qa-production"
|
147
|
+
when "staging-canary"
|
148
|
+
"qa-staging"
|
149
|
+
else
|
150
|
+
"qa-#{pipeline}"
|
151
|
+
end
|
152
|
+
error_msg = warn_exception(e)
|
153
|
+
|
154
|
+
return unless Runtime::Env.ci_commit_ref_name == Runtime::Env.default_branch
|
155
|
+
|
156
|
+
slack_options = {
|
157
|
+
slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
|
158
|
+
channel: channel,
|
159
|
+
username: "GitLab QA Bot",
|
160
|
+
icon_emoji: ':ci_failing:',
|
161
|
+
message: <<~MSG
|
162
|
+
An unexpected error occurred while reporting test results in issues.
|
163
|
+
The error occurred in job: #{Runtime::Env.ci_job_url}
|
164
|
+
`#{error_msg}`
|
165
|
+
MSG
|
166
|
+
}
|
167
|
+
puts "Posting Slack message to channel: #{channel}"
|
168
|
+
|
169
|
+
GitlabQuality::TestTooling::Slack::PostToSlack.new(**slack_options).invoke!
|
170
|
+
end
|
171
|
+
|
172
|
+
private
|
173
|
+
|
174
|
+
attr_reader :token, :project
|
175
|
+
|
176
|
+
def configure_gitlab_client
|
177
|
+
Gitlab.configure do |config|
|
178
|
+
config.endpoint = Runtime::Env.gitlab_api_base
|
179
|
+
config.private_token = token
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def abort_not_permitted
|
184
|
+
abort "You must have at least Reporter access to the project to use this feature."
|
185
|
+
end
|
186
|
+
|
187
|
+
def warn_exception(error)
|
188
|
+
error_msg = "#{error.class.name} #{error.message}"
|
189
|
+
warn(error_msg)
|
190
|
+
error_msg
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
class GitlabIssueDryClient < GitlabIssueClient
|
6
|
+
def create_issue(title:, description:, labels:, issue_type: 'issue')
|
7
|
+
attrs = { description: description, labels: labels }
|
8
|
+
|
9
|
+
puts "The following #{issue_type} would have been created:"
|
10
|
+
puts "project: #{project}, title: #{title}, attrs: #{attrs}"
|
11
|
+
end
|
12
|
+
|
13
|
+
def edit_issue(iid:, options: {})
|
14
|
+
puts "The #{project}##{iid} issue would have been updated with: #{options}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def create_issue_note(iid:, note:)
|
18
|
+
puts "The following note would have been posted on #{project}##{iid} issue: #{note}"
|
19
|
+
end
|
20
|
+
|
21
|
+
def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
|
22
|
+
puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{body}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module GitlabQuality
|
6
|
+
module TestTooling
|
7
|
+
module Report
|
8
|
+
module Concerns
|
9
|
+
module FindSetDri
|
10
|
+
def set_dri_via_group(product_group, test)
|
11
|
+
parse_json_with_sets
|
12
|
+
fetch_stage_sets(test)
|
13
|
+
|
14
|
+
return @sets.sample['username'] if @stage_sets.empty?
|
15
|
+
|
16
|
+
fetch_group_sets(product_group)
|
17
|
+
|
18
|
+
if @group_sets.empty?
|
19
|
+
@stage_sets.sample['username']
|
20
|
+
else
|
21
|
+
@group_sets.sample['username']
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def parse_json_with_sets
|
28
|
+
response = Support::HttpRequest.make_http_request(
|
29
|
+
url: 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json'
|
30
|
+
)
|
31
|
+
@sets = JSON.parse(response.body).select { |user| user['role'].include?('software-engineer-in-test') }
|
32
|
+
end
|
33
|
+
|
34
|
+
def fetch_stage_sets(test)
|
35
|
+
@stage_sets = @sets.select do |user|
|
36
|
+
user['role'].include?(test.stage.split("_").map(&:capitalize).join(" "))
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def fetch_group_sets(product_group)
|
41
|
+
@group_sets = @stage_sets.select do |user|
|
42
|
+
user['role'].include?(product_group.split("_").map do |word|
|
43
|
+
word == 'and' ? word : word.capitalize
|
44
|
+
end.join(" "))
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/enumerable'
|
4
|
+
|
5
|
+
module GitlabQuality
|
6
|
+
module TestTooling
|
7
|
+
module Report
|
8
|
+
module Concerns
|
9
|
+
module ResultsReporter
|
10
|
+
include Concerns::Utils
|
11
|
+
|
12
|
+
TEST_CASE_RESULTS_SECTION_TEMPLATE = "\n\n### DO NOT EDIT BELOW THIS LINE\n\nActive and historical test results:"
|
13
|
+
|
14
|
+
def find_issue(test)
|
15
|
+
issues = search_for_issues(test)
|
16
|
+
|
17
|
+
warn(%(Too many #{issue_type}s found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
|
18
|
+
|
19
|
+
puts "Found existing #{issue_type}: #{issues.first.web_url}" unless issues.empty?
|
20
|
+
|
21
|
+
issues.first
|
22
|
+
end
|
23
|
+
|
24
|
+
def find_issue_by_iid(iid)
|
25
|
+
issues = gitlab.find_issues(iid: iid) do |issue|
|
26
|
+
issue.state == 'opened' && issue.issue_type == issue_type
|
27
|
+
end
|
28
|
+
|
29
|
+
warn(%(#{issue_type} iid "#{iid}" not valid)) if issues.empty?
|
30
|
+
|
31
|
+
issues.first
|
32
|
+
end
|
33
|
+
|
34
|
+
def issue_title_needs_updating?(issue, test)
|
35
|
+
issue.title.strip != title_from_test(test) && !%w[canary production preprod release].include?(pipeline)
|
36
|
+
end
|
37
|
+
|
38
|
+
def new_issue_labels(_test)
|
39
|
+
%w[Quality status::automated]
|
40
|
+
end
|
41
|
+
|
42
|
+
def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
|
43
|
+
labels = super
|
44
|
+
labels |= new_issue_labels(test).to_set
|
45
|
+
labels.delete_if { |label| label.start_with?("#{pipeline}::") }
|
46
|
+
labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
|
47
|
+
end
|
48
|
+
|
49
|
+
def update_issue_title(issue, test)
|
50
|
+
old_title = issue.title.strip
|
51
|
+
new_title = title_from_test(test)
|
52
|
+
|
53
|
+
warn(%(#{issue_type} title needs to be updated from '#{old_title}' to '#{new_title}'))
|
54
|
+
|
55
|
+
new_description = updated_description(issue, test)
|
56
|
+
|
57
|
+
gitlab.edit_issue(iid: issue.iid, options: { title: new_title, description: new_description })
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def search_term(test)
|
63
|
+
%("#{partial_file_path(test.file)}" "#{search_safe(test.name)}")
|
64
|
+
end
|
65
|
+
|
66
|
+
def search_for_issues(test)
|
67
|
+
gitlab.find_issues(options: { search: search_term(test) }) do |issue|
|
68
|
+
issue.state == 'opened' && issue.issue_type == issue_type && issue.title.strip == title_from_test(test)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module Report
|
6
|
+
module Concerns
|
7
|
+
module Utils
|
8
|
+
MAX_TITLE_LENGTH = 255
|
9
|
+
|
10
|
+
def partial_file_path(path)
|
11
|
+
path.match(/((ee|api|browser_ui).*)/i)[1]
|
12
|
+
end
|
13
|
+
|
14
|
+
def new_issue_title(test)
|
15
|
+
"#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
|
16
|
+
end
|
17
|
+
|
18
|
+
def title_from_test(test)
|
19
|
+
title = new_issue_title(test)
|
20
|
+
|
21
|
+
return title unless title.length > MAX_TITLE_LENGTH
|
22
|
+
|
23
|
+
"#{title[0...MAX_TITLE_LENGTH - 3]}..."
|
24
|
+
end
|
25
|
+
|
26
|
+
def search_safe(value)
|
27
|
+
value.delete('"')
|
28
|
+
end
|
29
|
+
|
30
|
+
def pipeline
|
31
|
+
# Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
|
32
|
+
#
|
33
|
+
# Tests can be run in several pipelines:
|
34
|
+
# gitlab, nightly, staging, canary, production, preprod, MRs, and the default branch (master/main)
|
35
|
+
#
|
36
|
+
# Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
|
37
|
+
# nightly, staging, canary, production, and preprod
|
38
|
+
#
|
39
|
+
# MR, master/main, and gitlab tests run in gitlab-qa, but we only want to report tests run on
|
40
|
+
# master/main because the other pipelines will be monitored by the author of the MR that triggered them.
|
41
|
+
# So we assume that we're reporting a master/main pipeline if the project name is 'gitlab'.
|
42
|
+
|
43
|
+
@pipeline ||= Runtime::Env.pipeline_from_project_name
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,275 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'erb'
|
4
|
+
require 'date'
|
5
|
+
|
6
|
+
module GitlabQuality
|
7
|
+
module TestTooling
|
8
|
+
module Report
|
9
|
+
class GenerateTestSession < ReportAsIssue
|
10
|
+
def initialize(**kwargs)
|
11
|
+
super
|
12
|
+
@issue_type = 'issue'
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
# rubocop:disable Metrics/AbcSize
|
18
|
+
def run!
|
19
|
+
puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
20
|
+
|
21
|
+
tests = Dir.glob(files).flat_map do |path|
|
22
|
+
puts "Loading tests in #{path}"
|
23
|
+
|
24
|
+
TestResults::JsonTestResults.new(path).to_a
|
25
|
+
end
|
26
|
+
|
27
|
+
issue = gitlab.create_issue(
|
28
|
+
title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
|
29
|
+
description: generate_description(tests),
|
30
|
+
labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
|
31
|
+
)
|
32
|
+
|
33
|
+
# Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/295493
|
34
|
+
unless Runtime::Env.qa_issue_url.to_s.empty?
|
35
|
+
gitlab.create_issue_note(
|
36
|
+
iid: issue.iid,
|
37
|
+
note: "/relate #{Runtime::Env.qa_issue_url}")
|
38
|
+
end
|
39
|
+
|
40
|
+
File.write('REPORT_ISSUE_URL', issue.web_url)
|
41
|
+
end
|
42
|
+
# rubocop:enable Metrics/AbcSize
|
43
|
+
|
44
|
+
def generate_description(tests)
|
45
|
+
<<~MARKDOWN.rstrip
|
46
|
+
## Session summary
|
47
|
+
|
48
|
+
* Deploy version: #{Runtime::Env.deploy_version}
|
49
|
+
* Deploy environment: #{Runtime::Env.deploy_environment}
|
50
|
+
* Pipeline: #{Runtime::Env.pipeline_from_project_name} [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
|
51
|
+
#{generate_summary(tests: tests)}
|
52
|
+
|
53
|
+
#{generate_failed_jobs_listing}
|
54
|
+
|
55
|
+
#{generate_stages_listing(tests)}
|
56
|
+
|
57
|
+
#{generate_qa_issue_relation}
|
58
|
+
|
59
|
+
#{generate_link_to_dashboard}
|
60
|
+
MARKDOWN
|
61
|
+
end
|
62
|
+
|
63
|
+
def generate_summary(tests:, tests_by_status: nil)
|
64
|
+
tests_by_status ||= tests.group_by(&:status)
|
65
|
+
total = tests.size
|
66
|
+
passed = tests_by_status['passed']&.size || 0
|
67
|
+
failed = tests_by_status['failed']&.size || 0
|
68
|
+
others = total - passed - failed
|
69
|
+
|
70
|
+
<<~MARKDOWN.chomp
|
71
|
+
* Total #{total} tests
|
72
|
+
* Passed #{passed} tests
|
73
|
+
* Failed #{failed} tests
|
74
|
+
* #{others} other tests (usually skipped)
|
75
|
+
MARKDOWN
|
76
|
+
end
|
77
|
+
|
78
|
+
def generate_failed_jobs_listing
|
79
|
+
failed_jobs = []
|
80
|
+
|
81
|
+
client = Gitlab.client(
|
82
|
+
endpoint: Runtime::Env.ci_api_v4_url,
|
83
|
+
private_token: Runtime::Env.gitlab_ci_api_token)
|
84
|
+
|
85
|
+
gitlab.handle_gitlab_client_exceptions do
|
86
|
+
failed_jobs = client.pipeline_jobs(
|
87
|
+
Runtime::Env.ci_project_id,
|
88
|
+
Runtime::Env.ci_pipeline_id,
|
89
|
+
scope: 'failed')
|
90
|
+
end
|
91
|
+
|
92
|
+
listings = failed_jobs.map do |job|
|
93
|
+
allowed_to_fail = ' (allowed to fail)' if job.allow_failure
|
94
|
+
|
95
|
+
"* [#{job.name}](#{job.web_url})#{allowed_to_fail}"
|
96
|
+
end.join("\n")
|
97
|
+
|
98
|
+
<<~MARKDOWN.chomp if failed_jobs.any?
|
99
|
+
## Failed jobs
|
100
|
+
|
101
|
+
#{listings}
|
102
|
+
MARKDOWN
|
103
|
+
end
|
104
|
+
|
105
|
+
def generate_stages_listing(tests)
|
106
|
+
generate_tests_by_stage(tests).map do |stage, tests_for_stage|
|
107
|
+
tests_by_status = tests_for_stage.group_by(&:status)
|
108
|
+
|
109
|
+
<<~MARKDOWN.chomp
|
110
|
+
### #{stage&.capitalize || 'Unknown'}
|
111
|
+
|
112
|
+
#{generate_summary(
|
113
|
+
tests: tests_for_stage, tests_by_status: tests_by_status)}
|
114
|
+
|
115
|
+
#{generate_testcase_listing_by_status(
|
116
|
+
tests: tests_for_stage, tests_by_status: tests_by_status)}
|
117
|
+
MARKDOWN
|
118
|
+
end.join("\n\n")
|
119
|
+
end
|
120
|
+
|
121
|
+
def generate_tests_by_stage(tests)
|
122
|
+
# https://about.gitlab.com/handbook/product/product-categories/#devops-stages
|
123
|
+
ordering = %w[
|
124
|
+
manage
|
125
|
+
plan
|
126
|
+
create
|
127
|
+
verify
|
128
|
+
package
|
129
|
+
release
|
130
|
+
configure
|
131
|
+
monitor
|
132
|
+
secure
|
133
|
+
defend
|
134
|
+
growth
|
135
|
+
fulfillment
|
136
|
+
enablement
|
137
|
+
]
|
138
|
+
|
139
|
+
tests.sort_by do |test|
|
140
|
+
ordering.index(test.stage) || ordering.size
|
141
|
+
end.group_by(&:stage)
|
142
|
+
end
|
143
|
+
|
144
|
+
def generate_testcase_listing_by_status(tests:, tests_by_status:)
|
145
|
+
failed_tests = tests_by_status['failed']
|
146
|
+
passed_tests = tests_by_status['passed']
|
147
|
+
other_tests = tests.reject do |test|
|
148
|
+
test.status == 'failed' || test.status == 'passed'
|
149
|
+
end
|
150
|
+
|
151
|
+
[
|
152
|
+
(failed_listings(failed_tests) if failed_tests),
|
153
|
+
(passed_listings(passed_tests) if passed_tests),
|
154
|
+
(other_listings(other_tests) if other_tests.any?)
|
155
|
+
].compact.join("\n\n")
|
156
|
+
end
|
157
|
+
|
158
|
+
def failed_listings(failed_tests)
|
159
|
+
generate_testcase_listing(failed_tests)
|
160
|
+
end
|
161
|
+
|
162
|
+
def passed_listings(passed_tests)
|
163
|
+
<<~MARKDOWN.chomp
|
164
|
+
<details><summary>Passed tests:</summary>
|
165
|
+
|
166
|
+
#{generate_testcase_listing(passed_tests, passed: true)}
|
167
|
+
|
168
|
+
</details>
|
169
|
+
MARKDOWN
|
170
|
+
end
|
171
|
+
|
172
|
+
def other_listings(other_tests)
|
173
|
+
<<~MARKDOWN.chomp
|
174
|
+
<details><summary>Other tests:</summary>
|
175
|
+
|
176
|
+
#{generate_testcase_listing(other_tests)}
|
177
|
+
|
178
|
+
</details>
|
179
|
+
MARKDOWN
|
180
|
+
end
|
181
|
+
|
182
|
+
def generate_testcase_listing(tests, passed: false)
|
183
|
+
body = tests.group_by(&:testcase).map do |testcase, tests_with_same_testcase|
|
184
|
+
tests_with_same_testcase.sort_by!(&:name)
|
185
|
+
[
|
186
|
+
generate_test_text(testcase, tests_with_same_testcase, passed),
|
187
|
+
generate_test_job(tests_with_same_testcase),
|
188
|
+
generate_test_status(tests_with_same_testcase),
|
189
|
+
generate_test_actions(tests_with_same_testcase)
|
190
|
+
].join(' | ')
|
191
|
+
end.join("\n")
|
192
|
+
|
193
|
+
<<~MARKDOWN.chomp
|
194
|
+
| Test | Job | Status | Action |
|
195
|
+
| - | - | - | - |
|
196
|
+
#{body}
|
197
|
+
MARKDOWN
|
198
|
+
end
|
199
|
+
|
200
|
+
def generate_test_text(testcase, tests_with_same_testcase, passed)
|
201
|
+
text = tests_with_same_testcase.map(&:name).uniq.join(', ')
|
202
|
+
encoded_text = ERB::Util.url_encode(text)
|
203
|
+
|
204
|
+
if testcase && !passed
|
205
|
+
# Workaround for reducing system notes on testcase issues
|
206
|
+
# The first regex extracts the link to the issues list page from a link to a single issue show page by removing the issue id.
|
207
|
+
"[#{text}](#{testcase.match(%r{[\s\S]+/[^/\d]+})}?state=opened&search=#{encoded_text})"
|
208
|
+
else
|
209
|
+
text
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
def generate_test_job(tests_with_same_testcase)
|
214
|
+
tests_with_same_testcase.map do |test|
|
215
|
+
ci_job_id = test.ci_job_url[/\d+\z/]
|
216
|
+
|
217
|
+
"[#{ci_job_id}](#{test.ci_job_url})#{' ~"quarantine"' if test.quarantine?}"
|
218
|
+
end.uniq.join(', ')
|
219
|
+
end
|
220
|
+
|
221
|
+
def generate_test_status(tests_with_same_testcase)
|
222
|
+
tests_with_same_testcase.map(&:status).uniq.map do |status|
|
223
|
+
%(~"#{status}")
|
224
|
+
end.join(', ')
|
225
|
+
end
|
226
|
+
|
227
|
+
def generate_test_actions(tests_with_same_testcase)
|
228
|
+
# All failed tests would be grouped together, meaning that
|
229
|
+
# if one failed, all the tests here would be failed too.
|
230
|
+
# So this check is safe. Same applies to 'passed'.
|
231
|
+
# But all other status might be mixing together,
|
232
|
+
# we cannot assume other statuses.
|
233
|
+
if tests_with_same_testcase.first.status == 'failed'
|
234
|
+
tests_having_failure_issue =
|
235
|
+
tests_with_same_testcase.select(&:failure_issue)
|
236
|
+
|
237
|
+
if tests_having_failure_issue.any?
|
238
|
+
items = tests_having_failure_issue.uniq(&:failure_issue).map do |test|
|
239
|
+
"<li>[ ] [failure issue](#{test.failure_issue})</li>"
|
240
|
+
end.join(' ')
|
241
|
+
|
242
|
+
"<ul>#{items}</ul>"
|
243
|
+
else
|
244
|
+
'<ul><li>[ ] failure issue exists or was created</li></ul>'
|
245
|
+
end
|
246
|
+
else
|
247
|
+
'-'
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
def generate_qa_issue_relation
|
252
|
+
return unless Runtime::Env.qa_issue_url
|
253
|
+
|
254
|
+
<<~MARKDOWN.chomp
|
255
|
+
## Release QA issue
|
256
|
+
|
257
|
+
* #{Runtime::Env.qa_issue_url}
|
258
|
+
|
259
|
+
/relate #{Runtime::Env.qa_issue_url}
|
260
|
+
MARKDOWN
|
261
|
+
end
|
262
|
+
|
263
|
+
def generate_link_to_dashboard
|
264
|
+
return unless Runtime::Env.qa_run_type
|
265
|
+
|
266
|
+
<<~MARKDOWN.chomp
|
267
|
+
## Link to Grafana dashboard for run-type of #{Runtime::Env.qa_run_type}
|
268
|
+
|
269
|
+
* https://dashboards.quality.gitlab.net/d/kuNYMgDnz/test-run-metrics?orgId=1&refresh=1m&var-run_type=#{Runtime::Env.qa_run_type}
|
270
|
+
MARKDOWN
|
271
|
+
end
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|