gitlab_quality-test_tooling 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +9 -4
  3. data/Gemfile.lock +1 -1
  4. data/Guardfile +0 -22
  5. data/exe/generate-test-session +49 -0
  6. data/exe/post-to-slack +57 -0
  7. data/exe/prepare-stage-reports +38 -0
  8. data/exe/relate-failure-issue +59 -0
  9. data/exe/report-results +56 -0
  10. data/exe/update-screenshot-paths +38 -0
  11. data/lib/gitlab_quality/test_tooling/gitlab_issue_client.rb +194 -0
  12. data/lib/gitlab_quality/test_tooling/gitlab_issue_dry_client.rb +26 -0
  13. data/lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb +51 -0
  14. data/lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb +75 -0
  15. data/lib/gitlab_quality/test_tooling/report/concerns/utils.rb +49 -0
  16. data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb +275 -0
  17. data/lib/gitlab_quality/test_tooling/report/prepare_stage_reports.rb +79 -0
  18. data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +377 -0
  19. data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb +134 -0
  20. data/lib/gitlab_quality/test_tooling/report/report_results.rb +83 -0
  21. data/lib/gitlab_quality/test_tooling/report/results_in_issues.rb +130 -0
  22. data/lib/gitlab_quality/test_tooling/report/results_in_testcases.rb +113 -0
  23. data/lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb +81 -0
  24. data/lib/gitlab_quality/test_tooling/runtime/env.rb +113 -0
  25. data/lib/gitlab_quality/test_tooling/runtime/logger.rb +92 -0
  26. data/lib/gitlab_quality/test_tooling/runtime/token_finder.rb +44 -0
  27. data/lib/gitlab_quality/test_tooling/slack/post_to_slack.rb +36 -0
  28. data/lib/gitlab_quality/test_tooling/summary_table.rb +41 -0
  29. data/lib/gitlab_quality/test_tooling/support/http_request.rb +34 -0
  30. data/lib/gitlab_quality/test_tooling/system_logs/finders/json_log_finder.rb +65 -0
  31. data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/api_log_finder.rb +21 -0
  32. data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/application_log_finder.rb +21 -0
  33. data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/exception_log_finder.rb +21 -0
  34. data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/graphql_log_finder.rb +21 -0
  35. data/lib/gitlab_quality/test_tooling/system_logs/log_types/log.rb +38 -0
  36. data/lib/gitlab_quality/test_tooling/system_logs/log_types/rails/api_log.rb +34 -0
  37. data/lib/gitlab_quality/test_tooling/system_logs/log_types/rails/application_log.rb +27 -0
  38. data/lib/gitlab_quality/test_tooling/system_logs/log_types/rails/exception_log.rb +23 -0
  39. data/lib/gitlab_quality/test_tooling/system_logs/log_types/rails/graphql_log.rb +30 -0
  40. data/lib/gitlab_quality/test_tooling/system_logs/shared_fields.rb +29 -0
  41. data/lib/gitlab_quality/test_tooling/system_logs/system_logs_formatter.rb +65 -0
  42. data/lib/gitlab_quality/test_tooling/test_results/base_test_results.rb +39 -0
  43. data/lib/gitlab_quality/test_tooling/test_results/builder.rb +35 -0
  44. data/lib/gitlab_quality/test_tooling/test_results/j_unit_test_results.rb +27 -0
  45. data/lib/gitlab_quality/test_tooling/test_results/json_test_results.rb +29 -0
  46. data/lib/gitlab_quality/test_tooling/test_results/test_result.rb +184 -0
  47. data/lib/gitlab_quality/test_tooling/version.rb +1 -1
  48. data/lib/gitlab_quality/test_tooling.rb +11 -2
  49. metadata +51 -3
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_support/core_ext/enumerable'
4
+
5
+ module GitlabQuality
6
+ module TestTooling
7
+ module Report
8
+ module Concerns
9
+ module ResultsReporter
10
+ include Concerns::Utils
11
+
12
+ TEST_CASE_RESULTS_SECTION_TEMPLATE = "\n\n### DO NOT EDIT BELOW THIS LINE\n\nActive and historical test results:"
13
+
14
+ def find_issue(test)
15
+ issues = search_for_issues(test)
16
+
17
+ warn(%(Too many #{issue_type}s found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
18
+
19
+ puts "Found existing #{issue_type}: #{issues.first.web_url}" unless issues.empty?
20
+
21
+ issues.first
22
+ end
23
+
24
+ def find_issue_by_iid(iid)
25
+ issues = gitlab.find_issues(iid: iid) do |issue|
26
+ issue.state == 'opened' && issue.issue_type == issue_type
27
+ end
28
+
29
+ warn(%(#{issue_type} iid "#{iid}" not valid)) if issues.empty?
30
+
31
+ issues.first
32
+ end
33
+
34
+ def issue_title_needs_updating?(issue, test)
35
+ issue.title.strip != title_from_test(test) && !%w[canary production preprod release].include?(pipeline)
36
+ end
37
+
38
+ def new_issue_labels(_test)
39
+ %w[Quality status::automated]
40
+ end
41
+
42
+ def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
43
+ labels = super
44
+ labels |= new_issue_labels(test).to_set
45
+ labels.delete_if { |label| label.start_with?("#{pipeline}::") }
46
+ labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
47
+ end
48
+
49
+ def update_issue_title(issue, test)
50
+ old_title = issue.title.strip
51
+ new_title = title_from_test(test)
52
+
53
+ warn(%(#{issue_type} title needs to be updated from '#{old_title}' to '#{new_title}'))
54
+
55
+ new_description = updated_description(issue, test)
56
+
57
+ gitlab.edit_issue(iid: issue.iid, options: { title: new_title, description: new_description })
58
+ end
59
+
60
+ private
61
+
62
+ def search_term(test)
63
+ %("#{partial_file_path(test.file)}" "#{search_safe(test.name)}")
64
+ end
65
+
66
+ def search_for_issues(test)
67
+ gitlab.find_issues(options: { search: search_term(test) }) do |issue|
68
+ issue.state == 'opened' && issue.issue_type == issue_type && issue.title.strip == title_from_test(test)
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module GitlabQuality
4
+ module TestTooling
5
+ module Report
6
+ module Concerns
7
+ module Utils
8
+ MAX_TITLE_LENGTH = 255
9
+
10
+ def partial_file_path(path)
11
+ path.match(/((ee|api|browser_ui).*)/i)[1]
12
+ end
13
+
14
+ def new_issue_title(test)
15
+ "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
16
+ end
17
+
18
+ def title_from_test(test)
19
+ title = new_issue_title(test)
20
+
21
+ return title unless title.length > MAX_TITLE_LENGTH
22
+
23
+ "#{title[0...MAX_TITLE_LENGTH - 3]}..."
24
+ end
25
+
26
+ def search_safe(value)
27
+ value.delete('"')
28
+ end
29
+
30
+ def pipeline
31
+ # Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
32
+ #
33
+ # Tests can be run in several pipelines:
34
+ # gitlab, nightly, staging, canary, production, preprod, MRs, and the default branch (master/main)
35
+ #
36
+ # Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
37
+ # nightly, staging, canary, production, and preprod
38
+ #
39
+ # MR, master/main, and gitlab tests run in gitlab-qa, but we only want to report tests run on
40
+ # master/main because the other pipelines will be monitored by the author of the MR that triggered them.
41
+ # So we assume that we're reporting a master/main pipeline if the project name is 'gitlab'.
42
+
43
+ @pipeline ||= Runtime::Env.pipeline_from_project_name
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,275 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'erb'
4
+ require 'date'
5
+
6
+ module GitlabQuality
7
+ module TestTooling
8
+ module Report
9
+ class GenerateTestSession < ReportAsIssue
10
+ def initialize(**kwargs)
11
+ super
12
+ @issue_type = 'issue'
13
+ end
14
+
15
+ private
16
+
17
+ # rubocop:disable Metrics/AbcSize
18
+ def run!
19
+ puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
20
+
21
+ tests = Dir.glob(files).flat_map do |path|
22
+ puts "Loading tests in #{path}"
23
+
24
+ TestResults::JsonTestResults.new(path).to_a
25
+ end
26
+
27
+ issue = gitlab.create_issue(
28
+ title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
29
+ description: generate_description(tests),
30
+ labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
31
+ )
32
+
33
+ # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/295493
34
+ unless Runtime::Env.qa_issue_url.to_s.empty?
35
+ gitlab.create_issue_note(
36
+ iid: issue.iid,
37
+ note: "/relate #{Runtime::Env.qa_issue_url}")
38
+ end
39
+
40
+ File.write('REPORT_ISSUE_URL', issue.web_url)
41
+ end
42
+ # rubocop:enable Metrics/AbcSize
43
+
44
+ def generate_description(tests)
45
+ <<~MARKDOWN.rstrip
46
+ ## Session summary
47
+
48
+ * Deploy version: #{Runtime::Env.deploy_version}
49
+ * Deploy environment: #{Runtime::Env.deploy_environment}
50
+ * Pipeline: #{Runtime::Env.pipeline_from_project_name} [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
51
+ #{generate_summary(tests: tests)}
52
+
53
+ #{generate_failed_jobs_listing}
54
+
55
+ #{generate_stages_listing(tests)}
56
+
57
+ #{generate_qa_issue_relation}
58
+
59
+ #{generate_link_to_dashboard}
60
+ MARKDOWN
61
+ end
62
+
63
+ def generate_summary(tests:, tests_by_status: nil)
64
+ tests_by_status ||= tests.group_by(&:status)
65
+ total = tests.size
66
+ passed = tests_by_status['passed']&.size || 0
67
+ failed = tests_by_status['failed']&.size || 0
68
+ others = total - passed - failed
69
+
70
+ <<~MARKDOWN.chomp
71
+ * Total #{total} tests
72
+ * Passed #{passed} tests
73
+ * Failed #{failed} tests
74
+ * #{others} other tests (usually skipped)
75
+ MARKDOWN
76
+ end
77
+
78
+ def generate_failed_jobs_listing
79
+ failed_jobs = []
80
+
81
+ client = Gitlab.client(
82
+ endpoint: Runtime::Env.ci_api_v4_url,
83
+ private_token: Runtime::Env.gitlab_ci_api_token)
84
+
85
+ gitlab.handle_gitlab_client_exceptions do
86
+ failed_jobs = client.pipeline_jobs(
87
+ Runtime::Env.ci_project_id,
88
+ Runtime::Env.ci_pipeline_id,
89
+ scope: 'failed')
90
+ end
91
+
92
+ listings = failed_jobs.map do |job|
93
+ allowed_to_fail = ' (allowed to fail)' if job.allow_failure
94
+
95
+ "* [#{job.name}](#{job.web_url})#{allowed_to_fail}"
96
+ end.join("\n")
97
+
98
+ <<~MARKDOWN.chomp if failed_jobs.any?
99
+ ## Failed jobs
100
+
101
+ #{listings}
102
+ MARKDOWN
103
+ end
104
+
105
+ def generate_stages_listing(tests)
106
+ generate_tests_by_stage(tests).map do |stage, tests_for_stage|
107
+ tests_by_status = tests_for_stage.group_by(&:status)
108
+
109
+ <<~MARKDOWN.chomp
110
+ ### #{stage&.capitalize || 'Unknown'}
111
+
112
+ #{generate_summary(
113
+ tests: tests_for_stage, tests_by_status: tests_by_status)}
114
+
115
+ #{generate_testcase_listing_by_status(
116
+ tests: tests_for_stage, tests_by_status: tests_by_status)}
117
+ MARKDOWN
118
+ end.join("\n\n")
119
+ end
120
+
121
+ def generate_tests_by_stage(tests)
122
+ # https://about.gitlab.com/handbook/product/product-categories/#devops-stages
123
+ ordering = %w[
124
+ manage
125
+ plan
126
+ create
127
+ verify
128
+ package
129
+ release
130
+ configure
131
+ monitor
132
+ secure
133
+ defend
134
+ growth
135
+ fulfillment
136
+ enablement
137
+ ]
138
+
139
+ tests.sort_by do |test|
140
+ ordering.index(test.stage) || ordering.size
141
+ end.group_by(&:stage)
142
+ end
143
+
144
+ def generate_testcase_listing_by_status(tests:, tests_by_status:)
145
+ failed_tests = tests_by_status['failed']
146
+ passed_tests = tests_by_status['passed']
147
+ other_tests = tests.reject do |test|
148
+ test.status == 'failed' || test.status == 'passed'
149
+ end
150
+
151
+ [
152
+ (failed_listings(failed_tests) if failed_tests),
153
+ (passed_listings(passed_tests) if passed_tests),
154
+ (other_listings(other_tests) if other_tests.any?)
155
+ ].compact.join("\n\n")
156
+ end
157
+
158
+ def failed_listings(failed_tests)
159
+ generate_testcase_listing(failed_tests)
160
+ end
161
+
162
+ def passed_listings(passed_tests)
163
+ <<~MARKDOWN.chomp
164
+ <details><summary>Passed tests:</summary>
165
+
166
+ #{generate_testcase_listing(passed_tests, passed: true)}
167
+
168
+ </details>
169
+ MARKDOWN
170
+ end
171
+
172
+ def other_listings(other_tests)
173
+ <<~MARKDOWN.chomp
174
+ <details><summary>Other tests:</summary>
175
+
176
+ #{generate_testcase_listing(other_tests)}
177
+
178
+ </details>
179
+ MARKDOWN
180
+ end
181
+
182
+ def generate_testcase_listing(tests, passed: false)
183
+ body = tests.group_by(&:testcase).map do |testcase, tests_with_same_testcase|
184
+ tests_with_same_testcase.sort_by!(&:name)
185
+ [
186
+ generate_test_text(testcase, tests_with_same_testcase, passed),
187
+ generate_test_job(tests_with_same_testcase),
188
+ generate_test_status(tests_with_same_testcase),
189
+ generate_test_actions(tests_with_same_testcase)
190
+ ].join(' | ')
191
+ end.join("\n")
192
+
193
+ <<~MARKDOWN.chomp
194
+ | Test | Job | Status | Action |
195
+ | - | - | - | - |
196
+ #{body}
197
+ MARKDOWN
198
+ end
199
+
200
+ def generate_test_text(testcase, tests_with_same_testcase, passed)
201
+ text = tests_with_same_testcase.map(&:name).uniq.join(', ')
202
+ encoded_text = ERB::Util.url_encode(text)
203
+
204
+ if testcase && !passed
205
+ # Workaround for reducing system notes on testcase issues
206
+ # The first regex extracts the link to the issues list page from a link to a single issue show page by removing the issue id.
207
+ "[#{text}](#{testcase.match(%r{[\s\S]+/[^/\d]+})}?state=opened&search=#{encoded_text})"
208
+ else
209
+ text
210
+ end
211
+ end
212
+
213
+ def generate_test_job(tests_with_same_testcase)
214
+ tests_with_same_testcase.map do |test|
215
+ ci_job_id = test.ci_job_url[/\d+\z/]
216
+
217
+ "[#{ci_job_id}](#{test.ci_job_url})#{' ~"quarantine"' if test.quarantine?}"
218
+ end.uniq.join(', ')
219
+ end
220
+
221
+ def generate_test_status(tests_with_same_testcase)
222
+ tests_with_same_testcase.map(&:status).uniq.map do |status|
223
+ %(~"#{status}")
224
+ end.join(', ')
225
+ end
226
+
227
+ def generate_test_actions(tests_with_same_testcase)
228
+ # All failed tests would be grouped together, meaning that
229
+ # if one failed, all the tests here would be failed too.
230
+ # So this check is safe. Same applies to 'passed'.
231
+ # But all other status might be mixing together,
232
+ # we cannot assume other statuses.
233
+ if tests_with_same_testcase.first.status == 'failed'
234
+ tests_having_failure_issue =
235
+ tests_with_same_testcase.select(&:failure_issue)
236
+
237
+ if tests_having_failure_issue.any?
238
+ items = tests_having_failure_issue.uniq(&:failure_issue).map do |test|
239
+ "<li>[ ] [failure issue](#{test.failure_issue})</li>"
240
+ end.join(' ')
241
+
242
+ "<ul>#{items}</ul>"
243
+ else
244
+ '<ul><li>[ ] failure issue exists or was created</li></ul>'
245
+ end
246
+ else
247
+ '-'
248
+ end
249
+ end
250
+
251
+ def generate_qa_issue_relation
252
+ return unless Runtime::Env.qa_issue_url
253
+
254
+ <<~MARKDOWN.chomp
255
+ ## Release QA issue
256
+
257
+ * #{Runtime::Env.qa_issue_url}
258
+
259
+ /relate #{Runtime::Env.qa_issue_url}
260
+ MARKDOWN
261
+ end
262
+
263
+ def generate_link_to_dashboard
264
+ return unless Runtime::Env.qa_run_type
265
+
266
+ <<~MARKDOWN.chomp
267
+ ## Link to Grafana dashboard for run-type of #{Runtime::Env.qa_run_type}
268
+
269
+ * https://dashboards.quality.gitlab.net/d/kuNYMgDnz/test-run-metrics?orgId=1&refresh=1m&var-run_type=#{Runtime::Env.qa_run_type}
270
+ MARKDOWN
271
+ end
272
+ end
273
+ end
274
+ end
275
+ end
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'nokogiri'
4
+
5
+ module GitlabQuality
6
+ module TestTooling
7
+ module Report
8
+ # Create a new JUnit report file for each Stage, containing tests from that Stage alone
9
+ class PrepareStageReports
10
+ EXTRACT_STAGE_FROM_TEST_FILE_REGEX = %r{(?:api|browser_ui)/(?:[0-9]+_)?(?<stage>[_\w]+)/}i
11
+
12
+ def initialize(junit_files:)
13
+ @junit_files = junit_files
14
+ end
15
+
16
+ def invoke!
17
+ collate_test_cases.each do |stage, tests|
18
+ filename = "#{stage}.xml"
19
+
20
+ File.write(filename, junit_report(tests).to_s)
21
+
22
+ puts "Saved #{filename}"
23
+ end
24
+ end
25
+
26
+ private
27
+
28
+ attr_reader :junit_files
29
+
30
+ # Collect the test cases from the original reports and group them by Stage
31
+ def collate_test_cases
32
+ Dir.glob(junit_files)
33
+ .each_with_object(Hash.new { |h, k| h[k] = [] }) do |junit_file, test_cases|
34
+ report = Nokogiri::XML(File.open(junit_file))
35
+ report.xpath('//testcase').each do |test_case|
36
+ # The test file paths could start with any of
37
+ # /qa/specs/features/api/<stage>
38
+ # /qa/specs/features/browser_ui/<stage>
39
+ # /qa/specs/features/ee/api/<stage>
40
+ # /qa/specs/features/ee/browser_ui/<stage>
41
+ # For now we assume the Stage is whatever follows api/ or browser_ui/
42
+ test_file_match = test_case['file'].match(EXTRACT_STAGE_FROM_TEST_FILE_REGEX)
43
+ next unless test_file_match
44
+
45
+ stage = test_file_match[:stage]
46
+ test_cases[stage] << test_case
47
+ end
48
+ end
49
+ end
50
+
51
+ def junit_report(test_cases)
52
+ Nokogiri::XML::Document.new.tap do |report|
53
+ test_suite_node = report.create_element('testsuite', name: 'rspec', **collect_stats(test_cases))
54
+ report.root = test_suite_node
55
+
56
+ test_cases.each do |test_case|
57
+ test_suite_node.add_child(test_case)
58
+ end
59
+ end
60
+ end
61
+
62
+ def collect_stats(test_cases)
63
+ stats = {
64
+ tests: test_cases.size,
65
+ failures: 0,
66
+ errors: 0,
67
+ skipped: 0
68
+ }
69
+
70
+ test_cases.each_with_object(stats) do |test_case, memo|
71
+ memo[:failures] += 1 unless test_case.search('failure').empty?
72
+ memo[:errors] += 1 unless test_case.search('error').empty?
73
+ memo[:skipped] += 1 unless test_case.search('skipped').empty?
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end