gitlab-qa 6.7.0 → 6.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +28 -2
- data/lib/gitlab/qa.rb +3 -0
- data/lib/gitlab/qa/report/base_test_results.rb +6 -3
- data/lib/gitlab/qa/report/generate_test_session.rb +203 -0
- data/lib/gitlab/qa/report/gitlab_issue_client.rb +21 -7
- data/lib/gitlab/qa/report/gitlab_issue_dry_client.rb +28 -0
- data/lib/gitlab/qa/report/json_test_results.rb +2 -2
- data/lib/gitlab/qa/report/junit_test_results.rb +2 -2
- data/lib/gitlab/qa/report/relate_failure_issue.rb +139 -0
- data/lib/gitlab/qa/report/report_as_issue.rb +101 -3
- data/lib/gitlab/qa/report/results_in_issues.rb +16 -84
- data/lib/gitlab/qa/report/test_result.rb +22 -1
- data/lib/gitlab/qa/reporter.rb +30 -2
- data/lib/gitlab/qa/runtime/env.rb +29 -3
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: daf050b7d871f8eefdc1fa68b052608c11ee186b927fe89accfceca15291ed45
|
4
|
+
data.tar.gz: 4510e98db8e41c11a1585070a39cd19769384917cff551195942955974dcd78d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3a3a6dc22aac1a77fe46f1379a43b489d08cc771b38ea629f9ff3a8f24fd9886f767ed3b3ca8aa18bfeabadc9019aefe2933261c969167e6c8e95f7f75e2c6a1
|
7
|
+
data.tar.gz: c612e3b42691a28864bcf703fdf89cbbf57f75817341f867d1e8486a25adcbfcf262f3506157841d734813e699e628620dd524af22b4f8da17c689a96c6116cf
|
data/.gitlab-ci.yml
CHANGED
@@ -2,6 +2,7 @@ stages:
|
|
2
2
|
- check
|
3
3
|
- release
|
4
4
|
- test
|
5
|
+
- report
|
5
6
|
- notify
|
6
7
|
|
7
8
|
default:
|
@@ -48,6 +49,10 @@ variables:
|
|
48
49
|
QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
|
49
50
|
QA_CAN_TEST_PRAEFECT: "false"
|
50
51
|
QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
|
52
|
+
QA_FAILURES_REPORTING_PROJECT: "gitlab-org/gitlab"
|
53
|
+
# The --dry-run or --max-diff-ratio option can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
|
54
|
+
QA_FAILURES_REPORTER_OPTIONS: "--dry-run"
|
55
|
+
QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
|
51
56
|
|
52
57
|
.check-base:
|
53
58
|
stage: check
|
@@ -95,6 +100,7 @@ release:
|
|
95
100
|
- exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
96
101
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
97
102
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
103
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
98
104
|
- exit $test_run_exit_code
|
99
105
|
|
100
106
|
.ce-qa:
|
@@ -333,6 +339,7 @@ ce:update:
|
|
333
339
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
334
340
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
335
341
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
342
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
336
343
|
- exit $test_run_exit_code
|
337
344
|
extends:
|
338
345
|
- .test
|
@@ -347,6 +354,7 @@ ce:update-quarantine:
|
|
347
354
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
348
355
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
349
356
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
357
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
350
358
|
- exit $test_run_exit_code
|
351
359
|
extends:
|
352
360
|
- .test
|
@@ -360,6 +368,7 @@ ee:update:
|
|
360
368
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
361
369
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
362
370
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
371
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
363
372
|
- exit $test_run_exit_code
|
364
373
|
extends:
|
365
374
|
- .test
|
@@ -374,6 +383,7 @@ ee:update-quarantine:
|
|
374
383
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
375
384
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
376
385
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
386
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
377
387
|
- exit $test_run_exit_code
|
378
388
|
extends:
|
379
389
|
- .test
|
@@ -941,6 +951,22 @@ staging:
|
|
941
951
|
- .only-qa
|
942
952
|
allow_failure: true
|
943
953
|
|
954
|
+
generate_test_session:
|
955
|
+
stage: report
|
956
|
+
rules:
|
957
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == "master"'
|
958
|
+
when: always
|
959
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB =~ /\Ahttps:\/\/ops.gitlab.net\//'
|
960
|
+
when: always
|
961
|
+
artifacts:
|
962
|
+
when: always
|
963
|
+
expire_in: 10d
|
964
|
+
paths:
|
965
|
+
- REPORT_ISSUE_URL
|
966
|
+
script:
|
967
|
+
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
968
|
+
- exe/gitlab-qa-report --generate-test-session "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASE_SESSIONS_PROJECT"
|
969
|
+
|
944
970
|
.notify_upstream_commit:
|
945
971
|
stage: notify
|
946
972
|
image: ruby:2.6
|
@@ -966,7 +992,7 @@ notify_upstream_commit:failure:
|
|
966
992
|
.notify_slack:
|
967
993
|
image: alpine
|
968
994
|
stage: notify
|
969
|
-
dependencies: []
|
995
|
+
dependencies: ['generate_test_session']
|
970
996
|
cache: {}
|
971
997
|
before_script:
|
972
998
|
- apk update && apk add git curl bash
|
@@ -985,4 +1011,4 @@ notify_slack:
|
|
985
1011
|
- echo "RELEASE is ${RELEASE}"
|
986
1012
|
- echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
|
987
1013
|
- echo "TOP_UPSTREAM_SOURCE_JOB is $TOP_UPSTREAM_SOURCE_JOB"
|
988
|
-
- bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing
|
1014
|
+
- 'bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See the test session report: $(cat REPORT_ISSUE_URL), and pipeline: $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing'
|
data/lib/gitlab/qa.rb
CHANGED
@@ -99,12 +99,15 @@ module Gitlab
|
|
99
99
|
|
100
100
|
module Report
|
101
101
|
autoload :GitlabIssueClient, 'gitlab/qa/report/gitlab_issue_client'
|
102
|
+
autoload :GitlabIssueDryClient, 'gitlab/qa/report/gitlab_issue_dry_client'
|
102
103
|
autoload :BaseTestResults, 'gitlab/qa/report/base_test_results'
|
104
|
+
autoload :RelateFailureIssue, 'gitlab/qa/report/relate_failure_issue'
|
103
105
|
autoload :JsonTestResults, 'gitlab/qa/report/json_test_results'
|
104
106
|
autoload :JUnitTestResults, 'gitlab/qa/report/junit_test_results'
|
105
107
|
autoload :PrepareStageReports, 'gitlab/qa/report/prepare_stage_reports'
|
106
108
|
autoload :ReportAsIssue, 'gitlab/qa/report/report_as_issue'
|
107
109
|
autoload :ResultsInIssues, 'gitlab/qa/report/results_in_issues'
|
110
|
+
autoload :GenerateTestSession, 'gitlab/qa/report/generate_test_session'
|
108
111
|
autoload :SummaryTable, 'gitlab/qa/report/summary_table'
|
109
112
|
autoload :TestResult, 'gitlab/qa/report/test_result'
|
110
113
|
autoload :UpdateScreenshotPath, 'gitlab/qa/report/update_screenshot_path'
|
@@ -6,8 +6,11 @@ module Gitlab
|
|
6
6
|
class BaseTestResults
|
7
7
|
include Enumerable
|
8
8
|
|
9
|
+
attr_reader :path
|
10
|
+
|
9
11
|
def initialize(path)
|
10
|
-
@
|
12
|
+
@path = path
|
13
|
+
@results = parse
|
11
14
|
@testcases = process
|
12
15
|
end
|
13
16
|
|
@@ -15,7 +18,7 @@ module Gitlab
|
|
15
18
|
testcases.each(&block)
|
16
19
|
end
|
17
20
|
|
18
|
-
def write
|
21
|
+
def write
|
19
22
|
raise NotImplementedError
|
20
23
|
end
|
21
24
|
|
@@ -23,7 +26,7 @@ module Gitlab
|
|
23
26
|
|
24
27
|
attr_reader :results, :testcases
|
25
28
|
|
26
|
-
def parse
|
29
|
+
def parse
|
27
30
|
raise NotImplementedError
|
28
31
|
end
|
29
32
|
|
@@ -0,0 +1,203 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Report
|
6
|
+
class GenerateTestSession < ReportAsIssue
|
7
|
+
private
|
8
|
+
|
9
|
+
def run!
|
10
|
+
puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
11
|
+
|
12
|
+
tests = Dir.glob(files).flat_map do |path|
|
13
|
+
puts "Loading tests in #{path}"
|
14
|
+
|
15
|
+
Report::JsonTestResults.new(path).to_a
|
16
|
+
end
|
17
|
+
|
18
|
+
issue = gitlab.create_issue(
|
19
|
+
title: "Test session report | #{Runtime::Env.deploy_environment}",
|
20
|
+
description: generate_description(tests),
|
21
|
+
labels: ['Quality', 'QA', 'triage report']
|
22
|
+
)
|
23
|
+
|
24
|
+
File.write('REPORT_ISSUE_URL', issue.web_url)
|
25
|
+
end
|
26
|
+
|
27
|
+
def generate_description(tests)
|
28
|
+
<<~MARKDOWN
|
29
|
+
## Session summary
|
30
|
+
|
31
|
+
* Deploy version: #{Runtime::Env.deploy_version}
|
32
|
+
* Pipeline: [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
|
33
|
+
#{generate_summary(tests: tests)}
|
34
|
+
|
35
|
+
#{generate_stages_listing(tests)}
|
36
|
+
|
37
|
+
## Release QA issue
|
38
|
+
|
39
|
+
* #{Runtime::Env.qa_issue_url}
|
40
|
+
MARKDOWN
|
41
|
+
end
|
42
|
+
|
43
|
+
def generate_summary(tests:, tests_by_status: nil)
|
44
|
+
tests_by_status ||= tests.group_by(&:status)
|
45
|
+
total = tests.size
|
46
|
+
passed = tests_by_status['passed']&.size || 0
|
47
|
+
failed = tests_by_status['failed']&.size || 0
|
48
|
+
others = total - passed - failed
|
49
|
+
|
50
|
+
<<~MARKDOWN.chomp
|
51
|
+
* Total #{total} tests
|
52
|
+
* Passed #{passed} tests
|
53
|
+
* Failed #{failed} tests
|
54
|
+
* #{others} other tests (usually skipped)
|
55
|
+
MARKDOWN
|
56
|
+
end
|
57
|
+
|
58
|
+
def generate_stages_listing(tests)
|
59
|
+
generate_tests_by_stage(tests).map do |stage, tests_for_stage|
|
60
|
+
tests_by_status = tests_for_stage.group_by(&:status)
|
61
|
+
|
62
|
+
<<~MARKDOWN.chomp
|
63
|
+
### #{stage&.capitalize || 'Unknown'}
|
64
|
+
|
65
|
+
#{generate_summary(
|
66
|
+
tests: tests_for_stage, tests_by_status: tests_by_status)}
|
67
|
+
|
68
|
+
#{generate_testcase_listing_by_status(
|
69
|
+
tests: tests_for_stage, tests_by_status: tests_by_status)}
|
70
|
+
MARKDOWN
|
71
|
+
end.join("\n\n")
|
72
|
+
end
|
73
|
+
|
74
|
+
def generate_tests_by_stage(tests)
|
75
|
+
# https://about.gitlab.com/handbook/product/product-categories/#devops-stages
|
76
|
+
ordering = %w[
|
77
|
+
manage
|
78
|
+
plan
|
79
|
+
create
|
80
|
+
verify
|
81
|
+
package
|
82
|
+
release
|
83
|
+
configure
|
84
|
+
monitor
|
85
|
+
secure
|
86
|
+
defend
|
87
|
+
growth
|
88
|
+
fulfillment
|
89
|
+
enablement
|
90
|
+
]
|
91
|
+
|
92
|
+
tests.sort_by do |test|
|
93
|
+
ordering.index(test.stage) || ordering.size
|
94
|
+
end.group_by(&:stage)
|
95
|
+
end
|
96
|
+
|
97
|
+
def generate_testcase_listing_by_status(tests:, tests_by_status:)
|
98
|
+
failed_tests = tests_by_status['failed']
|
99
|
+
passed_tests = tests_by_status['passed']
|
100
|
+
other_tests = tests.reject do |test|
|
101
|
+
test.status == 'failed' || test.status == 'passed'
|
102
|
+
end
|
103
|
+
|
104
|
+
[
|
105
|
+
(failed_listings(failed_tests) if failed_tests),
|
106
|
+
(passed_listings(passed_tests) if passed_tests),
|
107
|
+
(other_listings(other_tests) if other_tests.any?)
|
108
|
+
].compact.join("\n\n")
|
109
|
+
end
|
110
|
+
|
111
|
+
def failed_listings(failed_tests)
|
112
|
+
generate_testcase_listing(failed_tests)
|
113
|
+
end
|
114
|
+
|
115
|
+
def passed_listings(passed_tests)
|
116
|
+
<<~MARKDOWN.chomp
|
117
|
+
<details><summary>Passed tests:</summary>
|
118
|
+
|
119
|
+
#{generate_testcase_listing(passed_tests)}
|
120
|
+
|
121
|
+
</details>
|
122
|
+
MARKDOWN
|
123
|
+
end
|
124
|
+
|
125
|
+
def other_listings(other_tests)
|
126
|
+
<<~MARKDOWN.chomp
|
127
|
+
<details><summary>Other tests:</summary>
|
128
|
+
|
129
|
+
#{generate_testcase_listing(other_tests)}
|
130
|
+
|
131
|
+
</details>
|
132
|
+
MARKDOWN
|
133
|
+
end
|
134
|
+
|
135
|
+
def generate_testcase_listing(tests)
|
136
|
+
body = tests.group_by(&:testcase).map do |testcase, tests_with_same_testcase|
|
137
|
+
tests_with_same_testcase.sort_by!(&:name)
|
138
|
+
[
|
139
|
+
generate_test_text(testcase, tests_with_same_testcase),
|
140
|
+
generate_test_job(tests_with_same_testcase),
|
141
|
+
generate_test_status(tests_with_same_testcase),
|
142
|
+
generate_test_actions(tests_with_same_testcase)
|
143
|
+
].join(' | ')
|
144
|
+
end.join("\n")
|
145
|
+
|
146
|
+
<<~MARKDOWN.chomp
|
147
|
+
| Test | Job | Status | Action |
|
148
|
+
| - | - | - | - |
|
149
|
+
#{body}
|
150
|
+
MARKDOWN
|
151
|
+
end
|
152
|
+
|
153
|
+
def generate_test_text(testcase, tests_with_same_testcase)
|
154
|
+
text = tests_with_same_testcase.map(&:name).uniq.join(', ')
|
155
|
+
|
156
|
+
if testcase
|
157
|
+
"[#{text}](#{testcase})"
|
158
|
+
else
|
159
|
+
text
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
def generate_test_job(tests_with_same_testcase)
|
164
|
+
tests_with_same_testcase.map(&:ci_job_url).uniq.map do |ci_job_url|
|
165
|
+
ci_job_id = ci_job_url[/\d+\z/]
|
166
|
+
|
167
|
+
"[#{ci_job_id}](#{ci_job_url})"
|
168
|
+
end.join(', ')
|
169
|
+
end
|
170
|
+
|
171
|
+
def generate_test_status(tests_with_same_testcase)
|
172
|
+
tests_with_same_testcase.map(&:status).uniq.map do |status|
|
173
|
+
%(~"#{status}")
|
174
|
+
end.join(', ')
|
175
|
+
end
|
176
|
+
|
177
|
+
def generate_test_actions(tests_with_same_testcase)
|
178
|
+
# All failed tests would be grouped together, meaning that
|
179
|
+
# if one failed, all the tests here would be failed too.
|
180
|
+
# So this check is safe. Same applies to 'passed'.
|
181
|
+
# But all other status might be mixing together,
|
182
|
+
# we cannot assume other statuses.
|
183
|
+
if tests_with_same_testcase.first.status == 'failed'
|
184
|
+
tests_having_failure_issue =
|
185
|
+
tests_with_same_testcase.select(&:failure_issue)
|
186
|
+
|
187
|
+
if tests_having_failure_issue.any?
|
188
|
+
items = tests_having_failure_issue.map do |test|
|
189
|
+
"<li>[ ] [failure issue](#{test.failure_issue})</li>"
|
190
|
+
end.join(' ')
|
191
|
+
|
192
|
+
"<ul>#{items}</ul>"
|
193
|
+
else
|
194
|
+
'<ul><li>[ ] failure issue exists or was created</li></ul>'
|
195
|
+
end
|
196
|
+
else
|
197
|
+
'-'
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
@@ -45,7 +45,7 @@ module Gitlab
|
|
45
45
|
abort_not_permitted
|
46
46
|
end
|
47
47
|
|
48
|
-
def find_issues(iid
|
48
|
+
def find_issues(iid: nil, options: {}, &select)
|
49
49
|
select ||= :itself
|
50
50
|
|
51
51
|
handle_gitlab_client_exceptions do
|
@@ -57,15 +57,17 @@ module Gitlab
|
|
57
57
|
end
|
58
58
|
end
|
59
59
|
|
60
|
+
def find_issue_discussions(iid:)
|
61
|
+
handle_gitlab_client_exceptions do
|
62
|
+
Gitlab.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc')
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
60
66
|
def create_issue(title:, description:, labels:)
|
61
|
-
|
67
|
+
attrs = { description: description, labels: labels }
|
62
68
|
|
63
69
|
handle_gitlab_client_exceptions do
|
64
|
-
Gitlab.create_issue(
|
65
|
-
project,
|
66
|
-
title,
|
67
|
-
{ description: description, labels: labels }
|
68
|
-
)
|
70
|
+
Gitlab.create_issue(project, title, attrs)
|
69
71
|
end
|
70
72
|
end
|
71
73
|
|
@@ -75,6 +77,18 @@ module Gitlab
|
|
75
77
|
end
|
76
78
|
end
|
77
79
|
|
80
|
+
def create_issue_note(iid:, note:)
|
81
|
+
handle_gitlab_client_exceptions do
|
82
|
+
Gitlab.create_issue_note(project, iid, note)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
|
87
|
+
handle_gitlab_client_exceptions do
|
88
|
+
Gitlab.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: body)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
78
92
|
def handle_gitlab_client_exceptions
|
79
93
|
yield
|
80
94
|
rescue Gitlab::Error::NotFound
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Report
|
6
|
+
class GitlabIssueDryClient < GitlabIssueClient
|
7
|
+
def create_issue(title:, description:, labels:)
|
8
|
+
attrs = { description: description, labels: labels }
|
9
|
+
|
10
|
+
puts "The following issue would have been created:"
|
11
|
+
puts "project: #{project}, title: #{title}, attrs: #{attrs}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def edit_issue(iid:, options: {})
|
15
|
+
puts "The #{project}##{iid} issue would have been updated with: #{options}"
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_issue_note(iid:, note:)
|
19
|
+
puts "The following note would have been posted on #{project}##{iid} issue: #{note}"
|
20
|
+
end
|
21
|
+
|
22
|
+
def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
|
23
|
+
puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{body}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -6,7 +6,7 @@ module Gitlab
|
|
6
6
|
module QA
|
7
7
|
module Report
|
8
8
|
class JsonTestResults < BaseTestResults
|
9
|
-
def write
|
9
|
+
def write
|
10
10
|
json = results.merge('examples' => testcases.map(&:report))
|
11
11
|
|
12
12
|
File.write(path, JSON.pretty_generate(json))
|
@@ -14,7 +14,7 @@ module Gitlab
|
|
14
14
|
|
15
15
|
private
|
16
16
|
|
17
|
-
def parse
|
17
|
+
def parse
|
18
18
|
JSON.parse(File.read(path))
|
19
19
|
end
|
20
20
|
|
@@ -0,0 +1,139 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'nokogiri'
|
4
|
+
require 'active_support/core_ext/enumerable'
|
5
|
+
require 'rubygems/text'
|
6
|
+
|
7
|
+
module Gitlab
|
8
|
+
module QA
|
9
|
+
module Report
|
10
|
+
# Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
|
11
|
+
class RelateFailureIssue < ReportAsIssue
|
12
|
+
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
|
13
|
+
STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error: .+)(\1)}m.freeze
|
14
|
+
NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
|
15
|
+
|
16
|
+
MultipleIssuesFound = Class.new(StandardError)
|
17
|
+
|
18
|
+
def initialize(max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION, **kwargs)
|
19
|
+
super
|
20
|
+
@max_diff_ratio = max_diff_ratio.to_f
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
attr_reader :max_diff_ratio
|
26
|
+
|
27
|
+
def run!
|
28
|
+
puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
29
|
+
|
30
|
+
test_results_per_file do |test_results|
|
31
|
+
puts "=> Reporting tests in #{test_results.path}"
|
32
|
+
|
33
|
+
test_results.each do |test|
|
34
|
+
next if test.failures.empty?
|
35
|
+
|
36
|
+
relate_test_to_issue(test)
|
37
|
+
end
|
38
|
+
|
39
|
+
test_results.write
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def relate_test_to_issue(test)
|
44
|
+
puts " => Searching issues for test '#{test.name}'..."
|
45
|
+
|
46
|
+
begin
|
47
|
+
issue = find_or_create_issue(test)
|
48
|
+
return unless issue
|
49
|
+
|
50
|
+
post_failed_job_note(issue, test)
|
51
|
+
puts " => Marked #{issue.web_url} as related to #{test.testcase}."
|
52
|
+
rescue MultipleIssuesFound => e
|
53
|
+
warn(e.message)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def find_or_create_issue(test)
|
58
|
+
issue, diff_ratio = find_failure_issue(test)
|
59
|
+
|
60
|
+
if issue
|
61
|
+
puts " => Found issue #{issue.web_url} for test '#{test.name}' with a diff ratio of #{(diff_ratio * 100).round(2)}%."
|
62
|
+
else
|
63
|
+
issue = create_issue(test)
|
64
|
+
puts " => Created new issue: #{issue.web_url} for test '#{test.name}'." if issue
|
65
|
+
end
|
66
|
+
|
67
|
+
issue
|
68
|
+
end
|
69
|
+
|
70
|
+
def failure_issues(test)
|
71
|
+
gitlab.find_issues(options: { state: 'opened', labels: 'QA' }).select do |issue|
|
72
|
+
issue_title = issue.title.strip
|
73
|
+
issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
|
78
|
+
ld = Class.new.extend(Gem::Text).method(:levenshtein_distance)
|
79
|
+
first_test_failure_stacktrace = test.failures.first['message_lines'].join("\n")
|
80
|
+
|
81
|
+
# Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
|
82
|
+
failure_issues(test).each_with_object({}) do |issue, memo|
|
83
|
+
relevant_issue_stacktrace = find_issue_stacktrace(issue)
|
84
|
+
unless relevant_issue_stacktrace
|
85
|
+
puts " => [DEBUG] Found issue #{issue.web_url} but stacktrace doesn't match."
|
86
|
+
next
|
87
|
+
end
|
88
|
+
|
89
|
+
distance = ld.call(first_test_failure_stacktrace, relevant_issue_stacktrace)
|
90
|
+
diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
|
91
|
+
if diff_ratio <= max_diff_ratio
|
92
|
+
memo[issue] = diff_ratio
|
93
|
+
else
|
94
|
+
puts " => [DEBUG] Found issue #{issue.web_url} but stacktrace is too different (#{(diff_ratio * 100).round(2)}%)."
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def find_issue_stacktrace(issue)
|
100
|
+
issue_stacktrace_match = issue.description.match(STACKTRACE_REGEX)
|
101
|
+
|
102
|
+
if issue_stacktrace_match
|
103
|
+
issue_stacktrace_match[2].gsub(/^#.*$/, '').strip
|
104
|
+
else
|
105
|
+
puts "\n => Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def find_failure_issue(test)
|
110
|
+
relevant_issues = find_relevant_failure_issues(test)
|
111
|
+
|
112
|
+
return nil if relevant_issues.empty?
|
113
|
+
|
114
|
+
best_matching_issue, smaller_diff_ratio = relevant_issues.min_by { |_, diff_ratio| diff_ratio }
|
115
|
+
|
116
|
+
unless relevant_issues.values.count(smaller_diff_ratio) == 1 # rubocop:disable Style/IfUnlessModifier
|
117
|
+
raise(MultipleIssuesFound, %(Too many issues found for test '#{test.name}' (`#{test.file}`)!))
|
118
|
+
end
|
119
|
+
|
120
|
+
test.failure_issue ||= best_matching_issue.web_url
|
121
|
+
|
122
|
+
[best_matching_issue, smaller_diff_ratio]
|
123
|
+
end
|
124
|
+
|
125
|
+
def new_issue_description(test)
|
126
|
+
super + "\n\n### Stack trace\n\n```\n#{test.failures.first['message_lines'].join("\n")}\n```"
|
127
|
+
end
|
128
|
+
|
129
|
+
def new_issue_labels(test)
|
130
|
+
NEW_ISSUE_LABELS + up_to_date_labels(test: test)
|
131
|
+
end
|
132
|
+
|
133
|
+
def post_failed_job_note(issue, test)
|
134
|
+
gitlab.create_issue_note(iid: issue.iid, note: "/relate #{test.testcase}")
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
@@ -1,13 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'set'
|
4
|
+
|
3
5
|
module Gitlab
|
4
6
|
module QA
|
5
7
|
module Report
|
6
8
|
class ReportAsIssue
|
7
|
-
|
8
|
-
|
9
|
-
|
9
|
+
MAX_TITLE_LENGTH = 255
|
10
|
+
|
11
|
+
def initialize(token:, input_files:, project: nil, dry_run: false, **kwargs)
|
10
12
|
@project = project
|
13
|
+
@gitlab = (dry_run ? GitlabIssueDryClient : GitlabIssueClient).new(token: token, project: project)
|
14
|
+
@files = Array(input_files)
|
11
15
|
end
|
12
16
|
|
13
17
|
def invoke!
|
@@ -24,6 +28,14 @@ module Gitlab
|
|
24
28
|
raise NotImplementedError
|
25
29
|
end
|
26
30
|
|
31
|
+
def new_issue_description(test)
|
32
|
+
"### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def new_issue_labels(test)
|
36
|
+
[]
|
37
|
+
end
|
38
|
+
|
27
39
|
def validate_input!
|
28
40
|
assert_project!
|
29
41
|
assert_input_files!(files)
|
@@ -41,6 +53,92 @@ module Gitlab
|
|
41
53
|
|
42
54
|
abort "Please provide valid JUnit report files. No files were found matching `#{files.join(',')}`"
|
43
55
|
end
|
56
|
+
|
57
|
+
def test_results_per_file
|
58
|
+
Dir.glob(files).each do |path|
|
59
|
+
extension = File.extname(path)
|
60
|
+
|
61
|
+
test_results =
|
62
|
+
case extension
|
63
|
+
when '.json'
|
64
|
+
Report::JsonTestResults.new(path)
|
65
|
+
when '.xml'
|
66
|
+
Report::JUnitTestResults.new(path)
|
67
|
+
else
|
68
|
+
raise "Unknown extension #{extension}"
|
69
|
+
end
|
70
|
+
|
71
|
+
yield test_results
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def create_issue(test)
|
76
|
+
gitlab.create_issue(
|
77
|
+
title: title_from_test(test),
|
78
|
+
description: new_issue_description(test),
|
79
|
+
labels: new_issue_labels(test)
|
80
|
+
)
|
81
|
+
end
|
82
|
+
|
83
|
+
def issue_labels(issue)
|
84
|
+
issue&.labels&.to_set || Set.new
|
85
|
+
end
|
86
|
+
|
87
|
+
def update_labels(issue, test)
|
88
|
+
new_labels = up_to_date_labels(test: test, issue: issue)
|
89
|
+
|
90
|
+
return if issue_labels(issue) == new_labels
|
91
|
+
|
92
|
+
gitlab.edit_issue(iid: issue.iid, options: { labels: new_labels.to_a })
|
93
|
+
end
|
94
|
+
|
95
|
+
def up_to_date_labels(test:, issue: nil)
|
96
|
+
labels = issue_labels(issue)
|
97
|
+
labels << "Enterprise Edition" if ee_test?(test)
|
98
|
+
quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
|
99
|
+
|
100
|
+
labels
|
101
|
+
end
|
102
|
+
|
103
|
+
def ee_test?(test)
|
104
|
+
test.file =~ %r{features/ee/(api|browser_ui)}
|
105
|
+
end
|
106
|
+
|
107
|
+
def quarantine_job?
|
108
|
+
Runtime::Env.ci_job_name&.include?('quarantine')
|
109
|
+
end
|
110
|
+
|
111
|
+
def partial_file_path(path)
|
112
|
+
path.match(/((api|browser_ui).*)/i)[1]
|
113
|
+
end
|
114
|
+
|
115
|
+
def title_from_test(test)
|
116
|
+
title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
|
117
|
+
|
118
|
+
return title unless title.length > MAX_TITLE_LENGTH
|
119
|
+
|
120
|
+
"#{title[0...MAX_TITLE_LENGTH - 3]}..."
|
121
|
+
end
|
122
|
+
|
123
|
+
def search_safe(value)
|
124
|
+
value.delete('"')
|
125
|
+
end
|
126
|
+
|
127
|
+
def pipeline
|
128
|
+
# Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
|
129
|
+
#
|
130
|
+
# Tests can be run in several pipelines:
|
131
|
+
# gitlab-qa, nightly, master, staging, canary, production, preprod, and MRs
|
132
|
+
#
|
133
|
+
# Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
|
134
|
+
# nightly, staging, canary, production, and preprod
|
135
|
+
#
|
136
|
+
# MR, master, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on master
|
137
|
+
# because the other pipelines will be monitored by the author of the MR that triggered them.
|
138
|
+
# So we assume that we're reporting a master pipeline if the project name is 'gitlab-qa'.
|
139
|
+
|
140
|
+
@pipeline ||= Runtime::Env.pipeline_from_project_name
|
141
|
+
end
|
44
142
|
end
|
45
143
|
end
|
46
144
|
end
|
@@ -8,31 +8,19 @@ module Gitlab
|
|
8
8
|
module Report
|
9
9
|
# Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
|
10
10
|
class ResultsInIssues < ReportAsIssue
|
11
|
-
MAX_TITLE_LENGTH = 255
|
12
|
-
|
13
11
|
private
|
14
12
|
|
15
13
|
def run!
|
16
14
|
puts "Reporting test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
17
15
|
|
18
|
-
|
19
|
-
puts "Reporting tests in #{path}"
|
20
|
-
extension = File.extname(path)
|
21
|
-
|
22
|
-
case extension
|
23
|
-
when '.json'
|
24
|
-
test_results = Report::JsonTestResults.new(path)
|
25
|
-
when '.xml'
|
26
|
-
test_results = Report::JUnitTestResults.new(path)
|
27
|
-
else
|
28
|
-
raise "Unknown extension #{extension}"
|
29
|
-
end
|
16
|
+
test_results_per_file do |test_results|
|
17
|
+
puts "Reporting tests in #{test_results.path}"
|
30
18
|
|
31
19
|
test_results.each do |test|
|
32
20
|
report_test(test)
|
33
21
|
end
|
34
22
|
|
35
|
-
test_results.write
|
23
|
+
test_results.write
|
36
24
|
end
|
37
25
|
end
|
38
26
|
|
@@ -72,12 +60,14 @@ module Gitlab
|
|
72
60
|
issues.first
|
73
61
|
end
|
74
62
|
|
75
|
-
def
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
63
|
+
def new_issue_labels(test)
|
64
|
+
%w[status::automated]
|
65
|
+
end
|
66
|
+
|
67
|
+
def up_to_date_labels(test:, issue: nil)
|
68
|
+
labels = super
|
69
|
+
labels.delete_if { |label| label.start_with?("#{pipeline}::") }
|
70
|
+
labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
|
81
71
|
end
|
82
72
|
|
83
73
|
def iid_from_testcase_url(url)
|
@@ -85,23 +75,7 @@ module Gitlab
|
|
85
75
|
end
|
86
76
|
|
87
77
|
def search_term(test)
|
88
|
-
%("#{test.file}" "#{search_safe(test.name)}")
|
89
|
-
end
|
90
|
-
|
91
|
-
def title_from_test(test)
|
92
|
-
title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
|
93
|
-
|
94
|
-
return title unless title.length > MAX_TITLE_LENGTH
|
95
|
-
|
96
|
-
"#{title[0...MAX_TITLE_LENGTH - 3]}..."
|
97
|
-
end
|
98
|
-
|
99
|
-
def partial_file_path(path)
|
100
|
-
path.match(/((api|browser_ui).*)/i)[1]
|
101
|
-
end
|
102
|
-
|
103
|
-
def search_safe(value)
|
104
|
-
value.delete('"')
|
78
|
+
%("#{partial_file_path(test.file)}" "#{search_safe(test.name)}")
|
105
79
|
end
|
106
80
|
|
107
81
|
def note_status(issue, test)
|
@@ -109,13 +83,11 @@ module Gitlab
|
|
109
83
|
|
110
84
|
note = note_content(test)
|
111
85
|
|
112
|
-
gitlab.
|
113
|
-
|
114
|
-
return add_note_to_discussion(issue.iid, discussion.id) if new_note_matches_discussion?(note, discussion)
|
115
|
-
end
|
116
|
-
|
117
|
-
Gitlab.create_issue_note(project, issue.iid, note)
|
86
|
+
gitlab.find_issue_discussions(iid: issue.iid).each do |discussion|
|
87
|
+
return gitlab.add_note_to_issue_discussion_as_thread(iid: issue.iid, discussion_id: discussion.id, body: failure_summary) if new_note_matches_discussion?(note, discussion)
|
118
88
|
end
|
89
|
+
|
90
|
+
gitlab.create_issue_note(iid: issue.iid, note: note)
|
119
91
|
end
|
120
92
|
|
121
93
|
def note_content(test)
|
@@ -143,10 +115,6 @@ module Gitlab
|
|
143
115
|
summary.join(' ')
|
144
116
|
end
|
145
117
|
|
146
|
-
def quarantine_job?
|
147
|
-
Runtime::Env.ci_job_name&.include?('quarantine')
|
148
|
-
end
|
149
|
-
|
150
118
|
def new_note_matches_discussion?(note, discussion)
|
151
119
|
note_error = error_and_stack_trace(note)
|
152
120
|
discussion_error = error_and_stack_trace(discussion.notes.first['body'])
|
@@ -163,42 +131,6 @@ module Gitlab
|
|
163
131
|
|
164
132
|
result
|
165
133
|
end
|
166
|
-
|
167
|
-
def add_note_to_discussion(issue_iid, discussion_id)
|
168
|
-
gitlab.handle_gitlab_client_exceptions do
|
169
|
-
Gitlab.add_note_to_issue_discussion_as_thread(project, issue_iid, discussion_id, body: failure_summary)
|
170
|
-
end
|
171
|
-
end
|
172
|
-
|
173
|
-
def update_labels(issue, test)
|
174
|
-
labels = issue.labels
|
175
|
-
labels.delete_if { |label| label.start_with?("#{pipeline}::") }
|
176
|
-
labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
|
177
|
-
labels << "Enterprise Edition" if ee_test?(test)
|
178
|
-
quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
|
179
|
-
|
180
|
-
gitlab.edit_issue(iid: issue.iid, options: { labels: labels })
|
181
|
-
end
|
182
|
-
|
183
|
-
def ee_test?(test)
|
184
|
-
test.file =~ %r{features/ee/(api|browser_ui)}
|
185
|
-
end
|
186
|
-
|
187
|
-
def pipeline
|
188
|
-
# Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
|
189
|
-
#
|
190
|
-
# Tests can be run in several pipelines:
|
191
|
-
# gitlab-qa, nightly, master, staging, canary, production, preprod, and MRs
|
192
|
-
#
|
193
|
-
# Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
|
194
|
-
# nightly, staging, canary, production, and preprod
|
195
|
-
#
|
196
|
-
# MR, master, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on master
|
197
|
-
# because the other pipelines will be monitored by the author of the MR that triggered them.
|
198
|
-
# So we assume that we're reporting a master pipeline if the project name is 'gitlab-qa'.
|
199
|
-
|
200
|
-
Runtime::Env.pipeline_from_project_name
|
201
|
-
end
|
202
134
|
end
|
203
135
|
end
|
204
136
|
end
|
@@ -19,6 +19,10 @@ module Gitlab
|
|
19
19
|
self.failures = failures_from_exceptions
|
20
20
|
end
|
21
21
|
|
22
|
+
def stage
|
23
|
+
@stage ||= file[%r{(?:api|browser_ui)/(?:(?:\d+_)?(\w+))}, 1]
|
24
|
+
end
|
25
|
+
|
22
26
|
def name
|
23
27
|
raise NotImplementedError
|
24
28
|
end
|
@@ -46,8 +50,16 @@ module Gitlab
|
|
46
50
|
report['file_path']
|
47
51
|
end
|
48
52
|
|
53
|
+
def status
|
54
|
+
report['status']
|
55
|
+
end
|
56
|
+
|
57
|
+
def ci_job_url
|
58
|
+
report['ci_job_url']
|
59
|
+
end
|
60
|
+
|
49
61
|
def skipped
|
50
|
-
|
62
|
+
status == 'pending'
|
51
63
|
end
|
52
64
|
|
53
65
|
def testcase
|
@@ -58,6 +70,14 @@ module Gitlab
|
|
58
70
|
report['testcase'] = new_testcase
|
59
71
|
end
|
60
72
|
|
73
|
+
def failure_issue
|
74
|
+
report['failure_issue']
|
75
|
+
end
|
76
|
+
|
77
|
+
def failure_issue=(new_failure_issue)
|
78
|
+
report['failure_issue'] = new_failure_issue
|
79
|
+
end
|
80
|
+
|
61
81
|
private
|
62
82
|
|
63
83
|
# rubocop:disable Metrics/AbcSize
|
@@ -71,6 +91,7 @@ module Gitlab
|
|
71
91
|
|
72
92
|
{
|
73
93
|
'message' => "#{exception['class']}: #{exception['message']}",
|
94
|
+
'message_lines' => exception['message_lines'],
|
74
95
|
'stacktrace' => "#{exception['message_lines'].join("\n")}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}"
|
75
96
|
}
|
76
97
|
end
|
data/lib/gitlab/qa/reporter.rb
CHANGED
@@ -4,6 +4,7 @@ module Gitlab
|
|
4
4
|
module QA
|
5
5
|
class Reporter
|
6
6
|
# rubocop:disable Metrics/AbcSize
|
7
|
+
# rubocop:disable Metrics/PerceivedComplexity
|
7
8
|
def self.invoke(args)
|
8
9
|
report_options = {}
|
9
10
|
slack_options = {}
|
@@ -21,11 +22,25 @@ module Gitlab
|
|
21
22
|
report_options[:input_files] = files if files
|
22
23
|
end
|
23
24
|
|
24
|
-
opts.on('-
|
25
|
+
opts.on('--relate-failure-issue FILES', String, 'Relate test failures to failure issues from RSpec JSON files') do |files|
|
26
|
+
report_options[:relate_failure_issue] = true
|
27
|
+
report_options[:input_files] = files if files
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on('--max-diff-ratio DIFF_RATO', Float, 'Max stacktrace diff ratio for QA failure issues detection. Used by with --relate-failure-issue') do |value|
|
31
|
+
report_options[:max_diff_ratio] = value
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on('-p', '--project PROJECT_ID', String, 'A valid project ID. Can be an integer or a group/project string. Required by --report-in-issues and --relate-failure-issue') do |value|
|
25
35
|
report_options[:project] = value
|
26
36
|
end
|
27
37
|
|
28
|
-
opts.on('-
|
38
|
+
opts.on('--generate-test-session FILES', String, 'Generate test session report') do |files|
|
39
|
+
report_options[:generate_test_session] = true
|
40
|
+
report_options[:input_files] = files if files
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on('-t', '--token ACCESS_TOKEN', String, 'A valid access token. Required by --report-in-issues and --relate-failure-issue') do |value|
|
29
44
|
report_options[:token] = value
|
30
45
|
end
|
31
46
|
|
@@ -45,6 +60,10 @@ module Gitlab
|
|
45
60
|
report_options[:files] = files
|
46
61
|
end
|
47
62
|
|
63
|
+
opts.on('--dry-run', "Perform a dry-run (don't create or update issues)") do |files|
|
64
|
+
report_options[:dry_run] = true
|
65
|
+
end
|
66
|
+
|
48
67
|
opts.on_tail('-v', '--version', 'Show the version') do
|
49
68
|
require 'gitlab/qa/version'
|
50
69
|
puts "#{$PROGRAM_NAME} : #{VERSION}"
|
@@ -63,10 +82,18 @@ module Gitlab
|
|
63
82
|
if report_options.delete(:prepare_stage_reports)
|
64
83
|
Gitlab::QA::Report::PrepareStageReports.new(**report_options).invoke!
|
65
84
|
|
85
|
+
elsif report_options.delete(:relate_failure_issue)
|
86
|
+
report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
|
87
|
+
Gitlab::QA::Report::RelateFailureIssue.new(**report_options).invoke!
|
88
|
+
|
66
89
|
elsif report_options.delete(:report_in_issues)
|
67
90
|
report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
|
68
91
|
Gitlab::QA::Report::ResultsInIssues.new(**report_options).invoke!
|
69
92
|
|
93
|
+
elsif report_options.delete(:generate_test_session)
|
94
|
+
report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
|
95
|
+
Gitlab::QA::Report::GenerateTestSession.new(**report_options).invoke!
|
96
|
+
|
70
97
|
elsif slack_options.delete(:post_to_slack)
|
71
98
|
Gitlab::QA::Slack::PostToSlack.new(**slack_options).invoke!
|
72
99
|
|
@@ -79,6 +106,7 @@ module Gitlab
|
|
79
106
|
exit 1
|
80
107
|
end
|
81
108
|
end
|
109
|
+
# rubocop:enable Metrics/PerceivedComplexity
|
82
110
|
# rubocop:enable Metrics/AbcSize
|
83
111
|
end
|
84
112
|
end
|
@@ -62,7 +62,6 @@ module Gitlab
|
|
62
62
|
'KNAPSACK_TEST_FILE_PATTERN' => :knapsack_test_file_pattern,
|
63
63
|
'KNAPSACK_TEST_DIR' => :knapsack_test_dir,
|
64
64
|
'CI' => :ci,
|
65
|
-
'CI_JOB_ID' => :ci_job_id,
|
66
65
|
'CI_JOB_URL' => :ci_job_url,
|
67
66
|
'CI_RUNNER_ID' => :ci_runner_id,
|
68
67
|
'CI_SERVER_HOST' => :ci_server_host,
|
@@ -91,7 +90,10 @@ module Gitlab
|
|
91
90
|
attr_writer(method_name)
|
92
91
|
|
93
92
|
define_method(method_name) do
|
94
|
-
ENV[env_name] ||
|
93
|
+
ENV[env_name] ||
|
94
|
+
if instance_variable_defined?("@#{method_name}")
|
95
|
+
instance_variable_get("@#{method_name}")
|
96
|
+
end
|
95
97
|
end
|
96
98
|
end
|
97
99
|
|
@@ -123,12 +125,28 @@ module Gitlab
|
|
123
125
|
ENV['CI_PIPELINE_SOURCE']
|
124
126
|
end
|
125
127
|
|
128
|
+
def ci_pipeline_url
|
129
|
+
ENV['CI_PIPELINE_URL']
|
130
|
+
end
|
131
|
+
|
132
|
+
def ci_pipeline_id
|
133
|
+
ENV['CI_PIPELINE_ID']
|
134
|
+
end
|
135
|
+
|
126
136
|
def ci_project_name
|
127
137
|
ENV['CI_PROJECT_NAME']
|
128
138
|
end
|
129
139
|
|
130
140
|
def pipeline_from_project_name
|
131
|
-
ci_project_name.to_s.start_with?('gitlab-qa')
|
141
|
+
if ci_project_name.to_s.start_with?('gitlab-qa')
|
142
|
+
if ENV['TOP_UPSTREAM_SOURCE_JOB'].to_s.start_with?('https://ops.gitlab.net')
|
143
|
+
'staging-orchestrated'
|
144
|
+
else
|
145
|
+
'master'
|
146
|
+
end
|
147
|
+
else
|
148
|
+
ci_project_name
|
149
|
+
end
|
132
150
|
end
|
133
151
|
|
134
152
|
def run_id
|
@@ -147,6 +165,14 @@ module Gitlab
|
|
147
165
|
ENV['GITLAB_QA_CONTAINER_REGISTRY_ACCESS_TOKEN']
|
148
166
|
end
|
149
167
|
|
168
|
+
def qa_issue_url
|
169
|
+
ENV['GITLAB_QA_ISSUE_URL']
|
170
|
+
end
|
171
|
+
|
172
|
+
def deploy_environment
|
173
|
+
ENV['DEPLOY_ENVIRONMENT'] || pipeline_from_project_name
|
174
|
+
end
|
175
|
+
|
150
176
|
def host_artifacts_dir
|
151
177
|
@host_artifacts_dir ||= File.join(ENV['QA_ARTIFACTS_DIR'] || '/tmp/gitlab-qa', Runtime::Env.run_id)
|
152
178
|
end
|
data/lib/gitlab/qa/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.
|
4
|
+
version: 6.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Grzegorz Bizon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-11-
|
11
|
+
date: 2020-11-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -259,10 +259,13 @@ files:
|
|
259
259
|
- lib/gitlab/qa/docker/volumes.rb
|
260
260
|
- lib/gitlab/qa/release.rb
|
261
261
|
- lib/gitlab/qa/report/base_test_results.rb
|
262
|
+
- lib/gitlab/qa/report/generate_test_session.rb
|
262
263
|
- lib/gitlab/qa/report/gitlab_issue_client.rb
|
264
|
+
- lib/gitlab/qa/report/gitlab_issue_dry_client.rb
|
263
265
|
- lib/gitlab/qa/report/json_test_results.rb
|
264
266
|
- lib/gitlab/qa/report/junit_test_results.rb
|
265
267
|
- lib/gitlab/qa/report/prepare_stage_reports.rb
|
268
|
+
- lib/gitlab/qa/report/relate_failure_issue.rb
|
266
269
|
- lib/gitlab/qa/report/report_as_issue.rb
|
267
270
|
- lib/gitlab/qa/report/results_in_issues.rb
|
268
271
|
- lib/gitlab/qa/report/summary_table.rb
|