gitlab-qa 6.6.0 → 6.10.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +29 -3
- data/.rubocop.yml +2 -0
- data/.rubocop_todo.yml +98 -0
- data/gitlab-qa.gemspec +5 -5
- data/lib/gitlab/qa.rb +3 -0
- data/lib/gitlab/qa/component/base.rb +8 -1
- data/lib/gitlab/qa/docker/engine.rb +8 -0
- data/lib/gitlab/qa/docker/shellout.rb +1 -1
- data/lib/gitlab/qa/report/base_test_results.rb +6 -3
- data/lib/gitlab/qa/report/generate_test_session.rb +203 -0
- data/lib/gitlab/qa/report/gitlab_issue_client.rb +24 -7
- data/lib/gitlab/qa/report/gitlab_issue_dry_client.rb +28 -0
- data/lib/gitlab/qa/report/json_test_results.rb +2 -2
- data/lib/gitlab/qa/report/junit_test_results.rb +2 -2
- data/lib/gitlab/qa/report/relate_failure_issue.rb +169 -0
- data/lib/gitlab/qa/report/report_as_issue.rb +101 -3
- data/lib/gitlab/qa/report/results_in_issues.rb +31 -95
- data/lib/gitlab/qa/report/test_result.rb +31 -1
- data/lib/gitlab/qa/reporter.rb +30 -2
- data/lib/gitlab/qa/runtime/env.rb +35 -3
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +18 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f7798682f401a1349003d47532109adc5523cca0d4dfb43a0036669d0432168f
|
4
|
+
data.tar.gz: eaa7d08e48b241b2d2a521f493543f40beebaef4dcb86e04584eb34cbfbba1c8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fd9312eb0cf93c3ec671b1115d96ae3926fe977de8c11134817fa90cdfb431b88015ce0c28dab8009492f7a14cba173ec13ffca8e466115e37e1b547f5cbe56d
|
7
|
+
data.tar.gz: cd2810c3e560b25337835d9f8e324549bff222cd45529ae7bd372351284a7080c33aaa611bfe95605f2cb12b3a7c958f272436d2e587b98fa8a425cdfdccfd90
|
data/.gitlab-ci.yml
CHANGED
@@ -2,10 +2,11 @@ stages:
|
|
2
2
|
- check
|
3
3
|
- release
|
4
4
|
- test
|
5
|
+
- report
|
5
6
|
- notify
|
6
7
|
|
7
8
|
default:
|
8
|
-
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-ruby-2.
|
9
|
+
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-ruby-2.7
|
9
10
|
tags:
|
10
11
|
- gitlab-org
|
11
12
|
cache:
|
@@ -48,6 +49,10 @@ variables:
|
|
48
49
|
QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
|
49
50
|
QA_CAN_TEST_PRAEFECT: "false"
|
50
51
|
QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
|
52
|
+
QA_FAILURES_REPORTING_PROJECT: "rymai/gitlab-qa-issues"
|
53
|
+
# The --dry-run or --max-diff-ratio option can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
|
54
|
+
QA_FAILURES_REPORTER_OPTIONS: ""
|
55
|
+
QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
|
51
56
|
|
52
57
|
.check-base:
|
53
58
|
stage: check
|
@@ -95,6 +100,7 @@ release:
|
|
95
100
|
- exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
96
101
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
97
102
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
103
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
98
104
|
- exit $test_run_exit_code
|
99
105
|
|
100
106
|
.ce-qa:
|
@@ -333,6 +339,7 @@ ce:update:
|
|
333
339
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
334
340
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
335
341
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
342
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
336
343
|
- exit $test_run_exit_code
|
337
344
|
extends:
|
338
345
|
- .test
|
@@ -347,6 +354,7 @@ ce:update-quarantine:
|
|
347
354
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
348
355
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
349
356
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
357
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
350
358
|
- exit $test_run_exit_code
|
351
359
|
extends:
|
352
360
|
- .test
|
@@ -360,6 +368,7 @@ ee:update:
|
|
360
368
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
361
369
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
362
370
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
371
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
363
372
|
- exit $test_run_exit_code
|
364
373
|
extends:
|
365
374
|
- .test
|
@@ -374,6 +383,7 @@ ee:update-quarantine:
|
|
374
383
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
375
384
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
376
385
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
386
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
377
387
|
- exit $test_run_exit_code
|
378
388
|
extends:
|
379
389
|
- .test
|
@@ -941,6 +951,22 @@ staging:
|
|
941
951
|
- .only-qa
|
942
952
|
allow_failure: true
|
943
953
|
|
954
|
+
generate_test_session:
|
955
|
+
stage: report
|
956
|
+
rules:
|
957
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == "master"'
|
958
|
+
when: always
|
959
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB =~ /\Ahttps:\/\/ops.gitlab.net\//'
|
960
|
+
when: always
|
961
|
+
artifacts:
|
962
|
+
when: always
|
963
|
+
expire_in: 10d
|
964
|
+
paths:
|
965
|
+
- REPORT_ISSUE_URL
|
966
|
+
script:
|
967
|
+
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
968
|
+
- exe/gitlab-qa-report --generate-test-session "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASE_SESSIONS_PROJECT"
|
969
|
+
|
944
970
|
.notify_upstream_commit:
|
945
971
|
stage: notify
|
946
972
|
image: ruby:2.6
|
@@ -966,7 +992,7 @@ notify_upstream_commit:failure:
|
|
966
992
|
.notify_slack:
|
967
993
|
image: alpine
|
968
994
|
stage: notify
|
969
|
-
dependencies: []
|
995
|
+
dependencies: ['generate_test_session']
|
970
996
|
cache: {}
|
971
997
|
before_script:
|
972
998
|
- apk update && apk add git curl bash
|
@@ -985,4 +1011,4 @@ notify_slack:
|
|
985
1011
|
- echo "RELEASE is ${RELEASE}"
|
986
1012
|
- echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
|
987
1013
|
- echo "TOP_UPSTREAM_SOURCE_JOB is $TOP_UPSTREAM_SOURCE_JOB"
|
988
|
-
- bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing
|
1014
|
+
- 'bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See the test session report: $(cat REPORT_ISSUE_URL), and pipeline: $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing'
|
data/.rubocop.yml
CHANGED
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config`
|
3
|
+
# on 2020-10-31 07:26:03 -0700 using RuboCop version 0.82.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 51
|
10
|
+
# Cop supports --auto-correct.
|
11
|
+
# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
12
|
+
# URISchemes: http, https
|
13
|
+
Layout/LineLength:
|
14
|
+
Max: 210
|
15
|
+
|
16
|
+
# Offense count: 2
|
17
|
+
# Cop supports --auto-correct.
|
18
|
+
Lint/RedundantCopDisableDirective:
|
19
|
+
Exclude:
|
20
|
+
- 'lib/gitlab/qa/component/staging.rb'
|
21
|
+
- 'lib/gitlab/qa/runtime/scenario.rb'
|
22
|
+
|
23
|
+
# Offense count: 1
|
24
|
+
# Cop supports --auto-correct.
|
25
|
+
# Configuration parameters: PreferredName.
|
26
|
+
Naming/RescuedExceptionsVariableName:
|
27
|
+
Exclude:
|
28
|
+
- 'lib/gitlab/qa/component/staging.rb'
|
29
|
+
|
30
|
+
# Offense count: 1
|
31
|
+
# Cop supports --auto-correct.
|
32
|
+
Performance/RegexpMatch:
|
33
|
+
Exclude:
|
34
|
+
- 'lib/gitlab/qa/component/gitlab.rb'
|
35
|
+
|
36
|
+
# Offense count: 3
|
37
|
+
# Cop supports --auto-correct.
|
38
|
+
RSpec/EmptyLineAfterLetBlock:
|
39
|
+
Exclude:
|
40
|
+
- 'spec/gitlab/qa/support/dev_eeqa_image_spec.rb'
|
41
|
+
|
42
|
+
# Offense count: 4
|
43
|
+
# Cop supports --auto-correct.
|
44
|
+
# Configuration parameters: CustomTransform, IgnoredWords.
|
45
|
+
RSpec/ExampleWording:
|
46
|
+
Exclude:
|
47
|
+
- 'spec/gitlab/qa/component/gitlab_spec.rb'
|
48
|
+
|
49
|
+
# Offense count: 2
|
50
|
+
RSpec/LeakyConstantDeclaration:
|
51
|
+
Exclude:
|
52
|
+
- 'spec/gitlab/qa/scenario/test/instance/deployment_base_spec.rb'
|
53
|
+
|
54
|
+
# Offense count: 93
|
55
|
+
# Cop supports --auto-correct.
|
56
|
+
# Configuration parameters: EnforcedStyle.
|
57
|
+
# SupportedStyles: always, always_true, never
|
58
|
+
Style/FrozenStringLiteralComment:
|
59
|
+
Enabled: false
|
60
|
+
|
61
|
+
# Offense count: 1
|
62
|
+
# Cop supports --auto-correct.
|
63
|
+
Style/HashTransformKeys:
|
64
|
+
Exclude:
|
65
|
+
- 'lib/gitlab/qa/docker/volumes.rb'
|
66
|
+
|
67
|
+
# Offense count: 6
|
68
|
+
# Cop supports --auto-correct.
|
69
|
+
Style/IfUnlessModifier:
|
70
|
+
Exclude:
|
71
|
+
- 'lib/gitlab/qa/component/gitlab.rb'
|
72
|
+
- 'lib/gitlab/qa/release.rb'
|
73
|
+
- 'lib/gitlab/qa/runtime/env.rb'
|
74
|
+
- 'lib/gitlab/qa/runtime/scenario.rb'
|
75
|
+
- 'lib/gitlab/qa/scenario/test/omnibus/upgrade.rb'
|
76
|
+
- 'lib/gitlab/qa/support/http_request.rb'
|
77
|
+
|
78
|
+
# Offense count: 1
|
79
|
+
Style/MethodMissingSuper:
|
80
|
+
Exclude:
|
81
|
+
- 'lib/gitlab/qa/runtime/scenario.rb'
|
82
|
+
|
83
|
+
# Offense count: 1
|
84
|
+
Style/MissingRespondToMissing:
|
85
|
+
Exclude:
|
86
|
+
- 'lib/gitlab/qa/runtime/scenario.rb'
|
87
|
+
|
88
|
+
# Offense count: 5
|
89
|
+
# Cop supports --auto-correct.
|
90
|
+
# Configuration parameters: EnforcedStyle.
|
91
|
+
# SupportedStyles: literals, strict
|
92
|
+
Style/MutableConstant:
|
93
|
+
Exclude:
|
94
|
+
- 'db/migrate/**/*'
|
95
|
+
- 'db/post_migrate/**/*'
|
96
|
+
- 'db/geo/migrate/**/*'
|
97
|
+
- 'lib/gitlab/qa/release.rb'
|
98
|
+
- 'lib/gitlab/qa/report/update_screenshot_path.rb'
|
data/gitlab-qa.gemspec
CHANGED
@@ -20,15 +20,15 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
# Some dependencies are pinned, to prevent new cops from breaking the CI pipelines
|
22
22
|
spec.add_development_dependency 'climate_control', '~> 0.2'
|
23
|
-
spec.add_development_dependency 'gitlab-styles', '
|
23
|
+
spec.add_development_dependency 'gitlab-styles', '~> 4.3.0'
|
24
24
|
spec.add_development_dependency 'pry', '~> 0.11'
|
25
25
|
spec.add_development_dependency 'rake', '~> 12.2'
|
26
26
|
spec.add_development_dependency 'rspec', '~> 3.7'
|
27
|
-
spec.add_development_dependency 'rubocop', '~> 0.
|
28
|
-
spec.add_development_dependency 'rubocop-rspec', '1.
|
27
|
+
spec.add_development_dependency 'rubocop', '~> 0.82.0'
|
28
|
+
spec.add_development_dependency 'rubocop-rspec', '~> 1.36'
|
29
29
|
spec.add_development_dependency 'webmock', '3.7.0'
|
30
|
-
spec.add_runtime_dependency 'activesupport', '~>
|
31
|
-
spec.add_runtime_dependency 'gitlab', '~>
|
30
|
+
spec.add_runtime_dependency 'activesupport', '~> 6.0.2'
|
31
|
+
spec.add_runtime_dependency 'gitlab', '~> 4.16.1'
|
32
32
|
spec.add_runtime_dependency 'http', '4.3.0'
|
33
33
|
spec.add_runtime_dependency 'nokogiri', '~> 1.10'
|
34
34
|
spec.add_runtime_dependency 'table_print', '1.5.6'
|
data/lib/gitlab/qa.rb
CHANGED
@@ -99,12 +99,15 @@ module Gitlab
|
|
99
99
|
|
100
100
|
module Report
|
101
101
|
autoload :GitlabIssueClient, 'gitlab/qa/report/gitlab_issue_client'
|
102
|
+
autoload :GitlabIssueDryClient, 'gitlab/qa/report/gitlab_issue_dry_client'
|
102
103
|
autoload :BaseTestResults, 'gitlab/qa/report/base_test_results'
|
104
|
+
autoload :RelateFailureIssue, 'gitlab/qa/report/relate_failure_issue'
|
103
105
|
autoload :JsonTestResults, 'gitlab/qa/report/json_test_results'
|
104
106
|
autoload :JUnitTestResults, 'gitlab/qa/report/junit_test_results'
|
105
107
|
autoload :PrepareStageReports, 'gitlab/qa/report/prepare_stage_reports'
|
106
108
|
autoload :ReportAsIssue, 'gitlab/qa/report/report_as_issue'
|
107
109
|
autoload :ResultsInIssues, 'gitlab/qa/report/results_in_issues'
|
110
|
+
autoload :GenerateTestSession, 'gitlab/qa/report/generate_test_session'
|
108
111
|
autoload :SummaryTable, 'gitlab/qa/report/summary_table'
|
109
112
|
autoload :TestResult, 'gitlab/qa/report/test_result'
|
110
113
|
autoload :UpdateScreenshotPath, 'gitlab/qa/report/update_screenshot_path'
|
@@ -53,6 +53,7 @@ module Gitlab
|
|
53
53
|
|
54
54
|
def prepare
|
55
55
|
prepare_docker_image
|
56
|
+
prepare_docker_container
|
56
57
|
prepare_network
|
57
58
|
end
|
58
59
|
|
@@ -61,7 +62,7 @@ module Gitlab
|
|
61
62
|
end
|
62
63
|
|
63
64
|
def prepare_network
|
64
|
-
if runner_network && !docker.network_exists?(runner_network)
|
65
|
+
if runner_network && !docker.network_exists?(runner_network) # rubocop:disable Style/IfUnlessModifier
|
65
66
|
docker.network_create("--driver=bridge --internal #{runner_network}")
|
66
67
|
end
|
67
68
|
|
@@ -70,6 +71,12 @@ module Gitlab
|
|
70
71
|
docker.network_create(network)
|
71
72
|
end
|
72
73
|
|
74
|
+
def prepare_docker_container
|
75
|
+
return unless docker.container_exists?(name)
|
76
|
+
|
77
|
+
docker.remove(name)
|
78
|
+
end
|
79
|
+
|
73
80
|
def start # rubocop:disable Metrics/AbcSize
|
74
81
|
docker.run(image, tag) do |command|
|
75
82
|
command << "-d"
|
@@ -63,6 +63,14 @@ module Gitlab
|
|
63
63
|
Docker::Command.execute("rm -f #{name}")
|
64
64
|
end
|
65
65
|
|
66
|
+
def container_exists?(name)
|
67
|
+
Docker::Command.execute("container inspect #{name}")
|
68
|
+
rescue Docker::Shellout::StatusError
|
69
|
+
false
|
70
|
+
else
|
71
|
+
true
|
72
|
+
end
|
73
|
+
|
66
74
|
def network_exists?(name)
|
67
75
|
Docker::Command.execute("network inspect #{name}")
|
68
76
|
rescue Docker::Shellout::StatusError
|
@@ -27,7 +27,7 @@ module Gitlab
|
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
-
if wait.value.exited? && wait.value.exitstatus.nonzero?
|
30
|
+
if wait.value.exited? && wait.value.exitstatus.nonzero? # rubocop:disable Style/IfUnlessModifier
|
31
31
|
raise StatusError, "Docker command `#{@command.mask_secrets}` failed!"
|
32
32
|
end
|
33
33
|
end
|
@@ -6,8 +6,11 @@ module Gitlab
|
|
6
6
|
class BaseTestResults
|
7
7
|
include Enumerable
|
8
8
|
|
9
|
+
attr_reader :path
|
10
|
+
|
9
11
|
def initialize(path)
|
10
|
-
@
|
12
|
+
@path = path
|
13
|
+
@results = parse
|
11
14
|
@testcases = process
|
12
15
|
end
|
13
16
|
|
@@ -15,7 +18,7 @@ module Gitlab
|
|
15
18
|
testcases.each(&block)
|
16
19
|
end
|
17
20
|
|
18
|
-
def write
|
21
|
+
def write
|
19
22
|
raise NotImplementedError
|
20
23
|
end
|
21
24
|
|
@@ -23,7 +26,7 @@ module Gitlab
|
|
23
26
|
|
24
27
|
attr_reader :results, :testcases
|
25
28
|
|
26
|
-
def parse
|
29
|
+
def parse
|
27
30
|
raise NotImplementedError
|
28
31
|
end
|
29
32
|
|
@@ -0,0 +1,203 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Report
|
6
|
+
class GenerateTestSession < ReportAsIssue
|
7
|
+
private
|
8
|
+
|
9
|
+
def run!
|
10
|
+
puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
11
|
+
|
12
|
+
tests = Dir.glob(files).flat_map do |path|
|
13
|
+
puts "Loading tests in #{path}"
|
14
|
+
|
15
|
+
Report::JsonTestResults.new(path).to_a
|
16
|
+
end
|
17
|
+
|
18
|
+
issue = gitlab.create_issue(
|
19
|
+
title: "Test session report | #{Runtime::Env.deploy_environment}",
|
20
|
+
description: generate_description(tests),
|
21
|
+
labels: ['Quality', 'QA', 'triage report']
|
22
|
+
)
|
23
|
+
|
24
|
+
File.write('REPORT_ISSUE_URL', issue.web_url)
|
25
|
+
end
|
26
|
+
|
27
|
+
def generate_description(tests)
|
28
|
+
<<~MARKDOWN
|
29
|
+
## Session summary
|
30
|
+
|
31
|
+
* Deploy version: #{Runtime::Env.deploy_version}
|
32
|
+
* Pipeline: [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
|
33
|
+
#{generate_summary(tests: tests)}
|
34
|
+
|
35
|
+
#{generate_stages_listing(tests)}
|
36
|
+
|
37
|
+
## Release QA issue
|
38
|
+
|
39
|
+
* #{Runtime::Env.qa_issue_url}
|
40
|
+
MARKDOWN
|
41
|
+
end
|
42
|
+
|
43
|
+
def generate_summary(tests:, tests_by_status: nil)
|
44
|
+
tests_by_status ||= tests.group_by(&:status)
|
45
|
+
total = tests.size
|
46
|
+
passed = tests_by_status['passed']&.size || 0
|
47
|
+
failed = tests_by_status['failed']&.size || 0
|
48
|
+
others = total - passed - failed
|
49
|
+
|
50
|
+
<<~MARKDOWN.chomp
|
51
|
+
* Total #{total} tests
|
52
|
+
* Passed #{passed} tests
|
53
|
+
* Failed #{failed} tests
|
54
|
+
* #{others} other tests (usually skipped)
|
55
|
+
MARKDOWN
|
56
|
+
end
|
57
|
+
|
58
|
+
def generate_stages_listing(tests)
|
59
|
+
generate_tests_by_stage(tests).map do |stage, tests_for_stage|
|
60
|
+
tests_by_status = tests_for_stage.group_by(&:status)
|
61
|
+
|
62
|
+
<<~MARKDOWN.chomp
|
63
|
+
### #{stage&.capitalize || 'Unknown'}
|
64
|
+
|
65
|
+
#{generate_summary(
|
66
|
+
tests: tests_for_stage, tests_by_status: tests_by_status)}
|
67
|
+
|
68
|
+
#{generate_testcase_listing_by_status(
|
69
|
+
tests: tests_for_stage, tests_by_status: tests_by_status)}
|
70
|
+
MARKDOWN
|
71
|
+
end.join("\n\n")
|
72
|
+
end
|
73
|
+
|
74
|
+
def generate_tests_by_stage(tests)
|
75
|
+
# https://about.gitlab.com/handbook/product/product-categories/#devops-stages
|
76
|
+
ordering = %w[
|
77
|
+
manage
|
78
|
+
plan
|
79
|
+
create
|
80
|
+
verify
|
81
|
+
package
|
82
|
+
release
|
83
|
+
configure
|
84
|
+
monitor
|
85
|
+
secure
|
86
|
+
defend
|
87
|
+
growth
|
88
|
+
fulfillment
|
89
|
+
enablement
|
90
|
+
]
|
91
|
+
|
92
|
+
tests.sort_by do |test|
|
93
|
+
ordering.index(test.stage) || ordering.size
|
94
|
+
end.group_by(&:stage)
|
95
|
+
end
|
96
|
+
|
97
|
+
def generate_testcase_listing_by_status(tests:, tests_by_status:)
|
98
|
+
failed_tests = tests_by_status['failed']
|
99
|
+
passed_tests = tests_by_status['passed']
|
100
|
+
other_tests = tests.reject do |test|
|
101
|
+
test.status == 'failed' || test.status == 'passed'
|
102
|
+
end
|
103
|
+
|
104
|
+
[
|
105
|
+
(failed_listings(failed_tests) if failed_tests),
|
106
|
+
(passed_listings(passed_tests) if passed_tests),
|
107
|
+
(other_listings(other_tests) if other_tests.any?)
|
108
|
+
].compact.join("\n\n")
|
109
|
+
end
|
110
|
+
|
111
|
+
def failed_listings(failed_tests)
|
112
|
+
generate_testcase_listing(failed_tests)
|
113
|
+
end
|
114
|
+
|
115
|
+
def passed_listings(passed_tests)
|
116
|
+
<<~MARKDOWN.chomp
|
117
|
+
<details><summary>Passed tests:</summary>
|
118
|
+
|
119
|
+
#{generate_testcase_listing(passed_tests)}
|
120
|
+
|
121
|
+
</details>
|
122
|
+
MARKDOWN
|
123
|
+
end
|
124
|
+
|
125
|
+
def other_listings(other_tests)
|
126
|
+
<<~MARKDOWN.chomp
|
127
|
+
<details><summary>Other tests:</summary>
|
128
|
+
|
129
|
+
#{generate_testcase_listing(other_tests)}
|
130
|
+
|
131
|
+
</details>
|
132
|
+
MARKDOWN
|
133
|
+
end
|
134
|
+
|
135
|
+
def generate_testcase_listing(tests)
|
136
|
+
body = tests.group_by(&:testcase).map do |testcase, tests_with_same_testcase|
|
137
|
+
tests_with_same_testcase.sort_by!(&:name)
|
138
|
+
[
|
139
|
+
generate_test_text(testcase, tests_with_same_testcase),
|
140
|
+
generate_test_job(tests_with_same_testcase),
|
141
|
+
generate_test_status(tests_with_same_testcase),
|
142
|
+
generate_test_actions(tests_with_same_testcase)
|
143
|
+
].join(' | ')
|
144
|
+
end.join("\n")
|
145
|
+
|
146
|
+
<<~MARKDOWN.chomp
|
147
|
+
| Test | Job | Status | Action |
|
148
|
+
| - | - | - | - |
|
149
|
+
#{body}
|
150
|
+
MARKDOWN
|
151
|
+
end
|
152
|
+
|
153
|
+
def generate_test_text(testcase, tests_with_same_testcase)
|
154
|
+
text = tests_with_same_testcase.map(&:name).uniq.join(', ')
|
155
|
+
|
156
|
+
if testcase
|
157
|
+
"[#{text}](#{testcase})"
|
158
|
+
else
|
159
|
+
text
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
def generate_test_job(tests_with_same_testcase)
|
164
|
+
tests_with_same_testcase.map do |test|
|
165
|
+
ci_job_id = test.ci_job_url[/\d+\z/]
|
166
|
+
|
167
|
+
"[#{ci_job_id}](#{test.ci_job_url})#{' ~"quarantine"' if test.quarantine?}"
|
168
|
+
end.uniq.join(', ')
|
169
|
+
end
|
170
|
+
|
171
|
+
def generate_test_status(tests_with_same_testcase)
|
172
|
+
tests_with_same_testcase.map(&:status).uniq.map do |status|
|
173
|
+
%(~"#{status}")
|
174
|
+
end.join(', ')
|
175
|
+
end
|
176
|
+
|
177
|
+
def generate_test_actions(tests_with_same_testcase)
|
178
|
+
# All failed tests would be grouped together, meaning that
|
179
|
+
# if one failed, all the tests here would be failed too.
|
180
|
+
# So this check is safe. Same applies to 'passed'.
|
181
|
+
# But all other status might be mixing together,
|
182
|
+
# we cannot assume other statuses.
|
183
|
+
if tests_with_same_testcase.first.status == 'failed'
|
184
|
+
tests_having_failure_issue =
|
185
|
+
tests_with_same_testcase.select(&:failure_issue)
|
186
|
+
|
187
|
+
if tests_having_failure_issue.any?
|
188
|
+
items = tests_having_failure_issue.map do |test|
|
189
|
+
"<li>[ ] [failure issue](#{test.failure_issue})</li>"
|
190
|
+
end.join(' ')
|
191
|
+
|
192
|
+
"<ul>#{items}</ul>"
|
193
|
+
else
|
194
|
+
'<ul><li>[ ] failure issue exists or was created</li></ul>'
|
195
|
+
end
|
196
|
+
else
|
197
|
+
'-'
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|